DBZ-7083 Introduce strategy for MySQL and MariaDB
This commit is contained in:
parent
e57c607bcf
commit
1d6a1ec6c1
@ -176,6 +176,7 @@
|
|||||||
<mysql.init.timeout>60000</mysql.init.timeout> <!-- 60 seconds -->
|
<mysql.init.timeout>60000</mysql.init.timeout> <!-- 60 seconds -->
|
||||||
<apicurio.port>8080</apicurio.port>
|
<apicurio.port>8080</apicurio.port>
|
||||||
<apicurio.init.timeout>60000</apicurio.init.timeout> <!-- 60 seconds -->
|
<apicurio.init.timeout>60000</apicurio.init.timeout> <!-- 60 seconds -->
|
||||||
|
<mysql.database.adapter>mysql</mysql.database.adapter>
|
||||||
<mysql.database.protocol>jdbc:mysql</mysql.database.protocol>
|
<mysql.database.protocol>jdbc:mysql</mysql.database.protocol>
|
||||||
<mysql.database.jdbc.driver>com.mysql.cj.jdbc.Driver</mysql.database.jdbc.driver>
|
<mysql.database.jdbc.driver>com.mysql.cj.jdbc.Driver</mysql.database.jdbc.driver>
|
||||||
<!--
|
<!--
|
||||||
@ -548,6 +549,7 @@
|
|||||||
<database.replica.hostname>${docker.host.address}</database.replica.hostname>
|
<database.replica.hostname>${docker.host.address}</database.replica.hostname>
|
||||||
<database.replica.port>${mysql.replica.port}</database.replica.port>
|
<database.replica.port>${mysql.replica.port}</database.replica.port>
|
||||||
<!-- Specifies which driver to use for the tests -->
|
<!-- Specifies which driver to use for the tests -->
|
||||||
|
<connector.adapter>${mysql.database.adapter}</connector.adapter>
|
||||||
<database.protocol>${mysql.database.protocol}</database.protocol>
|
<database.protocol>${mysql.database.protocol}</database.protocol>
|
||||||
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
||||||
<skipLongRunningTests>${skipLongRunningTests}</skipLongRunningTests>
|
<skipLongRunningTests>${skipLongRunningTests}</skipLongRunningTests>
|
||||||
@ -696,6 +698,7 @@
|
|||||||
<database.replica.port>${mysql.port}</database.replica.port>
|
<database.replica.port>${mysql.port}</database.replica.port>
|
||||||
<database.ssl.mode>disabled</database.ssl.mode>
|
<database.ssl.mode>disabled</database.ssl.mode>
|
||||||
<!-- Specifies which driver to use for the tests -->
|
<!-- Specifies which driver to use for the tests -->
|
||||||
|
<connector.adapter>${mysql.database.adapter}</connector.adapter>
|
||||||
<database.protocol>${mysql.database.protocol}</database.protocol>
|
<database.protocol>${mysql.database.protocol}</database.protocol>
|
||||||
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
||||||
<skipLongRunningTests>false</skipLongRunningTests>
|
<skipLongRunningTests>false</skipLongRunningTests>
|
||||||
@ -716,6 +719,7 @@
|
|||||||
<database.port>${mysql.port}</database.port>
|
<database.port>${mysql.port}</database.port>
|
||||||
<database.replica.port>${mysql.port}</database.replica.port>
|
<database.replica.port>${mysql.port}</database.replica.port>
|
||||||
<!-- Specifies which driver to use for the tests -->
|
<!-- Specifies which driver to use for the tests -->
|
||||||
|
<connector.adapter>${mysql.database.adapter}</connector.adapter>
|
||||||
<database.protocol>${mysql.database.protocol}</database.protocol>
|
<database.protocol>${mysql.database.protocol}</database.protocol>
|
||||||
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
||||||
</systemPropertyVariables>
|
</systemPropertyVariables>
|
||||||
@ -732,6 +736,7 @@
|
|||||||
<database.port>${mysql.gtid.port}</database.port>
|
<database.port>${mysql.gtid.port}</database.port>
|
||||||
<database.replica.port>${mysql.gtid.replica.port}</database.replica.port>
|
<database.replica.port>${mysql.gtid.replica.port}</database.replica.port>
|
||||||
<!-- Specifies which driver to use for the tests -->
|
<!-- Specifies which driver to use for the tests -->
|
||||||
|
<connector.adapter>${mysql.database.adapter}</connector.adapter>
|
||||||
<database.protocol>${mysql.database.protocol}</database.protocol>
|
<database.protocol>${mysql.database.protocol}</database.protocol>
|
||||||
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
||||||
</systemPropertyVariables>
|
</systemPropertyVariables>
|
||||||
@ -768,6 +773,7 @@
|
|||||||
<database.port>${mysql.ssl.port}</database.port>
|
<database.port>${mysql.ssl.port}</database.port>
|
||||||
<database.replica.port>${mysql.ssl.port}</database.replica.port>
|
<database.replica.port>${mysql.ssl.port}</database.replica.port>
|
||||||
<!-- Specifies which driver to use for the tests -->
|
<!-- Specifies which driver to use for the tests -->
|
||||||
|
<connector.adapter>${mysql.database.adapter}</connector.adapter>
|
||||||
<database.protocol>${mysql.database.protocol}</database.protocol>
|
<database.protocol>${mysql.database.protocol}</database.protocol>
|
||||||
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
||||||
</systemPropertyVariables>
|
</systemPropertyVariables>
|
||||||
@ -932,23 +938,9 @@
|
|||||||
<activeByDefault>false</activeByDefault>
|
<activeByDefault>false</activeByDefault>
|
||||||
</activation>
|
</activation>
|
||||||
<properties>
|
<properties>
|
||||||
<docker.dbs>debezium/mysql-server-test-database</docker.dbs>
|
|
||||||
<mysql.server.image.source>${mariadb.server.image.source}</mysql.server.image.source>
|
|
||||||
<version.mysql.server>${version.mariadb.server}</version.mysql.server>
|
|
||||||
<mysql.database.protocol>jdbc:mariadb</mysql.database.protocol>
|
|
||||||
<mysql.database.jdbc.driver>org.mariadb.jdbc.Driver</mysql.database.jdbc.driver>
|
|
||||||
</properties>
|
|
||||||
</profile>
|
|
||||||
<!-- Profile for MariaDB SQL support -->
|
|
||||||
<profile>
|
|
||||||
<id>mariadb-ssl</id>
|
|
||||||
<activation>
|
|
||||||
<activeByDefault>false</activeByDefault>
|
|
||||||
</activation>
|
|
||||||
<properties>
|
|
||||||
<docker.dbs>debezium/mysql-server-test-database-ssl</docker.dbs>
|
|
||||||
<mysql.server.image.source>${mariadb.server.image.source}</mysql.server.image.source>
|
<mysql.server.image.source>${mariadb.server.image.source}</mysql.server.image.source>
|
||||||
<version.mysql.server>${version.mariadb.server}</version.mysql.server>
|
<version.mysql.server>${version.mariadb.server}</version.mysql.server>
|
||||||
|
<mysql.database.adapter>mariadb</mysql.database.adapter>
|
||||||
<mysql.database.protocol>jdbc:mariadb</mysql.database.protocol>
|
<mysql.database.protocol>jdbc:mariadb</mysql.database.protocol>
|
||||||
<mysql.database.jdbc.driver>org.mariadb.jdbc.Driver</mysql.database.jdbc.driver>
|
<mysql.database.jdbc.driver>org.mariadb.jdbc.Driver</mysql.database.jdbc.driver>
|
||||||
</properties>
|
</properties>
|
||||||
@ -1016,6 +1008,7 @@
|
|||||||
<!-- Specifies which driver to use for the tests -->
|
<!-- Specifies which driver to use for the tests -->
|
||||||
<database.protocol>${mysql.database.protocol}</database.protocol>
|
<database.protocol>${mysql.database.protocol}</database.protocol>
|
||||||
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
||||||
|
<connector.adapter>${mysql.database.adapter}</connector.adapter>
|
||||||
<skipLongRunningTests>false</skipLongRunningTests>
|
<skipLongRunningTests>false</skipLongRunningTests>
|
||||||
<isAssemblyProfileActive>true</isAssemblyProfileActive>
|
<isAssemblyProfileActive>true</isAssemblyProfileActive>
|
||||||
</systemPropertyVariables>
|
</systemPropertyVariables>
|
||||||
@ -1096,6 +1089,7 @@
|
|||||||
<!-- Specifies which driver to use for the tests -->
|
<!-- Specifies which driver to use for the tests -->
|
||||||
<database.protocol>${mysql.database.protocol}</database.protocol>
|
<database.protocol>${mysql.database.protocol}</database.protocol>
|
||||||
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
||||||
|
<connector.adapter>${mysql.database.adapter}</connector.adapter>
|
||||||
<skipLongRunningTests>false</skipLongRunningTests>
|
<skipLongRunningTests>false</skipLongRunningTests>
|
||||||
<isAssemblyProfileActive>true</isAssemblyProfileActive>
|
<isAssemblyProfileActive>true</isAssemblyProfileActive>
|
||||||
</systemPropertyVariables>
|
</systemPropertyVariables>
|
||||||
@ -1174,6 +1168,7 @@
|
|||||||
<!-- Specifies which driver to use for the tests -->
|
<!-- Specifies which driver to use for the tests -->
|
||||||
<database.protocol>${mysql.database.protocol}</database.protocol>
|
<database.protocol>${mysql.database.protocol}</database.protocol>
|
||||||
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
||||||
|
<connector.adapter>${mysql.database.adapter}</connector.adapter>
|
||||||
<skipLongRunningTests>false</skipLongRunningTests>
|
<skipLongRunningTests>false</skipLongRunningTests>
|
||||||
<isAssemblyProfileActive>true</isAssemblyProfileActive>
|
<isAssemblyProfileActive>true</isAssemblyProfileActive>
|
||||||
</systemPropertyVariables>
|
</systemPropertyVariables>
|
||||||
@ -1252,6 +1247,7 @@
|
|||||||
<!-- Specifies which driver to use for the tests -->
|
<!-- Specifies which driver to use for the tests -->
|
||||||
<database.protocol>${mysql.database.protocol}</database.protocol>
|
<database.protocol>${mysql.database.protocol}</database.protocol>
|
||||||
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
<database.jdbc.driver>${mysql.database.jdbc.driver}</database.jdbc.driver>
|
||||||
|
<connector.adapter>${mysql.database.adapter}</connector.adapter>
|
||||||
<skipLongRunningTests>false</skipLongRunningTests>
|
<skipLongRunningTests>false</skipLongRunningTests>
|
||||||
<isAssemblyProfileActive>true</isAssemblyProfileActive>
|
<isAssemblyProfileActive>true</isAssemblyProfileActive>
|
||||||
<database.ssl.truststore>${project.basedir}/src/test/resources/ssl/truststore</database.ssl.truststore>
|
<database.ssl.truststore>${project.basedir}/src/test/resources/ssl/truststore</database.ssl.truststore>
|
||||||
|
@ -87,7 +87,7 @@ public Object readField(ResultSet rs, int columnIndex, Column column, Table tabl
|
|||||||
try {
|
try {
|
||||||
String columnData = rs.getString(columnIndex);
|
String columnData = rs.getString(columnIndex);
|
||||||
if (columnData != null) {
|
if (columnData != null) {
|
||||||
return columnData.getBytes(MySqlConnection.getJavaEncodingForMysqlCharSet(column.charsetName()));
|
return columnData.getBytes(connectorConfig.getConnectorAdapter().getJavaEncodingForCharSet(column.charsetName()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (UnsupportedEncodingException e) {
|
catch (UnsupportedEncodingException e) {
|
||||||
|
@ -5,476 +5,64 @@
|
|||||||
*/
|
*/
|
||||||
package io.debezium.connector.mysql;
|
package io.debezium.connector.mysql;
|
||||||
|
|
||||||
import java.util.AbstractMap;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Iterator;
|
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.TreeMap;
|
|
||||||
import java.util.function.Predicate;
|
import java.util.function.Predicate;
|
||||||
import java.util.regex.Pattern;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import io.debezium.annotation.Immutable;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A set of MySQL GTIDs. This is an improvement of {@link com.github.shyiko.mysql.binlog.GtidSet} that is immutable,
|
* Represents a common contract for GTID behavior for MySQL and MariaDB.
|
||||||
* and more properly supports comparisons.
|
|
||||||
*
|
*
|
||||||
* @author Randall Hauch
|
* @author Randall Hauch, Chris Cranford
|
||||||
*/
|
*/
|
||||||
@Immutable
|
public interface GtidSet {
|
||||||
public final class GtidSet {
|
|
||||||
|
|
||||||
private final Map<String, UUIDSet> uuidSetsByServerId = new TreeMap<>(); // sorts on keys
|
|
||||||
public static Pattern GTID_DELIMITER = Pattern.compile(":");
|
|
||||||
|
|
||||||
protected GtidSet(Map<String, UUIDSet> uuidSetsByServerId) {
|
|
||||||
this.uuidSetsByServerId.putAll(uuidSetsByServerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param gtids the string representation of the GTIDs.
|
* Returns whether this {@link GtidSet} is empty.
|
||||||
*/
|
*/
|
||||||
public GtidSet(String gtids) {
|
boolean isEmpty();
|
||||||
gtids = gtids.replace("\n", "").replace("\r", "");
|
|
||||||
new com.github.shyiko.mysql.binlog.GtidSet(gtids).getUUIDSets().forEach(uuidSet -> {
|
|
||||||
uuidSetsByServerId.put(uuidSet.getUUID(), new UUIDSet(uuidSet));
|
|
||||||
});
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
uuidSetsByServerId.values().forEach(uuidSet -> {
|
|
||||||
if (sb.length() != 0) {
|
|
||||||
sb.append(',');
|
|
||||||
}
|
|
||||||
sb.append(uuidSet.toString());
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Obtain a copy of this {@link GtidSet} except with only the GTID ranges that have server UUIDs that match the given
|
* Obtain a copy of this {@link GtidSet} except with only the GTID ranges match the specified predicate.
|
||||||
* predicate.
|
|
||||||
*
|
*
|
||||||
* @param sourceFilter the predicate that returns whether a server UUID is to be included
|
* @param sourceFilter the predicate that returns whether a server identifier is to be included
|
||||||
* @return the new GtidSet, or this object if {@code sourceFilter} is null; never null
|
* @return the new GtidSet, or this object if {@code sourceFilter} is null; never null
|
||||||
*/
|
*/
|
||||||
public GtidSet retainAll(Predicate<String> sourceFilter) {
|
// todo: change to T
|
||||||
if (sourceFilter == null) {
|
GtidSet retainAll(Predicate<String> sourceFilter);
|
||||||
return this;
|
|
||||||
}
|
|
||||||
Map<String, UUIDSet> newSets = this.uuidSetsByServerId.entrySet()
|
|
||||||
.stream()
|
|
||||||
.filter(entry -> sourceFilter.test(entry.getKey()))
|
|
||||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
|
||||||
return new GtidSet(newSets);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get an immutable collection of the {@link UUIDSet range of GTIDs for a single server}.
|
* Determine whether the GTIDs represented by this object are contained completely within the supplied set.
|
||||||
*
|
|
||||||
* @return the {@link UUIDSet GTID ranges for each server}; never null
|
|
||||||
*/
|
|
||||||
public Collection<UUIDSet> getUUIDSets() {
|
|
||||||
return Collections.unmodifiableCollection(uuidSetsByServerId.values());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find the {@link UUIDSet} for the server with the specified Uuid.
|
|
||||||
*
|
|
||||||
* @param uuid the Uuid of the server
|
|
||||||
* @return the {@link UUIDSet} for the identified server, or {@code null} if there are no GTIDs from that server.
|
|
||||||
*/
|
|
||||||
public UUIDSet forServerWithId(String uuid) {
|
|
||||||
return uuidSetsByServerId.get(uuid);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine if the GTIDs represented by this object are contained completely within the supplied set of GTIDs.
|
|
||||||
*
|
*
|
||||||
* @param other the other set of GTIDs; may be null
|
* @param other the other set of GTIDs; may be null
|
||||||
* @return {@code true} if all of the GTIDs in this set are completely contained within the supplied set of GTIDs, or
|
* @return {@code true} if all GTIDs are present in the provided set, {@code false} otherwise
|
||||||
* {@code false} otherwise
|
|
||||||
*/
|
*/
|
||||||
public boolean isContainedWithin(GtidSet other) {
|
boolean isContainedWithin(GtidSet other);
|
||||||
if (other == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (this.equals(other)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
for (UUIDSet uuidSet : uuidSetsByServerId.values()) {
|
|
||||||
UUIDSet thatSet = other.forServerWithId(uuidSet.getUUID());
|
|
||||||
if (!uuidSet.isContainedWithin(thatSet)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Obtain a copy of this {@link GtidSet} except overwritten with all of the GTID ranges in the supplied {@link GtidSet}.
|
* Obtain a copy of this {@link GtidSet} except overwritten with all the GTID ranges in the supplied {@link GtidSet}.
|
||||||
* @param other the other {@link GtidSet} with ranges to add/overwrite on top of those in this set;
|
*
|
||||||
* @return the new GtidSet, or this object if {@code other} is null or empty; never null
|
* @param other the other {@link GtidSet} with ranges to add/overwrite on top of those in this set
|
||||||
|
* @return the new {@link GtidSet}, or this object if {@code other} is null or empty; never null
|
||||||
*/
|
*/
|
||||||
public GtidSet with(GtidSet other) {
|
GtidSet with(GtidSet other);
|
||||||
if (other == null || other.uuidSetsByServerId.isEmpty()) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
Map<String, UUIDSet> newSet = new HashMap<>();
|
|
||||||
newSet.putAll(this.uuidSetsByServerId);
|
|
||||||
newSet.putAll(other.uuidSetsByServerId);
|
|
||||||
return new GtidSet(newSet);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns a copy with all intervals set to beginning
|
* Returns a copy of this with all intervals set to the beginning.
|
||||||
* @return
|
|
||||||
*/
|
*/
|
||||||
public GtidSet getGtidSetBeginning() {
|
GtidSet getGtidSetBeginning();
|
||||||
Map<String, UUIDSet> newSet = new HashMap<>();
|
|
||||||
|
|
||||||
for (UUIDSet uuidSet : uuidSetsByServerId.values()) {
|
|
||||||
newSet.put(uuidSet.getUUID(), uuidSet.asIntervalBeginning());
|
|
||||||
}
|
|
||||||
|
|
||||||
return new GtidSet(newSet);
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean contains(String gtid) {
|
|
||||||
String[] split = GTID_DELIMITER.split(gtid);
|
|
||||||
String sourceId = split[0];
|
|
||||||
UUIDSet uuidSet = forServerWithId(sourceId);
|
|
||||||
if (uuidSet == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
long transactionId = Long.parseLong(split[1]);
|
|
||||||
return uuidSet.contains(transactionId);
|
|
||||||
}
|
|
||||||
|
|
||||||
public GtidSet subtract(GtidSet other) {
|
|
||||||
if (other == null) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
Map<String, UUIDSet> newSets = this.uuidSetsByServerId.entrySet()
|
|
||||||
.stream()
|
|
||||||
.filter(entry -> !entry.getValue().isContainedWithin(other.forServerWithId(entry.getKey())))
|
|
||||||
.map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey(), entry.getValue().subtract(other.forServerWithId(entry.getKey()))))
|
|
||||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
|
||||||
return new GtidSet(newSets);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int hashCode() {
|
|
||||||
return uuidSetsByServerId.keySet().hashCode();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean equals(Object obj) {
|
|
||||||
if (obj == this) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (obj instanceof GtidSet) {
|
|
||||||
GtidSet that = (GtidSet) obj;
|
|
||||||
return this.uuidSetsByServerId.equals(that.uuidSetsByServerId);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String toString() {
|
|
||||||
List<String> gtids = new ArrayList<String>();
|
|
||||||
for (UUIDSet uuidSet : uuidSetsByServerId.values()) {
|
|
||||||
gtids.add(uuidSet.toString());
|
|
||||||
}
|
|
||||||
return String.join(",", gtids);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A range of GTIDs for a single server with a specific Uuid.
|
* Return whether the specified GTID is present in this set.
|
||||||
|
*
|
||||||
|
* @param gtid the gtid to check; may not be null
|
||||||
|
* @return {@code true} if contained by this set, {@code false} otherwise
|
||||||
*/
|
*/
|
||||||
@Immutable
|
boolean contains(String gtid);
|
||||||
public static class UUIDSet {
|
|
||||||
|
|
||||||
private final String uuid;
|
/**
|
||||||
private final LinkedList<Interval> intervals = new LinkedList<>();
|
* Subtracts the two GTID sets.
|
||||||
|
*
|
||||||
|
* @param other ther other set; may be null
|
||||||
|
* @return a new {@link GtidSet} that contains the difference in GTIDs
|
||||||
|
*/
|
||||||
|
GtidSet subtract(GtidSet other);
|
||||||
|
|
||||||
protected UUIDSet(com.github.shyiko.mysql.binlog.GtidSet.UUIDSet uuidSet) {
|
|
||||||
this.uuid = uuidSet.getUUID();
|
|
||||||
uuidSet.getIntervals().forEach(interval -> {
|
|
||||||
intervals.add(new Interval(interval.getStart(), interval.getEnd()));
|
|
||||||
});
|
|
||||||
Collections.sort(this.intervals);
|
|
||||||
if (this.intervals.size() > 1) {
|
|
||||||
// Collapse adjacent intervals ...
|
|
||||||
for (int i = intervals.size() - 1; i != 0; --i) {
|
|
||||||
Interval before = this.intervals.get(i - 1);
|
|
||||||
Interval after = this.intervals.get(i);
|
|
||||||
if ((before.getEnd() + 1) == after.getStart()) {
|
|
||||||
this.intervals.set(i - 1, new Interval(before.getStart(), after.getEnd()));
|
|
||||||
this.intervals.remove(i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected UUIDSet(String uuid, Interval interval) {
|
|
||||||
this.uuid = uuid;
|
|
||||||
this.intervals.add(interval);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected UUIDSet(String uuid, List<Interval> intervals) {
|
|
||||||
this.uuid = uuid;
|
|
||||||
this.intervals.addAll(intervals);
|
|
||||||
}
|
|
||||||
|
|
||||||
public UUIDSet asIntervalBeginning() {
|
|
||||||
Interval start = new Interval(intervals.get(0).getStart(), intervals.get(0).getStart());
|
|
||||||
return new UUIDSet(this.uuid, start);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the Uuid for the server that generated the GTIDs.
|
|
||||||
*
|
|
||||||
* @return the server's Uuid; never null
|
|
||||||
*/
|
|
||||||
public String getUUID() {
|
|
||||||
return uuid;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the intervals of transaction numbers.
|
|
||||||
*
|
|
||||||
* @return the immutable transaction intervals; never null
|
|
||||||
*/
|
|
||||||
public List<Interval> getIntervals() {
|
|
||||||
return Collections.unmodifiableList(intervals);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine if the set of transaction numbers from this server is completely within the set of transaction numbers from
|
|
||||||
* the set of transaction numbers in the supplied set.
|
|
||||||
*
|
|
||||||
* @param other the set to compare with this set
|
|
||||||
* @return {@code true} if this server's transaction numbers are a subset of the transaction numbers of the supplied set,
|
|
||||||
* or false otherwise
|
|
||||||
*/
|
|
||||||
public boolean isContainedWithin(UUIDSet other) {
|
|
||||||
if (other == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!this.getUUID().equalsIgnoreCase(other.getUUID())) {
|
|
||||||
// Not even the same server ...
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (this.intervals.isEmpty()) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (other.intervals.isEmpty()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
assert this.intervals.size() > 0;
|
|
||||||
assert other.intervals.size() > 0;
|
|
||||||
|
|
||||||
// Every interval in this must be within an interval of the other ...
|
|
||||||
for (Interval thisInterval : this.intervals) {
|
|
||||||
boolean found = false;
|
|
||||||
for (Interval otherInterval : other.intervals) {
|
|
||||||
if (thisInterval.isContainedWithin(otherInterval)) {
|
|
||||||
found = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!found) {
|
|
||||||
return false; // didn't find a match
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean contains(long transactionId) {
|
|
||||||
for (Interval interval : this.intervals) {
|
|
||||||
if (interval.contains(transactionId)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int hashCode() {
|
|
||||||
return uuid.hashCode();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean equals(Object obj) {
|
|
||||||
if (obj == this) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (obj instanceof UUIDSet) {
|
|
||||||
UUIDSet that = (UUIDSet) obj;
|
|
||||||
return this.getUUID().equalsIgnoreCase(that.getUUID()) && this.getIntervals().equals(that.getIntervals());
|
|
||||||
}
|
|
||||||
return super.equals(obj);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String toString() {
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append(uuid).append(':');
|
|
||||||
Iterator<Interval> iter = intervals.iterator();
|
|
||||||
if (iter.hasNext()) {
|
|
||||||
sb.append(iter.next());
|
|
||||||
}
|
|
||||||
while (iter.hasNext()) {
|
|
||||||
sb.append(':');
|
|
||||||
sb.append(iter.next());
|
|
||||||
}
|
|
||||||
return sb.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
public UUIDSet subtract(UUIDSet other) {
|
|
||||||
if (!uuid.equals(other.getUUID())) {
|
|
||||||
throw new IllegalArgumentException("UUIDSet subtraction is supported only within a single server UUID");
|
|
||||||
}
|
|
||||||
List<Interval> result = new ArrayList<>();
|
|
||||||
for (Interval interval : intervals) {
|
|
||||||
result.addAll(interval.removeAll(other.getIntervals()));
|
|
||||||
}
|
|
||||||
return new UUIDSet(uuid, result);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Immutable
|
|
||||||
public static class Interval implements Comparable<Interval> {
|
|
||||||
|
|
||||||
private final long start;
|
|
||||||
private final long end;
|
|
||||||
|
|
||||||
public Interval(long start, long end) {
|
|
||||||
this.start = start;
|
|
||||||
this.end = end;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the starting transaction number in this interval.
|
|
||||||
*
|
|
||||||
* @return this interval's first transaction number
|
|
||||||
*/
|
|
||||||
public long getStart() {
|
|
||||||
return start;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the ending transaction number in this interval.
|
|
||||||
*
|
|
||||||
* @return this interval's last transaction number
|
|
||||||
*/
|
|
||||||
public long getEnd() {
|
|
||||||
return end;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine if this interval is completely within the supplied interval.
|
|
||||||
*
|
|
||||||
* @param other the interval to compare with
|
|
||||||
* @return {@code true} if the {@link #getStart() start} is greater than or equal to the supplied interval's
|
|
||||||
* {@link #getStart() start} and the {@link #getEnd() end} is less than or equal to the supplied interval's
|
|
||||||
* {@link #getEnd() end}, or {@code false} otherwise
|
|
||||||
*/
|
|
||||||
public boolean isContainedWithin(Interval other) {
|
|
||||||
if (other == this) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (other == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return this.getStart() >= other.getStart() && this.getEnd() <= other.getEnd();
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean contains(long transactionId) {
|
|
||||||
return getStart() <= transactionId && transactionId <= getEnd();
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean contains(Interval other) {
|
|
||||||
return getStart() <= other.getStart() && getEnd() >= other.getEnd();
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean nonintersecting(Interval other) {
|
|
||||||
return other.getEnd() < this.getStart() || other.getStart() > this.getEnd();
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<Interval> remove(Interval other) {
|
|
||||||
if (nonintersecting(other)) {
|
|
||||||
return Collections.singletonList(this);
|
|
||||||
}
|
|
||||||
if (other.contains(this)) {
|
|
||||||
return Collections.emptyList();
|
|
||||||
}
|
|
||||||
List<Interval> result = new LinkedList<>();
|
|
||||||
if (this.getStart() < other.getStart()) {
|
|
||||||
Interval part = new Interval(this.getStart(), other.getStart() - 1);
|
|
||||||
result.add(part);
|
|
||||||
}
|
|
||||||
if (other.getEnd() < this.getEnd()) {
|
|
||||||
Interval part = new Interval(other.getEnd() + 1, this.getEnd());
|
|
||||||
result.add(part);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<Interval> removeAll(List<Interval> otherIntervals) {
|
|
||||||
List<Interval> thisIntervals = new LinkedList<>();
|
|
||||||
thisIntervals.add(this);
|
|
||||||
List<Interval> result = new LinkedList<>();
|
|
||||||
result.add(this);
|
|
||||||
for (Interval other : otherIntervals) {
|
|
||||||
result = new LinkedList<>();
|
|
||||||
for (Interval thisInterval : thisIntervals) {
|
|
||||||
result.addAll(thisInterval.remove(other));
|
|
||||||
}
|
|
||||||
thisIntervals = result;
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int compareTo(Interval that) {
|
|
||||||
if (that == this) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
long diff = this.start - that.start;
|
|
||||||
if (diff > Integer.MAX_VALUE) {
|
|
||||||
return Integer.MAX_VALUE;
|
|
||||||
}
|
|
||||||
if (diff < Integer.MIN_VALUE) {
|
|
||||||
return Integer.MIN_VALUE;
|
|
||||||
}
|
|
||||||
return (int) diff;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int hashCode() {
|
|
||||||
return (int) getStart();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean equals(Object obj) {
|
|
||||||
if (this == obj) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (obj instanceof Interval) {
|
|
||||||
Interval that = (Interval) obj;
|
|
||||||
return this.getStart() == that.getStart() && this.getEnd() == that.getEnd();
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String toString() {
|
|
||||||
return "" + getStart() + "-" + getEnd();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
@ -11,6 +11,7 @@
|
|||||||
import org.apache.kafka.connect.source.SourceRecord;
|
import org.apache.kafka.connect.source.SourceRecord;
|
||||||
|
|
||||||
import io.debezium.connector.base.ChangeEventQueue;
|
import io.debezium.connector.base.ChangeEventQueue;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection;
|
||||||
import io.debezium.jdbc.MainConnectionProvidingConnectionFactory;
|
import io.debezium.jdbc.MainConnectionProvidingConnectionFactory;
|
||||||
import io.debezium.pipeline.DataChangeEvent;
|
import io.debezium.pipeline.DataChangeEvent;
|
||||||
import io.debezium.pipeline.ErrorHandler;
|
import io.debezium.pipeline.ErrorHandler;
|
||||||
@ -31,7 +32,7 @@
|
|||||||
public class MySqlChangeEventSourceFactory implements ChangeEventSourceFactory<MySqlPartition, MySqlOffsetContext> {
|
public class MySqlChangeEventSourceFactory implements ChangeEventSourceFactory<MySqlPartition, MySqlOffsetContext> {
|
||||||
|
|
||||||
private final MySqlConnectorConfig configuration;
|
private final MySqlConnectorConfig configuration;
|
||||||
private final MainConnectionProvidingConnectionFactory<MySqlConnection> connectionFactory;
|
private final MainConnectionProvidingConnectionFactory<AbstractConnectorConnection> connectionFactory;
|
||||||
private final ErrorHandler errorHandler;
|
private final ErrorHandler errorHandler;
|
||||||
private final EventDispatcher<MySqlPartition, TableId> dispatcher;
|
private final EventDispatcher<MySqlPartition, TableId> dispatcher;
|
||||||
private final Clock clock;
|
private final Clock clock;
|
||||||
@ -44,7 +45,7 @@ public class MySqlChangeEventSourceFactory implements ChangeEventSourceFactory<M
|
|||||||
// but in the core shared code.
|
// but in the core shared code.
|
||||||
private final ChangeEventQueue<DataChangeEvent> queue;
|
private final ChangeEventQueue<DataChangeEvent> queue;
|
||||||
|
|
||||||
public MySqlChangeEventSourceFactory(MySqlConnectorConfig configuration, MainConnectionProvidingConnectionFactory<MySqlConnection> connectionFactory,
|
public MySqlChangeEventSourceFactory(MySqlConnectorConfig configuration, MainConnectionProvidingConnectionFactory<AbstractConnectorConnection> connectionFactory,
|
||||||
ErrorHandler errorHandler, EventDispatcher<MySqlPartition, TableId> dispatcher, Clock clock, MySqlDatabaseSchema schema,
|
ErrorHandler errorHandler, EventDispatcher<MySqlPartition, TableId> dispatcher, Clock clock, MySqlDatabaseSchema schema,
|
||||||
MySqlTaskContext taskContext, MySqlStreamingChangeEventSourceMetrics streamingMetrics,
|
MySqlTaskContext taskContext, MySqlStreamingChangeEventSourceMetrics streamingMetrics,
|
||||||
ChangeEventQueue<DataChangeEvent> queue) {
|
ChangeEventQueue<DataChangeEvent> queue) {
|
||||||
@ -97,7 +98,7 @@ public StreamingChangeEventSource<MySqlPartition, MySqlOffsetContext> getStreami
|
|||||||
|
|
||||||
if (configuration.isReadOnlyConnection()) {
|
if (configuration.isReadOnlyConnection()) {
|
||||||
if (connectionFactory.mainConnection().isGtidModeEnabled()) {
|
if (connectionFactory.mainConnection().isGtidModeEnabled()) {
|
||||||
return Optional.of(new MySqlReadOnlyIncrementalSnapshotChangeEventSource<>(
|
return Optional.of(configuration.getConnectorAdapter().createIncrementalSnapshotChangeEventSource(
|
||||||
configuration,
|
configuration,
|
||||||
connectionFactory.mainConnection(),
|
connectionFactory.mainConnection(),
|
||||||
dispatcher,
|
dispatcher,
|
||||||
|
@ -1,690 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright Debezium Authors.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
package io.debezium.connector.mysql;
|
|
||||||
|
|
||||||
import static io.debezium.config.CommonConnectorConfig.DATABASE_CONFIG_PREFIX;
|
|
||||||
import static io.debezium.config.CommonConnectorConfig.DRIVER_CONFIG_PREFIX;
|
|
||||||
|
|
||||||
import java.sql.ResultSet;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.OptionalLong;
|
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import com.mysql.cj.CharsetMapping;
|
|
||||||
|
|
||||||
import io.debezium.DebeziumException;
|
|
||||||
import io.debezium.config.CommonConnectorConfig;
|
|
||||||
import io.debezium.config.CommonConnectorConfig.EventProcessingFailureHandlingMode;
|
|
||||||
import io.debezium.config.Configuration;
|
|
||||||
import io.debezium.config.Configuration.Builder;
|
|
||||||
import io.debezium.config.Field;
|
|
||||||
import io.debezium.connector.mysql.MySqlConnectorConfig.SecureConnectionMode;
|
|
||||||
import io.debezium.jdbc.JdbcConfiguration;
|
|
||||||
import io.debezium.jdbc.JdbcConnection;
|
|
||||||
import io.debezium.relational.Column;
|
|
||||||
import io.debezium.relational.Table;
|
|
||||||
import io.debezium.relational.TableId;
|
|
||||||
import io.debezium.util.Strings;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* {@link JdbcConnection} extension to be used with MySQL Server
|
|
||||||
*
|
|
||||||
* @author Jiri Pechanec, Randall Hauch
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
public class MySqlConnection extends JdbcConnection {
|
|
||||||
|
|
||||||
private static Logger LOGGER = LoggerFactory.getLogger(MySqlConnection.class);
|
|
||||||
|
|
||||||
private static final String SQL_SHOW_SYSTEM_VARIABLES = "SHOW VARIABLES";
|
|
||||||
private static final String SQL_SHOW_SYSTEM_VARIABLES_CHARACTER_SET = "SHOW VARIABLES WHERE Variable_name IN ('character_set_server','collation_server')";
|
|
||||||
private static final String SQL_SHOW_SESSION_VARIABLE_SSL_VERSION = "SHOW SESSION STATUS LIKE 'Ssl_version'";
|
|
||||||
private static final String QUOTED_CHARACTER = "`";
|
|
||||||
|
|
||||||
protected static final String URL_PATTERN = "${protocol}://${hostname}:${port}/?useInformationSchema=true&nullCatalogMeansCurrent=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=CONVERT_TO_NULL&connectTimeout=${connectTimeout}";
|
|
||||||
|
|
||||||
private final Map<String, String> originalSystemProperties = new HashMap<>();
|
|
||||||
private final MySqlConnectionConfiguration connectionConfig;
|
|
||||||
private final MySqlFieldReader mysqlFieldReader;
|
|
||||||
|
|
||||||
// Tracks whether this connection is with MariaDB, calculated lazily as needed.
|
|
||||||
private Boolean isMariaDb;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a new connection using the supplied configuration.
|
|
||||||
*
|
|
||||||
* @param connectionConfig {@link MySqlConnectionConfiguration} instance, may not be null.
|
|
||||||
* @param fieldReader binary or text protocol based readers
|
|
||||||
*/
|
|
||||||
public MySqlConnection(MySqlConnectionConfiguration connectionConfig, MySqlFieldReader fieldReader) {
|
|
||||||
super(connectionConfig.jdbcConfig, connectionConfig.factory(), QUOTED_CHARACTER, QUOTED_CHARACTER);
|
|
||||||
this.connectionConfig = connectionConfig;
|
|
||||||
this.mysqlFieldReader = fieldReader;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a new connection using the supplied configuration.
|
|
||||||
*
|
|
||||||
* @param connectionConfig {@link MySqlConnectionConfiguration} instance, may not be null.
|
|
||||||
*/
|
|
||||||
public MySqlConnection(MySqlConnectionConfiguration connectionConfig) {
|
|
||||||
this(connectionConfig, new MySqlTextProtocolFieldReader(null));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void close() throws SQLException {
|
|
||||||
try {
|
|
||||||
super.close();
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
// Reset the system properties to their original value ...
|
|
||||||
originalSystemProperties.forEach((name, value) -> {
|
|
||||||
if (value != null) {
|
|
||||||
System.setProperty(name, value);
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
System.clearProperty(name);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Read the MySQL charset-related system variables.
|
|
||||||
*
|
|
||||||
* @return the system variables that are related to server character sets; never null
|
|
||||||
*/
|
|
||||||
protected Map<String, String> readMySqlCharsetSystemVariables() {
|
|
||||||
// Read the system variables from the MySQL instance and get the current database name ...
|
|
||||||
LOGGER.debug("Reading MySQL charset-related system variables before parsing DDL history.");
|
|
||||||
return querySystemVariables(SQL_SHOW_SYSTEM_VARIABLES_CHARACTER_SET);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Read the MySQL system variables.
|
|
||||||
*
|
|
||||||
* @return the system variables that are related to server character sets; never null
|
|
||||||
*/
|
|
||||||
protected Map<String, String> readMySqlSystemVariables() {
|
|
||||||
// Read the system variables from the MySQL instance and get the current database name ...
|
|
||||||
LOGGER.debug("Reading MySQL system variables");
|
|
||||||
return querySystemVariables(SQL_SHOW_SYSTEM_VARIABLES);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, String> querySystemVariables(String statement) {
|
|
||||||
final Map<String, String> variables = new HashMap<>();
|
|
||||||
try {
|
|
||||||
query(statement, rs -> {
|
|
||||||
while (rs.next()) {
|
|
||||||
String varName = rs.getString(1);
|
|
||||||
String value = rs.getString(2);
|
|
||||||
if (varName != null && value != null) {
|
|
||||||
variables.put(varName, value);
|
|
||||||
LOGGER.debug("\t{} = {}",
|
|
||||||
Strings.pad(varName, 45, ' '),
|
|
||||||
Strings.pad(value, 45, ' '));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Error reading MySQL variables: " + e.getMessage(), e);
|
|
||||||
}
|
|
||||||
|
|
||||||
return variables;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected String setStatementFor(Map<String, String> variables) {
|
|
||||||
StringBuilder sb = new StringBuilder("SET ");
|
|
||||||
boolean first = true;
|
|
||||||
List<String> varNames = new ArrayList<>(variables.keySet());
|
|
||||||
Collections.sort(varNames);
|
|
||||||
for (String varName : varNames) {
|
|
||||||
if (first) {
|
|
||||||
first = false;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
sb.append(", ");
|
|
||||||
}
|
|
||||||
sb.append(varName).append("=");
|
|
||||||
String value = variables.get(varName);
|
|
||||||
if (value == null) {
|
|
||||||
value = "";
|
|
||||||
}
|
|
||||||
if (value.contains(",") || value.contains(";")) {
|
|
||||||
value = "'" + value + "'";
|
|
||||||
}
|
|
||||||
sb.append(value);
|
|
||||||
}
|
|
||||||
return sb.append(";").toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void setSystemProperty(String property, Field field, boolean showValueInError) {
|
|
||||||
String value = connectionConfig.originalConfig().getString(field);
|
|
||||||
if (value != null) {
|
|
||||||
value = value.trim();
|
|
||||||
String existingValue = System.getProperty(property);
|
|
||||||
if (existingValue == null) {
|
|
||||||
// There was no existing property ...
|
|
||||||
String existing = System.setProperty(property, value);
|
|
||||||
originalSystemProperties.put(property, existing); // the existing value may be null
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
existingValue = existingValue.trim();
|
|
||||||
if (!existingValue.equalsIgnoreCase(value)) {
|
|
||||||
// There was an existing property, and the value is different ...
|
|
||||||
String msg = "System or JVM property '" + property + "' is already defined, but the configuration property '"
|
|
||||||
+ field.name()
|
|
||||||
+ "' defines a different value";
|
|
||||||
if (showValueInError) {
|
|
||||||
msg = "System or JVM property '" + property + "' is already defined as " + existingValue
|
|
||||||
+ ", but the configuration property '" + field.name() + "' defines a different value '" + value + "'";
|
|
||||||
}
|
|
||||||
throw new DebeziumException(msg);
|
|
||||||
}
|
|
||||||
// Otherwise, there was an existing property, and the value is exactly the same (so do nothing!)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Read the Ssl Version session variable.
|
|
||||||
*
|
|
||||||
* @return the session variables that are related to sessions ssl version
|
|
||||||
*/
|
|
||||||
protected String getSessionVariableForSslVersion() {
|
|
||||||
final String SSL_VERSION = "Ssl_version";
|
|
||||||
LOGGER.debug("Reading MySQL Session variable for Ssl Version");
|
|
||||||
Map<String, String> sessionVariables = querySystemVariables(SQL_SHOW_SESSION_VARIABLE_SSL_VERSION);
|
|
||||||
if (!sessionVariables.isEmpty() && sessionVariables.containsKey(SSL_VERSION)) {
|
|
||||||
return sessionVariables.get(SSL_VERSION);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine whether the MySQL server has GTIDs enabled.
|
|
||||||
*
|
|
||||||
* @return {@code false} if the server's {@code gtid_mode} is set and is {@code OFF}, or {@code true} otherwise
|
|
||||||
*/
|
|
||||||
public boolean isGtidModeEnabled() {
|
|
||||||
try {
|
|
||||||
return queryAndMap("SHOW GLOBAL VARIABLES LIKE 'GTID_MODE'", rs -> {
|
|
||||||
if (rs.next()) {
|
|
||||||
return "ON".equalsIgnoreCase(rs.getString(2));
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Unexpected error while connecting to MySQL and looking at GTID mode: ", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine the executed GTID set for MySQL.
|
|
||||||
*
|
|
||||||
* @return the string representation of MySQL's GTID sets; never null but an empty string if the server does not use GTIDs
|
|
||||||
*/
|
|
||||||
public String knownGtidSet() {
|
|
||||||
try {
|
|
||||||
return queryAndMap("SHOW MASTER STATUS", rs -> {
|
|
||||||
if (rs.next() && rs.getMetaData().getColumnCount() > 4) {
|
|
||||||
return rs.getString(5); // GTID set, may be null, blank, or contain a GTID set
|
|
||||||
}
|
|
||||||
return "";
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Unexpected error while connecting to MySQL and looking at GTID mode: ", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine the difference between two sets.
|
|
||||||
*
|
|
||||||
* @return a subtraction of two GTID sets; never null
|
|
||||||
*/
|
|
||||||
public GtidSet subtractGtidSet(GtidSet set1, GtidSet set2) {
|
|
||||||
try {
|
|
||||||
return prepareQueryAndMap("SELECT GTID_SUBTRACT(?, ?)",
|
|
||||||
ps -> {
|
|
||||||
ps.setString(1, set1.toString());
|
|
||||||
ps.setString(2, set2.toString());
|
|
||||||
},
|
|
||||||
rs -> {
|
|
||||||
if (rs.next()) {
|
|
||||||
return new GtidSet(rs.getString(1));
|
|
||||||
}
|
|
||||||
return new GtidSet("");
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Unexpected error while connecting to MySQL and looking at GTID mode: ", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the purged GTID values from MySQL (gtid_purged value)
|
|
||||||
*
|
|
||||||
* @return A GTID set; may be empty if not using GTIDs or none have been purged yet
|
|
||||||
*/
|
|
||||||
public GtidSet purgedGtidSet() {
|
|
||||||
try {
|
|
||||||
return queryAndMap("SELECT @@global.gtid_purged", rs -> {
|
|
||||||
if (rs.next() && rs.getMetaData().getColumnCount() > 0) {
|
|
||||||
return new GtidSet(rs.getString(1)); // GTID set, may be null, blank, or contain a GTID set
|
|
||||||
}
|
|
||||||
return new GtidSet("");
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Unexpected error while connecting to MySQL and looking at gtid_purged variable: ", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine if the current user has the named privilege. Note that if the user has the "ALL" privilege this method
|
|
||||||
* returns {@code true}.
|
|
||||||
*
|
|
||||||
* @param grantName the name of the MySQL privilege; may not be null
|
|
||||||
* @return {@code true} if the user has the named privilege, or {@code false} otherwise
|
|
||||||
*/
|
|
||||||
public boolean userHasPrivileges(String grantName) {
|
|
||||||
try {
|
|
||||||
return queryAndMap("SHOW GRANTS FOR CURRENT_USER", rs -> {
|
|
||||||
while (rs.next()) {
|
|
||||||
String grants = rs.getString(1);
|
|
||||||
LOGGER.debug(grants);
|
|
||||||
if (grants == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
grants = grants.toUpperCase();
|
|
||||||
if (grants.contains("ALL") || grants.contains(grantName.toUpperCase())) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Unexpected error while connecting to MySQL and looking at privileges for current user: ", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine the earliest binlog filename that is still available in the server.
|
|
||||||
*
|
|
||||||
* @return the name of the earliest binlog filename, or null if there are none.
|
|
||||||
*/
|
|
||||||
public String earliestBinlogFilename() {
|
|
||||||
// Accumulate the available binlog filenames ...
|
|
||||||
List<String> logNames = new ArrayList<>();
|
|
||||||
try {
|
|
||||||
LOGGER.info("Checking all known binlogs from MySQL");
|
|
||||||
query("SHOW BINARY LOGS", rs -> {
|
|
||||||
while (rs.next()) {
|
|
||||||
logNames.add(rs.getString(1));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Unexpected error while connecting to MySQL and looking for binary logs: ", e);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (logNames.isEmpty()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return logNames.get(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine whether the MySQL server has the binlog_row_image set to 'FULL'.
|
|
||||||
*
|
|
||||||
* @return {@code true} if the server's {@code binlog_row_image} is set to {@code FULL}, or {@code false} otherwise
|
|
||||||
*/
|
|
||||||
protected boolean isBinlogRowImageFull() {
|
|
||||||
try {
|
|
||||||
final String rowImage = queryAndMap("SHOW GLOBAL VARIABLES LIKE 'binlog_row_image'", rs -> {
|
|
||||||
if (rs.next()) {
|
|
||||||
return rs.getString(2);
|
|
||||||
}
|
|
||||||
// This setting was introduced in MySQL 5.6+ with default of 'FULL'.
|
|
||||||
// For older versions, assume 'FULL'.
|
|
||||||
return "FULL";
|
|
||||||
});
|
|
||||||
LOGGER.debug("binlog_row_image={}", rowImage);
|
|
||||||
return "FULL".equalsIgnoreCase(rowImage);
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Unexpected error while connecting to MySQL and looking at BINLOG_ROW_IMAGE mode: ", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine whether the MySQL server has the row-level binlog enabled.
|
|
||||||
*
|
|
||||||
* @return {@code true} if the server's {@code binlog_format} is set to {@code ROW}, or {@code false} otherwise
|
|
||||||
*/
|
|
||||||
protected boolean isBinlogFormatRow() {
|
|
||||||
try {
|
|
||||||
final String mode = queryAndMap("SHOW GLOBAL VARIABLES LIKE 'binlog_format'", rs -> rs.next() ? rs.getString(2) : "");
|
|
||||||
LOGGER.debug("binlog_format={}", mode);
|
|
||||||
return "ROW".equalsIgnoreCase(mode);
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Unexpected error while connecting to MySQL and looking at BINLOG_FORMAT mode: ", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Query the database server to get the list of the binlog files availble.
|
|
||||||
*
|
|
||||||
* @return list of the binlog files
|
|
||||||
*/
|
|
||||||
public List<String> availableBinlogFiles() {
|
|
||||||
List<String> logNames = new ArrayList<>();
|
|
||||||
try {
|
|
||||||
LOGGER.info("Get all known binlogs from MySQL");
|
|
||||||
query("SHOW BINARY LOGS", rs -> {
|
|
||||||
while (rs.next()) {
|
|
||||||
logNames.add(rs.getString(1));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return logNames;
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Unexpected error while connecting to MySQL and looking for binary logs: ", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public OptionalLong getEstimatedTableSize(TableId tableId) {
|
|
||||||
try {
|
|
||||||
// Choose how we create statements based on the # of rows.
|
|
||||||
// This is approximate and less accurate then COUNT(*),
|
|
||||||
// but far more efficient for large InnoDB tables.
|
|
||||||
execute("USE `" + tableId.catalog() + "`;");
|
|
||||||
return queryAndMap("SHOW TABLE STATUS LIKE '" + tableId.table() + "';", rs -> {
|
|
||||||
if (rs.next()) {
|
|
||||||
return OptionalLong.of((rs.getLong(5)));
|
|
||||||
}
|
|
||||||
return OptionalLong.empty();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
LOGGER.debug("Error while getting number of rows in table {}: {}", tableId, e.getMessage(), e);
|
|
||||||
}
|
|
||||||
return OptionalLong.empty();
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isMariaDb() {
|
|
||||||
if (isMariaDb == null) {
|
|
||||||
final String version = querySystemVariables(SQL_SHOW_SYSTEM_VARIABLES).get("version");
|
|
||||||
isMariaDb = version.toLowerCase().contains("mariadb");
|
|
||||||
}
|
|
||||||
return isMariaDb;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isTableIdCaseSensitive() {
|
|
||||||
return !"0".equals(readMySqlSystemVariables().get(MySqlSystemVariables.LOWER_CASE_TABLE_NAMES));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Read the MySQL default character sets for exisiting databases.
|
|
||||||
*
|
|
||||||
* @return the map of database names with their default character sets; never null
|
|
||||||
*/
|
|
||||||
protected Map<String, DatabaseLocales> readDatabaseCollations() {
|
|
||||||
LOGGER.debug("Reading default database charsets");
|
|
||||||
try {
|
|
||||||
return queryAndMap("SELECT schema_name, default_character_set_name, default_collation_name FROM information_schema.schemata", rs -> {
|
|
||||||
final Map<String, DatabaseLocales> charsets = new HashMap<>();
|
|
||||||
while (rs.next()) {
|
|
||||||
String dbName = rs.getString(1);
|
|
||||||
String charset = rs.getString(2);
|
|
||||||
String collation = rs.getString(3);
|
|
||||||
if (dbName != null && (charset != null || collation != null)) {
|
|
||||||
charsets.put(dbName, new DatabaseLocales(charset, collation));
|
|
||||||
LOGGER.debug("\t{} = {}, {}",
|
|
||||||
Strings.pad(dbName, 45, ' '),
|
|
||||||
Strings.pad(charset, 45, ' '),
|
|
||||||
Strings.pad(collation, 45, ' '));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return charsets;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (SQLException e) {
|
|
||||||
throw new DebeziumException("Error reading default database charsets: " + e.getMessage(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public MySqlConnectionConfiguration connectionConfig() {
|
|
||||||
return connectionConfig;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String connectionString() {
|
|
||||||
return connectionString(URL_PATTERN);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String getJavaEncodingForMysqlCharSet(String mysqlCharsetName) {
|
|
||||||
return CharsetMappingWrapper.getJavaEncodingForMysqlCharSet(mysqlCharsetName);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper to gain access to protected method
|
|
||||||
*/
|
|
||||||
private final static class CharsetMappingWrapper extends CharsetMapping {
|
|
||||||
static String getJavaEncodingForMysqlCharSet(String mySqlCharsetName) {
|
|
||||||
return CharsetMapping.getStaticJavaEncodingForMysqlCharset(mySqlCharsetName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class MySqlConnectionConfiguration {
|
|
||||||
|
|
||||||
protected static final String JDBC_PROPERTY_CONNECTION_TIME_ZONE = "connectionTimeZone";
|
|
||||||
protected static final String JDBC_PROPERTY_MARIADB_TIME_ZONE = "timezone";
|
|
||||||
|
|
||||||
private final JdbcConfiguration jdbcConfig;
|
|
||||||
private final ConnectionFactory factory;
|
|
||||||
private final Configuration config;
|
|
||||||
|
|
||||||
public MySqlConnectionConfiguration(Configuration config) {
|
|
||||||
// Set up the JDBC connection without actually connecting, with extra MySQL-specific properties
|
|
||||||
// to give us better JDBC database metadata behavior, including using UTF-8 for the client-side character encoding
|
|
||||||
// per https://dev.mysql.com/doc/connector-j/5.1/en/connector-j-reference-charsets.html
|
|
||||||
this.config = config;
|
|
||||||
final boolean useSSL = sslModeEnabled();
|
|
||||||
final Configuration dbConfig = config
|
|
||||||
.edit()
|
|
||||||
.withDefault(MySqlConnectorConfig.PORT, MySqlConnectorConfig.PORT.defaultValue())
|
|
||||||
.withDefault(MySqlConnectorConfig.JDBC_PROTOCOL, MySqlConnectorConfig.JDBC_PROTOCOL.defaultValue())
|
|
||||||
.build()
|
|
||||||
.subset(DATABASE_CONFIG_PREFIX, true)
|
|
||||||
.merge(config.subset(DRIVER_CONFIG_PREFIX, true));
|
|
||||||
|
|
||||||
final Builder jdbcConfigBuilder = dbConfig
|
|
||||||
.edit()
|
|
||||||
.with("connectTimeout", Long.toString(getConnectionTimeout().toMillis()))
|
|
||||||
.with("sslMode", sslMode().getValue());
|
|
||||||
|
|
||||||
if (useSSL) {
|
|
||||||
if (!Strings.isNullOrBlank(sslTrustStore())) {
|
|
||||||
jdbcConfigBuilder.with("trustCertificateKeyStoreUrl", "file:" + sslTrustStore());
|
|
||||||
}
|
|
||||||
if (sslTrustStorePassword() != null) {
|
|
||||||
jdbcConfigBuilder.with("trustCertificateKeyStorePassword", String.valueOf(sslTrustStorePassword()));
|
|
||||||
}
|
|
||||||
if (!Strings.isNullOrBlank(sslKeyStore())) {
|
|
||||||
jdbcConfigBuilder.with("clientCertificateKeyStoreUrl", "file:" + sslKeyStore());
|
|
||||||
}
|
|
||||||
if (sslKeyStorePassword() != null) {
|
|
||||||
jdbcConfigBuilder.with("clientCertificateKeyStorePassword", String.valueOf(sslKeyStorePassword()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isUsingMariaDbProtocol(config)) {
|
|
||||||
jdbcConfigBuilder.with(JDBC_PROPERTY_MARIADB_TIME_ZONE, determineConnectionTimeZoneForMariaDbDriver(dbConfig));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
jdbcConfigBuilder.with(JDBC_PROPERTY_CONNECTION_TIME_ZONE, determineConnectionTimeZoneForMySqlDriver(dbConfig));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set and remove options to prevent potential vulnerabilities
|
|
||||||
jdbcConfigBuilder
|
|
||||||
.with("allowLoadLocalInfile", "false")
|
|
||||||
.with("allowUrlInLocalInfile", "false")
|
|
||||||
.with("autoDeserialize", false)
|
|
||||||
.without("queryInterceptors");
|
|
||||||
|
|
||||||
this.jdbcConfig = JdbcConfiguration.adapt(jdbcConfigBuilder.build());
|
|
||||||
String driverClassName = this.config.getString(MySqlConnectorConfig.JDBC_DRIVER);
|
|
||||||
Field protocol = MySqlConnectorConfig.JDBC_PROTOCOL;
|
|
||||||
|
|
||||||
factory = JdbcConnection.patternBasedFactory(MySqlConnection.URL_PATTERN, driverClassName, getClass().getClassLoader(), protocol);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static boolean isUsingMariaDbProtocol(Configuration config) {
|
|
||||||
final String jdbcProtocol = config.getString(MySqlConnectorConfig.JDBC_PROTOCOL);
|
|
||||||
return !Strings.isNullOrBlank(jdbcProtocol) && jdbcProtocol.equalsIgnoreCase("jdbc:mariadb");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String determineConnectionTimeZoneForMariaDbDriver(final Configuration dbConfig) {
|
|
||||||
// Debezium by default expected timezone data delivered in server timezone
|
|
||||||
String timezone = dbConfig.getString(JDBC_PROPERTY_MARIADB_TIME_ZONE);
|
|
||||||
return timezone != null ? timezone : "auto";
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String determineConnectionTimeZoneForMySqlDriver(final Configuration dbConfig) {
|
|
||||||
// Debezium by default expects timezoned data delivered in server timezone
|
|
||||||
String connectionTimeZone = dbConfig.getString(JDBC_PROPERTY_CONNECTION_TIME_ZONE);
|
|
||||||
|
|
||||||
if (connectionTimeZone != null) {
|
|
||||||
return connectionTimeZone;
|
|
||||||
}
|
|
||||||
|
|
||||||
return "SERVER";
|
|
||||||
}
|
|
||||||
|
|
||||||
public JdbcConfiguration config() {
|
|
||||||
return jdbcConfig;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Configuration originalConfig() {
|
|
||||||
return config;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ConnectionFactory factory() {
|
|
||||||
return factory;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String username() {
|
|
||||||
return config.getString(MySqlConnectorConfig.USER);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String password() {
|
|
||||||
return config.getString(MySqlConnectorConfig.PASSWORD);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String hostname() {
|
|
||||||
return config.getString(MySqlConnectorConfig.HOSTNAME);
|
|
||||||
}
|
|
||||||
|
|
||||||
public int port() {
|
|
||||||
return config.getInteger(MySqlConnectorConfig.PORT);
|
|
||||||
}
|
|
||||||
|
|
||||||
public SecureConnectionMode sslMode() {
|
|
||||||
String mode = config.getString(MySqlConnectorConfig.SSL_MODE);
|
|
||||||
return SecureConnectionMode.parse(mode);
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean sslModeEnabled() {
|
|
||||||
return sslMode() != SecureConnectionMode.DISABLED;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String sslKeyStore() {
|
|
||||||
return config.getString(MySqlConnectorConfig.SSL_KEYSTORE);
|
|
||||||
}
|
|
||||||
|
|
||||||
public char[] sslKeyStorePassword() {
|
|
||||||
String password = config.getString(MySqlConnectorConfig.SSL_KEYSTORE_PASSWORD);
|
|
||||||
return Strings.isNullOrBlank(password) ? null : password.toCharArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
public String sslTrustStore() {
|
|
||||||
return config.getString(MySqlConnectorConfig.SSL_TRUSTSTORE);
|
|
||||||
}
|
|
||||||
|
|
||||||
public char[] sslTrustStorePassword() {
|
|
||||||
String password = config.getString(MySqlConnectorConfig.SSL_TRUSTSTORE_PASSWORD);
|
|
||||||
return Strings.isNullOrBlank(password) ? null : password.toCharArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
public Duration getConnectionTimeout() {
|
|
||||||
return Duration.ofMillis(config.getLong(MySqlConnectorConfig.CONNECTION_TIMEOUT_MS));
|
|
||||||
}
|
|
||||||
|
|
||||||
public EventProcessingFailureHandlingMode eventProcessingFailureHandlingMode() {
|
|
||||||
String mode = config.getString(CommonConnectorConfig.EVENT_PROCESSING_FAILURE_HANDLING_MODE);
|
|
||||||
if (mode == null) {
|
|
||||||
mode = config.getString(MySqlConnectorConfig.EVENT_DESERIALIZATION_FAILURE_HANDLING_MODE);
|
|
||||||
}
|
|
||||||
return EventProcessingFailureHandlingMode.parse(mode);
|
|
||||||
}
|
|
||||||
|
|
||||||
public EventProcessingFailureHandlingMode inconsistentSchemaHandlingMode() {
|
|
||||||
String mode = config.getString(MySqlConnectorConfig.INCONSISTENT_SCHEMA_HANDLING_MODE);
|
|
||||||
return EventProcessingFailureHandlingMode.parse(mode);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Object getColumnValue(ResultSet rs, int columnIndex, Column column, Table table) throws SQLException {
|
|
||||||
return mysqlFieldReader.readField(rs, columnIndex, column, table);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String quotedTableIdString(TableId tableId) {
|
|
||||||
return tableId.toQuotedString('`');
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class DatabaseLocales {
|
|
||||||
private final String charset;
|
|
||||||
private final String collation;
|
|
||||||
|
|
||||||
public DatabaseLocales(String charset, String collation) {
|
|
||||||
this.charset = charset;
|
|
||||||
this.collation = collation;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void appendToDdlStatement(String dbName, StringBuilder ddl) {
|
|
||||||
if (charset != null) {
|
|
||||||
LOGGER.debug("Setting default charset '{}' for database '{}'", charset, dbName);
|
|
||||||
ddl.append(" CHARSET ").append(charset);
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
LOGGER.info("Default database charset for '{}' not found", dbName);
|
|
||||||
}
|
|
||||||
if (collation != null) {
|
|
||||||
LOGGER.debug("Setting default collation '{}' for database '{}'", collation, dbName);
|
|
||||||
ddl.append(" COLLATE ").append(collation);
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
LOGGER.info("Default database collation for '{}' not found", dbName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -20,7 +20,8 @@
|
|||||||
import io.debezium.annotation.Immutable;
|
import io.debezium.annotation.Immutable;
|
||||||
import io.debezium.config.Configuration;
|
import io.debezium.config.Configuration;
|
||||||
import io.debezium.connector.common.RelationalBaseSourceConnector;
|
import io.debezium.connector.common.RelationalBaseSourceConnector;
|
||||||
import io.debezium.connector.mysql.MySqlConnection.MySqlConnectionConfiguration;
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection;
|
||||||
|
import io.debezium.connector.mysql.strategy.ConnectorAdapter;
|
||||||
import io.debezium.relational.RelationalDatabaseConnectorConfig;
|
import io.debezium.relational.RelationalDatabaseConnectorConfig;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -78,17 +79,25 @@ public ConfigDef config() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void validateConnection(Map<String, ConfigValue> configValues, Configuration config) {
|
protected void validateConnection(Map<String, ConfigValue> configValues, Configuration config) {
|
||||||
|
ConfigValue adapterValue = configValues.get(MySqlConnectorConfig.CONNECTOR_ADAPTER.name());
|
||||||
ConfigValue hostnameValue = configValues.get(RelationalDatabaseConnectorConfig.HOSTNAME.name());
|
ConfigValue hostnameValue = configValues.get(RelationalDatabaseConnectorConfig.HOSTNAME.name());
|
||||||
|
|
||||||
|
ConnectorAdapter adapter = adapter(config);
|
||||||
|
if (adapter == null) {
|
||||||
|
LOGGER.error("Failed to resolve connection adapter.");
|
||||||
|
adapterValue.addErrorMessage("Failed to resolve the connector's connection adapter.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Try to connect to the database ...
|
// Try to connect to the database ...
|
||||||
final MySqlConnectionConfiguration connectionConfig = new MySqlConnectionConfiguration(config);
|
try (AbstractConnectorConnection connection = adapter.createConnection(config)) {
|
||||||
try (MySqlConnection connection = new MySqlConnection(connectionConfig)) {
|
|
||||||
try {
|
try {
|
||||||
connection.connect();
|
connection.connect();
|
||||||
connection.execute("SELECT version()");
|
connection.execute("SELECT version()");
|
||||||
LOGGER.info("Successfully tested connection for {} with user '{}'", connection.connectionString(), connectionConfig.username());
|
LOGGER.info("Successfully tested connection for {} with user '{}'", connection.connectionString(), connection.connectionConfig().username());
|
||||||
}
|
}
|
||||||
catch (SQLException e) {
|
catch (SQLException e) {
|
||||||
LOGGER.error("Failed testing connection for {} with user '{}'", connection.connectionString(), connectionConfig.username(), e);
|
LOGGER.error("Failed testing connection for {} with user '{}'", connection.connectionString(), connection.connectionConfig().username(), e);
|
||||||
hostnameValue.addErrorMessage("Unable to connect: " + e.getMessage());
|
hostnameValue.addErrorMessage("Unable to connect: " + e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -101,4 +110,9 @@ protected void validateConnection(Map<String, ConfigValue> configValues, Configu
|
|||||||
protected Map<String, ConfigValue> validateAllFields(Configuration config) {
|
protected Map<String, ConfigValue> validateAllFields(Configuration config) {
|
||||||
return config.validate(MySqlConnectorConfig.ALL_FIELDS);
|
return config.validate(MySqlConnectorConfig.ALL_FIELDS);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static ConnectorAdapter adapter(Configuration config) {
|
||||||
|
// todo: find a better way to handle this
|
||||||
|
return new MySqlConnectorConfig(config).getConnectorAdapter();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -25,6 +25,10 @@
|
|||||||
import io.debezium.config.Field.ValidationOutput;
|
import io.debezium.config.Field.ValidationOutput;
|
||||||
import io.debezium.connector.AbstractSourceInfo;
|
import io.debezium.connector.AbstractSourceInfo;
|
||||||
import io.debezium.connector.SourceInfoStructMaker;
|
import io.debezium.connector.SourceInfoStructMaker;
|
||||||
|
import io.debezium.connector.mysql.strategy.ConnectorAdapter;
|
||||||
|
import io.debezium.connector.mysql.strategy.mariadb.MariaDbConnectorAdapter;
|
||||||
|
import io.debezium.connector.mysql.strategy.mariadb.hybrid.MariaDbHybridConnectorAdapter;
|
||||||
|
import io.debezium.connector.mysql.strategy.mysql.MySqlConnectorAdapter;
|
||||||
import io.debezium.function.Predicates;
|
import io.debezium.function.Predicates;
|
||||||
import io.debezium.jdbc.JdbcValueConverters.BigIntUnsignedMode;
|
import io.debezium.jdbc.JdbcValueConverters.BigIntUnsignedMode;
|
||||||
import io.debezium.jdbc.TemporalPrecisionMode;
|
import io.debezium.jdbc.TemporalPrecisionMode;
|
||||||
@ -507,6 +511,92 @@ public static SecureConnectionMode parse(String value, String defaultValue) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set of predefined connector adapter modes.
|
||||||
|
*/
|
||||||
|
public enum ConnectorAdapterMode implements EnumeratedValue {
|
||||||
|
/**
|
||||||
|
* Expects the target database to be MySQL using the MySQL driver.
|
||||||
|
* This should also be used if the target database is MySQL compliant but isn't MariaDB.
|
||||||
|
*/
|
||||||
|
MYSQL("mysql") {
|
||||||
|
@Override
|
||||||
|
protected ConnectorAdapter getAdapter(MySqlConnectorConfig connectorConfig) {
|
||||||
|
LOGGER.info("Using " + MySqlConnectorAdapter.class.getName());
|
||||||
|
return new MySqlConnectorAdapter(connectorConfig);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Expects the target database to be MariaDB using the MariaDB driver.
|
||||||
|
*/
|
||||||
|
MARIADB("mariadb") {
|
||||||
|
@Override
|
||||||
|
protected ConnectorAdapter getAdapter(MySqlConnectorConfig connectorConfig) {
|
||||||
|
LOGGER.info("Using " + MariaDbConnectorAdapter.class.getName());
|
||||||
|
return new MariaDbConnectorAdapter(connectorConfig);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Expects the target database to be MariaDB but uses the MySQL driver.
|
||||||
|
*/
|
||||||
|
MARIADB_HYBRID("mariadb-hybrid") {
|
||||||
|
@Override
|
||||||
|
protected ConnectorAdapter getAdapter(MySqlConnectorConfig connectorConfig) {
|
||||||
|
LOGGER.info("Using " + MariaDbHybridConnectorAdapter.class.getName());
|
||||||
|
return new MariaDbHybridConnectorAdapter(connectorConfig);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
private final String value;
|
||||||
|
|
||||||
|
protected abstract ConnectorAdapter getAdapter(MySqlConnectorConfig connectorConfig);
|
||||||
|
|
||||||
|
ConnectorAdapterMode(String value) {
|
||||||
|
this.value = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getValue() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine if the supplied value is one of the predefined options.
|
||||||
|
*
|
||||||
|
* @param value the configuration property value; may not be null
|
||||||
|
* @return the matching option, or null if no match is found
|
||||||
|
*/
|
||||||
|
public static ConnectorAdapterMode parse(String value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
value = value.trim();
|
||||||
|
for (ConnectorAdapterMode option : ConnectorAdapterMode.values()) {
|
||||||
|
if (option.getValue().equalsIgnoreCase(value)) {
|
||||||
|
return option;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine if the supplied value is one of the predefined options.
|
||||||
|
*
|
||||||
|
* @param value the configuration property value; may not be null
|
||||||
|
* @param defaultValue the default value; may be null
|
||||||
|
* @return the matching option, or null if no match is found and the non-null default is invalid
|
||||||
|
*/
|
||||||
|
public static ConnectorAdapterMode parse(String value, String defaultValue) {
|
||||||
|
ConnectorAdapterMode mode = parse(value);
|
||||||
|
if (mode == null && defaultValue != null) {
|
||||||
|
mode = parse(defaultValue);
|
||||||
|
}
|
||||||
|
return mode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@link Integer#MIN_VALUE Minimum value} used for fetch size hint.
|
* {@link Integer#MIN_VALUE Minimum value} used for fetch size hint.
|
||||||
* See <a href="https://issues.jboss.org/browse/DBZ-94">DBZ-94</a> for details.
|
* See <a href="https://issues.jboss.org/browse/DBZ-94">DBZ-94</a> for details.
|
||||||
@ -877,6 +967,14 @@ public static SecureConnectionMode parse(String value, String defaultValue) {
|
|||||||
.withImportance(Importance.LOW)
|
.withImportance(Importance.LOW)
|
||||||
.withDescription("Switched connector to use alternative methods to deliver signals to Debezium instead of writing to signaling table");
|
.withDescription("Switched connector to use alternative methods to deliver signals to Debezium instead of writing to signaling table");
|
||||||
|
|
||||||
|
public static final Field CONNECTOR_ADAPTER = Field.create("connector.adapter")
|
||||||
|
.withDisplayName("Connection adapter to be used")
|
||||||
|
.withEnum(ConnectorAdapterMode.class, ConnectorAdapterMode.MYSQL)
|
||||||
|
.withGroup(Field.createGroupEntry(Field.Group.ADVANCED, 28))
|
||||||
|
.withWidth(Width.SHORT)
|
||||||
|
.withImportance(Importance.MEDIUM)
|
||||||
|
.withDescription("Specifies the connection adapter to be used");
|
||||||
|
|
||||||
public static final Field SOURCE_INFO_STRUCT_MAKER = CommonConnectorConfig.SOURCE_INFO_STRUCT_MAKER
|
public static final Field SOURCE_INFO_STRUCT_MAKER = CommonConnectorConfig.SOURCE_INFO_STRUCT_MAKER
|
||||||
.withDefault(MySqlSourceInfoStructMaker.class.getName());
|
.withDefault(MySqlSourceInfoStructMaker.class.getName());
|
||||||
|
|
||||||
@ -920,7 +1018,8 @@ public static SecureConnectionMode parse(String value, String defaultValue) {
|
|||||||
ROW_COUNT_FOR_STREAMING_RESULT_SETS,
|
ROW_COUNT_FOR_STREAMING_RESULT_SETS,
|
||||||
INCREMENTAL_SNAPSHOT_CHUNK_SIZE,
|
INCREMENTAL_SNAPSHOT_CHUNK_SIZE,
|
||||||
INCREMENTAL_SNAPSHOT_ALLOW_SCHEMA_CHANGES,
|
INCREMENTAL_SNAPSHOT_ALLOW_SCHEMA_CHANGES,
|
||||||
STORE_ONLY_CAPTURED_DATABASES_DDL)
|
STORE_ONLY_CAPTURED_DATABASES_DDL,
|
||||||
|
CONNECTOR_ADAPTER)
|
||||||
.events(
|
.events(
|
||||||
INCLUDE_SQL_QUERY,
|
INCLUDE_SQL_QUERY,
|
||||||
TABLE_IGNORE_BUILTIN,
|
TABLE_IGNORE_BUILTIN,
|
||||||
@ -967,6 +1066,7 @@ protected boolean supportsSchemaChangesDuringIncrementalSnapshot() {
|
|||||||
private final Predicate<String> gtidSourceFilter;
|
private final Predicate<String> gtidSourceFilter;
|
||||||
private final EventProcessingFailureHandlingMode inconsistentSchemaFailureHandlingMode;
|
private final EventProcessingFailureHandlingMode inconsistentSchemaFailureHandlingMode;
|
||||||
private final boolean readOnlyConnection;
|
private final boolean readOnlyConnection;
|
||||||
|
private final ConnectorAdapter connectorAdapter;
|
||||||
|
|
||||||
public MySqlConnectorConfig(Configuration config) {
|
public MySqlConnectorConfig(Configuration config) {
|
||||||
super(
|
super(
|
||||||
@ -999,6 +1099,9 @@ public MySqlConnectorConfig(Configuration config) {
|
|||||||
: (gtidSetExcludes != null ? Predicates.excludesUuids(gtidSetExcludes) : null);
|
: (gtidSetExcludes != null ? Predicates.excludesUuids(gtidSetExcludes) : null);
|
||||||
|
|
||||||
this.storeOnlyCapturedDatabasesDdl = config.getBoolean(STORE_ONLY_CAPTURED_DATABASES_DDL);
|
this.storeOnlyCapturedDatabasesDdl = config.getBoolean(STORE_ONLY_CAPTURED_DATABASES_DDL);
|
||||||
|
|
||||||
|
// This should always be last to guarantee the full configuration is passed in the constructor
|
||||||
|
this.connectorAdapter = ConnectorAdapterMode.parse(config.getString(CONNECTOR_ADAPTER)).getAdapter(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean useCursorFetch() {
|
public boolean useCursorFetch() {
|
||||||
@ -1156,6 +1259,10 @@ public int bufferSizeForStreamingChangeEventSource() {
|
|||||||
return config.getInteger(MySqlConnectorConfig.BUFFER_SIZE_FOR_BINLOG_READER);
|
return config.getInteger(MySqlConnectorConfig.BUFFER_SIZE_FOR_BINLOG_READER);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ConnectorAdapter getConnectorAdapter() {
|
||||||
|
return connectorAdapter;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the predicate function that will return {@code true} if a GTID source is to be included, or {@code false} if
|
* Get the predicate function that will return {@code true} if a GTID source is to be included, or {@code false} if
|
||||||
* a GTID source is to be excluded.
|
* a GTID source is to be excluded.
|
||||||
@ -1180,7 +1287,7 @@ public long rowCountForLargeTable() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected HistoryRecordComparator getHistoryRecordComparator() {
|
protected HistoryRecordComparator getHistoryRecordComparator() {
|
||||||
return new MySqlHistoryRecordComparator(gtidSourceFilter());
|
return connectorAdapter.getHistoryRecordComparator();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static boolean isBuiltInDatabase(String databaseName) {
|
public static boolean isBuiltInDatabase(String databaseName) {
|
||||||
|
@ -19,9 +19,12 @@
|
|||||||
import io.debezium.config.Field;
|
import io.debezium.config.Field;
|
||||||
import io.debezium.connector.base.ChangeEventQueue;
|
import io.debezium.connector.base.ChangeEventQueue;
|
||||||
import io.debezium.connector.common.BaseSourceTask;
|
import io.debezium.connector.common.BaseSourceTask;
|
||||||
import io.debezium.connector.mysql.MySqlConnection.MySqlConnectionConfiguration;
|
|
||||||
import io.debezium.connector.mysql.MySqlConnectorConfig.BigIntUnsignedHandlingMode;
|
import io.debezium.connector.mysql.MySqlConnectorConfig.BigIntUnsignedHandlingMode;
|
||||||
import io.debezium.connector.mysql.MySqlConnectorConfig.SnapshotMode;
|
import io.debezium.connector.mysql.MySqlConnectorConfig.SnapshotMode;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection;
|
||||||
|
import io.debezium.connector.mysql.strategy.ConnectorAdapter;
|
||||||
|
import io.debezium.connector.mysql.strategy.mysql.MySqlConnection;
|
||||||
|
import io.debezium.connector.mysql.strategy.mysql.MySqlConnectionConfiguration;
|
||||||
import io.debezium.document.DocumentReader;
|
import io.debezium.document.DocumentReader;
|
||||||
import io.debezium.jdbc.DefaultMainConnectionProvidingConnectionFactory;
|
import io.debezium.jdbc.DefaultMainConnectionProvidingConnectionFactory;
|
||||||
import io.debezium.jdbc.JdbcValueConverters.BigIntUnsignedMode;
|
import io.debezium.jdbc.JdbcValueConverters.BigIntUnsignedMode;
|
||||||
@ -56,7 +59,7 @@ public class MySqlConnectorTask extends BaseSourceTask<MySqlPartition, MySqlOffs
|
|||||||
|
|
||||||
private volatile MySqlTaskContext taskContext;
|
private volatile MySqlTaskContext taskContext;
|
||||||
private volatile ChangeEventQueue<DataChangeEvent> queue;
|
private volatile ChangeEventQueue<DataChangeEvent> queue;
|
||||||
private volatile MySqlConnection connection;
|
private volatile AbstractConnectorConnection connection;
|
||||||
private volatile ErrorHandler errorHandler;
|
private volatile ErrorHandler errorHandler;
|
||||||
private volatile MySqlDatabaseSchema schema;
|
private volatile MySqlDatabaseSchema schema;
|
||||||
|
|
||||||
@ -81,8 +84,10 @@ public ChangeEventSourceCoordinator<MySqlPartition, MySqlOffsetContext> start(Co
|
|||||||
.withDefault("database.useCursorFetch", connectorConfig.useCursorFetch())
|
.withDefault("database.useCursorFetch", connectorConfig.useCursorFetch())
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
MainConnectionProvidingConnectionFactory<MySqlConnection> connectionFactory = new DefaultMainConnectionProvidingConnectionFactory<>(
|
final ConnectorAdapter adapter = connectorConfig.getConnectorAdapter();
|
||||||
() -> new MySqlConnection(new MySqlConnectionConfiguration(config), getFieldReader(connectorConfig)));
|
|
||||||
|
MainConnectionProvidingConnectionFactory<AbstractConnectorConnection> connectionFactory = new DefaultMainConnectionProvidingConnectionFactory<>(
|
||||||
|
() -> adapter.createConnection(config));
|
||||||
|
|
||||||
connection = connectionFactory.mainConnection();
|
connection = connectionFactory.mainConnection();
|
||||||
|
|
||||||
@ -214,7 +219,7 @@ private MySqlValueConverters getValueConverters(MySqlConnectorConfig configurati
|
|||||||
final boolean timeAdjusterEnabled = configuration.getConfig().getBoolean(MySqlConnectorConfig.ENABLE_TIME_ADJUSTER);
|
final boolean timeAdjusterEnabled = configuration.getConfig().getBoolean(MySqlConnectorConfig.ENABLE_TIME_ADJUSTER);
|
||||||
return new MySqlValueConverters(decimalMode, timePrecisionMode, bigIntUnsignedMode,
|
return new MySqlValueConverters(decimalMode, timePrecisionMode, bigIntUnsignedMode,
|
||||||
configuration.binaryHandlingMode(), timeAdjusterEnabled ? MySqlValueConverters::adjustTemporal : x -> x,
|
configuration.binaryHandlingMode(), timeAdjusterEnabled ? MySqlValueConverters::adjustTemporal : x -> x,
|
||||||
MySqlValueConverters::defaultParsingErrorHandler);
|
MySqlValueConverters::defaultParsingErrorHandler, configuration.getConnectorAdapter());
|
||||||
}
|
}
|
||||||
|
|
||||||
private MySqlFieldReader getFieldReader(MySqlConnectorConfig configuration) {
|
private MySqlFieldReader getFieldReader(MySqlConnectorConfig configuration) {
|
||||||
@ -278,70 +283,6 @@ private void validateBinlogConfiguration(MySqlConnectorConfig config) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine whether the binlog position as set on the {@link MySqlOffsetContext} is available in the server.
|
|
||||||
*
|
|
||||||
* @return {@code true} if the server has the binlog coordinates, or {@code false} otherwise
|
|
||||||
*/
|
|
||||||
protected boolean isBinlogAvailable(MySqlConnectorConfig config, MySqlOffsetContext offset) {
|
|
||||||
String gtidStr = offset.gtidSet();
|
|
||||||
if (gtidStr != null) {
|
|
||||||
if (gtidStr.trim().isEmpty()) {
|
|
||||||
return true; // start at beginning ...
|
|
||||||
}
|
|
||||||
String availableGtidStr = connection.knownGtidSet();
|
|
||||||
if (availableGtidStr == null || availableGtidStr.trim().isEmpty()) {
|
|
||||||
// Last offsets had GTIDs but the server does not use them ...
|
|
||||||
LOGGER.info("Connector used GTIDs previously, but MySQL does not know of any GTIDs or they are not enabled");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// GTIDs are enabled, and we used them previously, but retain only those GTID ranges for the allowed source UUIDs ...
|
|
||||||
GtidSet gtidSet = new GtidSet(gtidStr).retainAll(config.gtidSourceFilter());
|
|
||||||
// Get the GTID set that is available in the server ...
|
|
||||||
GtidSet availableGtidSet = new GtidSet(availableGtidStr);
|
|
||||||
if (gtidSet.isContainedWithin(availableGtidSet)) {
|
|
||||||
LOGGER.info("MySQL current GTID set {} does contain the GTID set required by the connector {}", availableGtidSet, gtidSet);
|
|
||||||
final GtidSet knownServerSet = availableGtidSet.retainAll(config.gtidSourceFilter());
|
|
||||||
final GtidSet gtidSetToReplicate = connection.subtractGtidSet(knownServerSet, gtidSet);
|
|
||||||
final GtidSet purgedGtidSet = connection.purgedGtidSet();
|
|
||||||
LOGGER.info("Server has already purged {} GTIDs", purgedGtidSet);
|
|
||||||
final GtidSet nonPurgedGtidSetToReplicate = connection.subtractGtidSet(gtidSetToReplicate, purgedGtidSet);
|
|
||||||
LOGGER.info("GTIDs known by the server but not processed yet {}, for replication are available only {}", gtidSetToReplicate, nonPurgedGtidSetToReplicate);
|
|
||||||
if (!gtidSetToReplicate.equals(nonPurgedGtidSetToReplicate)) {
|
|
||||||
LOGGER.info("Some of the GTIDs needed to replicate have been already purged");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
LOGGER.info("Connector last known GTIDs are {}, but MySQL has {}", gtidSet, availableGtidSet);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
String binlogFilename = offset.getSource().binlogFilename();
|
|
||||||
if (binlogFilename == null) {
|
|
||||||
return true; // start at current position
|
|
||||||
}
|
|
||||||
if (binlogFilename.equals("")) {
|
|
||||||
return true; // start at beginning
|
|
||||||
}
|
|
||||||
|
|
||||||
// Accumulate the available binlog filenames ...
|
|
||||||
List<String> logNames = connection.availableBinlogFiles();
|
|
||||||
|
|
||||||
// And compare with the one we're supposed to use ...
|
|
||||||
boolean found = logNames.stream().anyMatch(binlogFilename::equals);
|
|
||||||
if (!found) {
|
|
||||||
if (LOGGER.isInfoEnabled()) {
|
|
||||||
LOGGER.info("Connector requires binlog file '{}', but MySQL only has {}", binlogFilename, String.join(", ", logNames));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
LOGGER.info("MySQL has the binlog file '{}' required by the connector", binlogFilename);
|
|
||||||
}
|
|
||||||
|
|
||||||
return found;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean validateAndLoadSchemaHistory(MySqlConnectorConfig config, MySqlPartition partition, MySqlOffsetContext offset, MySqlDatabaseSchema schema) {
|
private boolean validateAndLoadSchemaHistory(MySqlConnectorConfig config, MySqlPartition partition, MySqlOffsetContext offset, MySqlDatabaseSchema schema) {
|
||||||
if (offset == null) {
|
if (offset == null) {
|
||||||
if (config.getSnapshotMode().shouldSnapshotOnSchemaError()) {
|
if (config.getSnapshotMode().shouldSnapshotOnSchemaError()) {
|
||||||
@ -357,7 +298,7 @@ private boolean validateAndLoadSchemaHistory(MySqlConnectorConfig config, MySqlP
|
|||||||
LOGGER.warn("Database schema history was not found but was expected");
|
LOGGER.warn("Database schema history was not found but was expected");
|
||||||
if (config.getSnapshotMode().shouldSnapshotOnSchemaError()) {
|
if (config.getSnapshotMode().shouldSnapshotOnSchemaError()) {
|
||||||
// But check to see if the server still has those binlog coordinates ...
|
// But check to see if the server still has those binlog coordinates ...
|
||||||
if (!isBinlogAvailable(config, offset)) {
|
if (!connection.isBinlogPositionAvailable(config, offset.gtidSet(), offset.getSource().binlogFilename())) {
|
||||||
throw new DebeziumException("The connector is trying to read binlog starting at " + offset.getSource() + ", but this is no longer "
|
throw new DebeziumException("The connector is trying to read binlog starting at " + offset.getSource() + ", but this is no longer "
|
||||||
+ "available on the server. Reconfigure the connector to use a snapshot when needed.");
|
+ "available on the server. Reconfigure the connector to use a snapshot when needed.");
|
||||||
}
|
}
|
||||||
@ -388,7 +329,7 @@ private boolean validateSnapshotFeasibility(MySqlConnectorConfig config, MySqlOf
|
|||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
// But check to see if the server still has those binlog coordinates ...
|
// But check to see if the server still has those binlog coordinates ...
|
||||||
if (!isBinlogAvailable(config, offset)) {
|
if (!connection.isBinlogPositionAvailable(config, offset.gtidSet(), offset.getSource().binlogFilename())) {
|
||||||
if (!config.getSnapshotMode().shouldSnapshotOnDataError()) {
|
if (!config.getSnapshotMode().shouldSnapshotOnDataError()) {
|
||||||
throw new DebeziumException("The connector is trying to read binlog starting at " + offset.getSource() + ", but this is no longer "
|
throw new DebeziumException("The connector is trying to read binlog starting at " + offset.getSource() + ", but this is no longer "
|
||||||
+ "available on the server. Reconfigure the connector to use a snapshot when needed.");
|
+ "available on the server. Reconfigure the connector to use a snapshot when needed.");
|
||||||
@ -422,10 +363,8 @@ private void resetOffset(MySqlConnectorConfig connectorConfig, MySqlOffsetContex
|
|||||||
SignalProcessor<MySqlPartition, MySqlOffsetContext> signalProcessor) {
|
SignalProcessor<MySqlPartition, MySqlOffsetContext> signalProcessor) {
|
||||||
boolean isKafkaChannelEnabled = connectorConfig.getEnabledChannels().contains(KafkaSignalChannel.CHANNEL_NAME);
|
boolean isKafkaChannelEnabled = connectorConfig.getEnabledChannels().contains(KafkaSignalChannel.CHANNEL_NAME);
|
||||||
if (previousOffset != null && isKafkaChannelEnabled && connectorConfig.isReadOnlyConnection()) {
|
if (previousOffset != null && isKafkaChannelEnabled && connectorConfig.isReadOnlyConnection()) {
|
||||||
MySqlReadOnlyIncrementalSnapshotContext<TableId> readOnlyIncrementalSnapshotContext = (MySqlReadOnlyIncrementalSnapshotContext<TableId>) previousOffset
|
|
||||||
.getIncrementalSnapshotContext();
|
|
||||||
KafkaSignalChannel kafkaSignal = signalProcessor.getSignalChannel(KafkaSignalChannel.class);
|
KafkaSignalChannel kafkaSignal = signalProcessor.getSignalChannel(KafkaSignalChannel.class);
|
||||||
Long signalOffset = readOnlyIncrementalSnapshotContext.getSignalOffset();
|
Long signalOffset = connectorConfig.getConnectorAdapter().getReadOnlyIncrementalSnapshotSignalOffset(previousOffset);
|
||||||
if (signalOffset != null) {
|
if (signalOffset != null) {
|
||||||
LOGGER.info("Resetting Kafka Signal offset to {}", signalOffset);
|
LOGGER.info("Resetting Kafka Signal offset to {}", signalOffset);
|
||||||
kafkaSignal.reset(signalOffset);
|
kafkaSignal.reset(signalOffset);
|
||||||
|
@ -14,6 +14,7 @@
|
|||||||
import org.apache.kafka.connect.errors.ConnectException;
|
import org.apache.kafka.connect.errors.ConnectException;
|
||||||
|
|
||||||
import io.debezium.connector.SnapshotRecord;
|
import io.debezium.connector.SnapshotRecord;
|
||||||
|
import io.debezium.connector.mysql.strategy.mysql.MySqlReadOnlyIncrementalSnapshotContext;
|
||||||
import io.debezium.pipeline.CommonOffsetContext;
|
import io.debezium.pipeline.CommonOffsetContext;
|
||||||
import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotContext;
|
import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotContext;
|
||||||
import io.debezium.pipeline.source.snapshot.incremental.SignalBasedIncrementalSnapshotContext;
|
import io.debezium.pipeline.source.snapshot.incremental.SignalBasedIncrementalSnapshotContext;
|
||||||
@ -62,7 +63,8 @@ public MySqlOffsetContext(boolean snapshot, boolean snapshotCompleted, Transacti
|
|||||||
|
|
||||||
public MySqlOffsetContext(MySqlConnectorConfig connectorConfig, boolean snapshot, boolean snapshotCompleted, SourceInfo sourceInfo) {
|
public MySqlOffsetContext(MySqlConnectorConfig connectorConfig, boolean snapshot, boolean snapshotCompleted, SourceInfo sourceInfo) {
|
||||||
this(snapshot, snapshotCompleted, new TransactionContext(),
|
this(snapshot, snapshotCompleted, new TransactionContext(),
|
||||||
connectorConfig.isReadOnlyConnection() ? new MySqlReadOnlyIncrementalSnapshotContext<>() : new SignalBasedIncrementalSnapshotContext<>(),
|
connectorConfig.getConnectorAdapter().getIncrementalSnapshotContext(),
|
||||||
|
// connectorConfig.isReadOnlyConnection() ? new MySqlReadOnlyIncrementalSnapshotContext<>() : new SignalBasedIncrementalSnapshotContext<>(),
|
||||||
sourceInfo);
|
sourceInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -38,8 +38,9 @@
|
|||||||
|
|
||||||
import io.debezium.DebeziumException;
|
import io.debezium.DebeziumException;
|
||||||
import io.debezium.connector.SnapshotRecord;
|
import io.debezium.connector.SnapshotRecord;
|
||||||
import io.debezium.connector.mysql.MySqlConnection.DatabaseLocales;
|
|
||||||
import io.debezium.connector.mysql.MySqlOffsetContext.Loader;
|
import io.debezium.connector.mysql.MySqlOffsetContext.Loader;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection.DatabaseLocales;
|
||||||
import io.debezium.data.Envelope;
|
import io.debezium.data.Envelope;
|
||||||
import io.debezium.function.BlockingConsumer;
|
import io.debezium.function.BlockingConsumer;
|
||||||
import io.debezium.jdbc.JdbcConnection;
|
import io.debezium.jdbc.JdbcConnection;
|
||||||
@ -61,7 +62,7 @@ public class MySqlSnapshotChangeEventSource extends RelationalSnapshotChangeEven
|
|||||||
private static final Logger LOGGER = LoggerFactory.getLogger(MySqlSnapshotChangeEventSource.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(MySqlSnapshotChangeEventSource.class);
|
||||||
|
|
||||||
private final MySqlConnectorConfig connectorConfig;
|
private final MySqlConnectorConfig connectorConfig;
|
||||||
private final MySqlConnection connection;
|
private final AbstractConnectorConnection connection;
|
||||||
private long globalLockAcquiredAt = -1;
|
private long globalLockAcquiredAt = -1;
|
||||||
private long tableLockAcquiredAt = -1;
|
private long tableLockAcquiredAt = -1;
|
||||||
private final RelationalTableFilters filters;
|
private final RelationalTableFilters filters;
|
||||||
@ -72,7 +73,7 @@ public class MySqlSnapshotChangeEventSource extends RelationalSnapshotChangeEven
|
|||||||
private final BlockingConsumer<Function<SourceRecord, SourceRecord>> lastEventProcessor;
|
private final BlockingConsumer<Function<SourceRecord, SourceRecord>> lastEventProcessor;
|
||||||
private final Runnable preSnapshotAction;
|
private final Runnable preSnapshotAction;
|
||||||
|
|
||||||
public MySqlSnapshotChangeEventSource(MySqlConnectorConfig connectorConfig, MainConnectionProvidingConnectionFactory<MySqlConnection> connectionFactory,
|
public MySqlSnapshotChangeEventSource(MySqlConnectorConfig connectorConfig, MainConnectionProvidingConnectionFactory<AbstractConnectorConnection> connectionFactory,
|
||||||
MySqlDatabaseSchema schema, EventDispatcher<MySqlPartition, TableId> dispatcher, Clock clock,
|
MySqlDatabaseSchema schema, EventDispatcher<MySqlPartition, TableId> dispatcher, Clock clock,
|
||||||
MySqlSnapshotChangeEventSourceMetrics metrics,
|
MySqlSnapshotChangeEventSourceMetrics metrics,
|
||||||
BlockingConsumer<Function<SourceRecord, SourceRecord>> lastEventProcessor,
|
BlockingConsumer<Function<SourceRecord, SourceRecord>> lastEventProcessor,
|
||||||
@ -359,7 +360,7 @@ protected void readTableStructure(ChangeEventSourceContext sourceContext,
|
|||||||
|
|
||||||
if (!snapshottingTask.isBlocking()) {
|
if (!snapshottingTask.isBlocking()) {
|
||||||
// Record default charset
|
// Record default charset
|
||||||
addSchemaEvent(snapshotContext, "", connection.setStatementFor(connection.readMySqlCharsetSystemVariables()));
|
addSchemaEvent(snapshotContext, "", connection.setStatementFor(connection.readCharsetSystemVariables()));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (TableId tableId : capturedSchemaTables) {
|
for (TableId tableId : capturedSchemaTables) {
|
||||||
@ -584,7 +585,7 @@ protected OptionalLong rowCountForTable(TableId tableId) {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Statement readTableStatement(JdbcConnection jdbcConnection, OptionalLong rowCount) throws SQLException {
|
protected Statement readTableStatement(JdbcConnection jdbcConnection, OptionalLong rowCount) throws SQLException {
|
||||||
MySqlConnection connection = (MySqlConnection) jdbcConnection;
|
AbstractConnectorConnection connection = (AbstractConnectorConnection) jdbcConnection;
|
||||||
final long largeTableRowCount = connectorConfig.rowCountForLargeTable();
|
final long largeTableRowCount = connectorConfig.rowCountForLargeTable();
|
||||||
if (rowCount.isEmpty() || largeTableRowCount == 0 || rowCount.getAsLong() <= largeTableRowCount) {
|
if (rowCount.isEmpty() || largeTableRowCount == 0 || rowCount.getAsLong() <= largeTableRowCount) {
|
||||||
return super.readTableStatement(connection, rowCount);
|
return super.readTableStatement(connection, rowCount);
|
||||||
@ -610,7 +611,7 @@ protected Statement readTableStatement(JdbcConnection jdbcConnection, OptionalLo
|
|||||||
* @return the statement; never null
|
* @return the statement; never null
|
||||||
* @throws SQLException if there is a problem creating the statement
|
* @throws SQLException if there is a problem creating the statement
|
||||||
*/
|
*/
|
||||||
private Statement createStatementWithLargeResultSet(MySqlConnection connection) throws SQLException {
|
private Statement createStatementWithLargeResultSet(AbstractConnectorConnection connection) throws SQLException {
|
||||||
int fetchSize = connectorConfig.getSnapshotFetchSize();
|
int fetchSize = connectorConfig.getSnapshotFetchSize();
|
||||||
Statement stmt = connection.connection().createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
|
Statement stmt = connection.connection().createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
|
||||||
stmt.setFetchSize(fetchSize);
|
stmt.setFetchSize(fetchSize);
|
||||||
|
@ -5,21 +5,11 @@
|
|||||||
*/
|
*/
|
||||||
package io.debezium.connector.mysql;
|
package io.debezium.connector.mysql;
|
||||||
|
|
||||||
import static io.debezium.util.Strings.isNullOrEmpty;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.security.GeneralSecurityException;
|
|
||||||
import java.security.KeyStore;
|
|
||||||
import java.security.KeyStoreException;
|
|
||||||
import java.security.NoSuchAlgorithmException;
|
|
||||||
import java.security.UnrecoverableKeyException;
|
|
||||||
import java.security.cert.CertificateException;
|
|
||||||
import java.security.cert.X509Certificate;
|
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.EnumMap;
|
import java.util.EnumMap;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
@ -29,13 +19,6 @@
|
|||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
import java.util.function.Predicate;
|
import java.util.function.Predicate;
|
||||||
|
|
||||||
import javax.net.ssl.KeyManager;
|
|
||||||
import javax.net.ssl.KeyManagerFactory;
|
|
||||||
import javax.net.ssl.SSLContext;
|
|
||||||
import javax.net.ssl.TrustManager;
|
|
||||||
import javax.net.ssl.TrustManagerFactory;
|
|
||||||
import javax.net.ssl.X509TrustManager;
|
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.slf4j.event.Level;
|
import org.slf4j.event.Level;
|
||||||
@ -57,14 +40,9 @@
|
|||||||
import com.github.shyiko.mysql.binlog.event.TransactionPayloadEventData;
|
import com.github.shyiko.mysql.binlog.event.TransactionPayloadEventData;
|
||||||
import com.github.shyiko.mysql.binlog.event.UpdateRowsEventData;
|
import com.github.shyiko.mysql.binlog.event.UpdateRowsEventData;
|
||||||
import com.github.shyiko.mysql.binlog.event.WriteRowsEventData;
|
import com.github.shyiko.mysql.binlog.event.WriteRowsEventData;
|
||||||
import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializationException;
|
|
||||||
import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
|
import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
|
||||||
import com.github.shyiko.mysql.binlog.event.deserialization.GtidEventDataDeserializer;
|
|
||||||
import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
|
|
||||||
import com.github.shyiko.mysql.binlog.network.AuthenticationException;
|
import com.github.shyiko.mysql.binlog.network.AuthenticationException;
|
||||||
import com.github.shyiko.mysql.binlog.network.DefaultSSLSocketFactory;
|
|
||||||
import com.github.shyiko.mysql.binlog.network.SSLMode;
|
import com.github.shyiko.mysql.binlog.network.SSLMode;
|
||||||
import com.github.shyiko.mysql.binlog.network.SSLSocketFactory;
|
|
||||||
import com.github.shyiko.mysql.binlog.network.ServerException;
|
import com.github.shyiko.mysql.binlog.network.ServerException;
|
||||||
|
|
||||||
import io.debezium.DebeziumException;
|
import io.debezium.DebeziumException;
|
||||||
@ -72,6 +50,8 @@
|
|||||||
import io.debezium.config.CommonConnectorConfig.EventProcessingFailureHandlingMode;
|
import io.debezium.config.CommonConnectorConfig.EventProcessingFailureHandlingMode;
|
||||||
import io.debezium.config.Configuration;
|
import io.debezium.config.Configuration;
|
||||||
import io.debezium.connector.mysql.MySqlConnectorConfig.SecureConnectionMode;
|
import io.debezium.connector.mysql.MySqlConnectorConfig.SecureConnectionMode;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection;
|
||||||
|
import io.debezium.connector.mysql.strategy.ConnectorAdapter;
|
||||||
import io.debezium.data.Envelope.Operation;
|
import io.debezium.data.Envelope.Operation;
|
||||||
import io.debezium.function.BlockingConsumer;
|
import io.debezium.function.BlockingConsumer;
|
||||||
import io.debezium.pipeline.ErrorHandler;
|
import io.debezium.pipeline.ErrorHandler;
|
||||||
@ -112,13 +92,13 @@ public class MySqlStreamingChangeEventSource implements StreamingChangeEventSour
|
|||||||
private final AtomicLong totalRecordCounter = new AtomicLong();
|
private final AtomicLong totalRecordCounter = new AtomicLong();
|
||||||
private volatile Map<String, ?> lastOffset = null;
|
private volatile Map<String, ?> lastOffset = null;
|
||||||
private com.github.shyiko.mysql.binlog.GtidSet gtidSet;
|
private com.github.shyiko.mysql.binlog.GtidSet gtidSet;
|
||||||
private final float heartbeatIntervalFactor = 0.8f;
|
|
||||||
private final Map<String, Thread> binaryLogClientThreads = new ConcurrentHashMap<>(4);
|
private final Map<String, Thread> binaryLogClientThreads = new ConcurrentHashMap<>(4);
|
||||||
private final MySqlTaskContext taskContext;
|
private final MySqlTaskContext taskContext;
|
||||||
private final MySqlConnectorConfig connectorConfig;
|
private final MySqlConnectorConfig connectorConfig;
|
||||||
private final MySqlConnection connection;
|
private final AbstractConnectorConnection connection;
|
||||||
private final EventDispatcher<MySqlPartition, TableId> eventDispatcher;
|
private final EventDispatcher<MySqlPartition, TableId> eventDispatcher;
|
||||||
private final ErrorHandler errorHandler;
|
private final ErrorHandler errorHandler;
|
||||||
|
private final ConnectorAdapter connectorAdapter;
|
||||||
|
|
||||||
@SingleThreadAccess("binlog client thread")
|
@SingleThreadAccess("binlog client thread")
|
||||||
private Instant eventTimestamp;
|
private Instant eventTimestamp;
|
||||||
@ -184,7 +164,7 @@ private interface BinlogChangeEmitter<T> {
|
|||||||
void emit(TableId tableId, T data) throws InterruptedException;
|
void emit(TableId tableId, T data) throws InterruptedException;
|
||||||
}
|
}
|
||||||
|
|
||||||
public MySqlStreamingChangeEventSource(MySqlConnectorConfig connectorConfig, MySqlConnection connection,
|
public MySqlStreamingChangeEventSource(MySqlConnectorConfig connectorConfig, AbstractConnectorConnection connection,
|
||||||
EventDispatcher<MySqlPartition, TableId> dispatcher, ErrorHandler errorHandler, Clock clock,
|
EventDispatcher<MySqlPartition, TableId> dispatcher, ErrorHandler errorHandler, Clock clock,
|
||||||
MySqlTaskContext taskContext, MySqlStreamingChangeEventSourceMetrics metrics) {
|
MySqlTaskContext taskContext, MySqlStreamingChangeEventSourceMetrics metrics) {
|
||||||
|
|
||||||
@ -195,131 +175,21 @@ public MySqlStreamingChangeEventSource(MySqlConnectorConfig connectorConfig, MyS
|
|||||||
this.eventDispatcher = dispatcher;
|
this.eventDispatcher = dispatcher;
|
||||||
this.errorHandler = errorHandler;
|
this.errorHandler = errorHandler;
|
||||||
this.metrics = metrics;
|
this.metrics = metrics;
|
||||||
|
this.connectorAdapter = connectorConfig.getConnectorAdapter();
|
||||||
|
|
||||||
eventDeserializationFailureHandlingMode = connectorConfig.getEventProcessingFailureHandlingMode();
|
eventDeserializationFailureHandlingMode = connectorConfig.getEventProcessingFailureHandlingMode();
|
||||||
inconsistentSchemaHandlingMode = connectorConfig.inconsistentSchemaFailureHandlingMode();
|
inconsistentSchemaHandlingMode = connectorConfig.inconsistentSchemaFailureHandlingMode();
|
||||||
|
|
||||||
// Set up the log reader ...
|
// Set up the log reader ...
|
||||||
client = taskContext.getBinaryLogClient();
|
client = connectorAdapter.getBinaryLogClientConfigurator().configure(
|
||||||
// BinaryLogClient will overwrite thread names later
|
taskContext.getBinaryLogClient(),
|
||||||
client.setThreadFactory(
|
|
||||||
Threads.threadFactory(MySqlConnector.class, connectorConfig.getLogicalName(), "binlog-client", false, false,
|
Threads.threadFactory(MySqlConnector.class, connectorConfig.getLogicalName(), "binlog-client", false, false,
|
||||||
x -> binaryLogClientThreads.put(x.getName(), x)));
|
x -> binaryLogClientThreads.put(x.getName(), x)),
|
||||||
client.setServerId(connectorConfig.serverId());
|
connection);
|
||||||
client.setSSLMode(sslModeFor(connectorConfig.sslMode()));
|
|
||||||
if (connectorConfig.sslModeEnabled()) {
|
|
||||||
SSLSocketFactory sslSocketFactory = getBinlogSslSocketFactory(connectorConfig, connection);
|
|
||||||
if (sslSocketFactory != null) {
|
|
||||||
client.setSslSocketFactory(sslSocketFactory);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (connection.isMariaDb()) {
|
|
||||||
// This makes sure BEGIN events are emitted via QUERY events rather than GTIDs.
|
|
||||||
client.setMariaDbSlaveCapability(2);
|
|
||||||
}
|
|
||||||
Configuration configuration = connectorConfig.getConfig();
|
|
||||||
client.setKeepAlive(configuration.getBoolean(MySqlConnectorConfig.KEEP_ALIVE));
|
|
||||||
final long keepAliveInterval = configuration.getLong(MySqlConnectorConfig.KEEP_ALIVE_INTERVAL_MS);
|
|
||||||
client.setKeepAliveInterval(keepAliveInterval);
|
|
||||||
// Considering heartbeatInterval should be less than keepAliveInterval, we use the heartbeatIntervalFactor
|
|
||||||
// multiply by keepAliveInterval and set the result value to heartbeatInterval.The default value of heartbeatIntervalFactor
|
|
||||||
// is 0.8, and we believe the left time (0.2 * keepAliveInterval) is enough to process the packet received from the MySQL server.
|
|
||||||
client.setHeartbeatInterval((long) (keepAliveInterval * heartbeatIntervalFactor));
|
|
||||||
|
|
||||||
|
Configuration configuration = connectorConfig.getConfig();
|
||||||
boolean filterDmlEventsByGtidSource = configuration.getBoolean(MySqlConnectorConfig.GTID_SOURCE_FILTER_DML_EVENTS);
|
boolean filterDmlEventsByGtidSource = configuration.getBoolean(MySqlConnectorConfig.GTID_SOURCE_FILTER_DML_EVENTS);
|
||||||
gtidDmlSourceFilter = filterDmlEventsByGtidSource ? connectorConfig.gtidSourceFilter() : null;
|
gtidDmlSourceFilter = filterDmlEventsByGtidSource ? connectorConfig.gtidSourceFilter() : null;
|
||||||
|
|
||||||
// Set up the event deserializer with additional type(s) ...
|
|
||||||
final Map<Long, TableMapEventData> tableMapEventByTableId = new HashMap<Long, TableMapEventData>();
|
|
||||||
EventDeserializer eventDeserializer = new EventDeserializer() {
|
|
||||||
@Override
|
|
||||||
public Event nextEvent(ByteArrayInputStream inputStream) throws IOException {
|
|
||||||
try {
|
|
||||||
// Delegate to the superclass ...
|
|
||||||
Event event = super.nextEvent(inputStream);
|
|
||||||
|
|
||||||
// We have to record the most recent TableMapEventData for each table number for our custom deserializers ...
|
|
||||||
if (event.getHeader().getEventType() == EventType.TABLE_MAP) {
|
|
||||||
TableMapEventData tableMapEvent = event.getData();
|
|
||||||
tableMapEventByTableId.put(tableMapEvent.getTableId(), tableMapEvent);
|
|
||||||
}
|
|
||||||
|
|
||||||
// DBZ-2663 Handle for transaction payload and capture the table map event and add it to the map
|
|
||||||
if (event.getHeader().getEventType() == EventType.TRANSACTION_PAYLOAD) {
|
|
||||||
TransactionPayloadEventData transactionPayloadEventData = (TransactionPayloadEventData) event.getData();
|
|
||||||
/**
|
|
||||||
* Loop over the uncompressed events in the transaction payload event and add the table map
|
|
||||||
* event in the map of table events
|
|
||||||
**/
|
|
||||||
for (Event uncompressedEvent : transactionPayloadEventData.getUncompressedEvents()) {
|
|
||||||
if (uncompressedEvent.getHeader().getEventType() == EventType.TABLE_MAP
|
|
||||||
&& uncompressedEvent.getData() != null) {
|
|
||||||
TableMapEventData tableMapEvent = (TableMapEventData) uncompressedEvent.getData();
|
|
||||||
tableMapEventByTableId.put(tableMapEvent.getTableId(), tableMapEvent);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// DBZ-5126 Clean cache on rotate event to prevent it from growing indefinitely.
|
|
||||||
if (event.getHeader().getEventType() == EventType.ROTATE && event.getHeader().getTimestamp() != 0) {
|
|
||||||
tableMapEventByTableId.clear();
|
|
||||||
}
|
|
||||||
return event;
|
|
||||||
}
|
|
||||||
// DBZ-217 In case an event couldn't be read we create a pseudo-event for the sake of logging
|
|
||||||
catch (EventDataDeserializationException edde) {
|
|
||||||
// DBZ-3095 As of Java 15, when reaching EOF in the binlog stream, the polling loop in
|
|
||||||
// BinaryLogClient#listenForEventPackets() keeps returning values != -1 from peek();
|
|
||||||
// this causes the loop to never finish
|
|
||||||
// Propagating the exception (either EOF or socket closed) causes the loop to be aborted
|
|
||||||
// in this case
|
|
||||||
if (edde.getCause() instanceof IOException) {
|
|
||||||
throw edde;
|
|
||||||
}
|
|
||||||
|
|
||||||
EventHeaderV4 header = new EventHeaderV4();
|
|
||||||
header.setEventType(EventType.INCIDENT);
|
|
||||||
header.setTimestamp(edde.getEventHeader().getTimestamp());
|
|
||||||
header.setServerId(edde.getEventHeader().getServerId());
|
|
||||||
|
|
||||||
if (edde.getEventHeader() instanceof EventHeaderV4) {
|
|
||||||
header.setEventLength(((EventHeaderV4) edde.getEventHeader()).getEventLength());
|
|
||||||
header.setNextPosition(((EventHeaderV4) edde.getEventHeader()).getNextPosition());
|
|
||||||
header.setFlags(((EventHeaderV4) edde.getEventHeader()).getFlags());
|
|
||||||
}
|
|
||||||
|
|
||||||
EventData data = new EventDataDeserializationExceptionData(edde);
|
|
||||||
return new Event(header, data);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Add our custom deserializers ...
|
|
||||||
eventDeserializer.setEventDataDeserializer(EventType.STOP, new StopEventDataDeserializer());
|
|
||||||
eventDeserializer.setEventDataDeserializer(EventType.GTID, new GtidEventDataDeserializer());
|
|
||||||
eventDeserializer.setEventDataDeserializer(EventType.WRITE_ROWS,
|
|
||||||
new RowDeserializers.WriteRowsDeserializer(tableMapEventByTableId, eventDeserializationFailureHandlingMode));
|
|
||||||
eventDeserializer.setEventDataDeserializer(EventType.UPDATE_ROWS,
|
|
||||||
new RowDeserializers.UpdateRowsDeserializer(tableMapEventByTableId, eventDeserializationFailureHandlingMode));
|
|
||||||
eventDeserializer.setEventDataDeserializer(EventType.DELETE_ROWS,
|
|
||||||
new RowDeserializers.DeleteRowsDeserializer(tableMapEventByTableId, eventDeserializationFailureHandlingMode));
|
|
||||||
eventDeserializer.setEventDataDeserializer(EventType.EXT_WRITE_ROWS,
|
|
||||||
new RowDeserializers.WriteRowsDeserializer(
|
|
||||||
tableMapEventByTableId, eventDeserializationFailureHandlingMode).setMayContainExtraInformation(true));
|
|
||||||
eventDeserializer.setEventDataDeserializer(EventType.EXT_UPDATE_ROWS,
|
|
||||||
new RowDeserializers.UpdateRowsDeserializer(
|
|
||||||
tableMapEventByTableId, eventDeserializationFailureHandlingMode).setMayContainExtraInformation(true));
|
|
||||||
eventDeserializer.setEventDataDeserializer(EventType.EXT_DELETE_ROWS,
|
|
||||||
new RowDeserializers.DeleteRowsDeserializer(
|
|
||||||
tableMapEventByTableId, eventDeserializationFailureHandlingMode).setMayContainExtraInformation(true));
|
|
||||||
eventDeserializer.setEventDataDeserializer(EventType.TRANSACTION_PAYLOAD,
|
|
||||||
new TransactionPayloadDeserializer(tableMapEventByTableId, eventDeserializationFailureHandlingMode));
|
|
||||||
|
|
||||||
if (connection.isMariaDb()) {
|
|
||||||
eventDeserializer.setCompatibilityMode(EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY);
|
|
||||||
}
|
|
||||||
|
|
||||||
client.setEventDeserializer(eventDeserializer);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void onEvent(MySqlOffsetContext offsetContext, Event event) {
|
protected void onEvent(MySqlOffsetContext offsetContext, Event event) {
|
||||||
@ -547,19 +417,8 @@ protected void handleGtidEvent(MySqlOffsetContext offsetContext, Event event) {
|
|||||||
* @param event the database change data event to be processed; may not be null
|
* @param event the database change data event to be processed; may not be null
|
||||||
*/
|
*/
|
||||||
protected void handleRecordingQuery(MySqlOffsetContext offsetContext, Event event) {
|
protected void handleRecordingQuery(MySqlOffsetContext offsetContext, Event event) {
|
||||||
final String query;
|
|
||||||
if (!connection.isMariaDb()) {
|
|
||||||
// Unwrap the RowsQueryEvent
|
|
||||||
final RowsQueryEventData lastRowsQueryEventData = unwrapData(event);
|
|
||||||
query = lastRowsQueryEventData.getQuery();
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
// Unwrap the AnnotateRowsEventData
|
|
||||||
final AnnotateRowsEventData annotateRowsEventData = unwrapData(event);
|
|
||||||
query = annotateRowsEventData.getRowsQuery();
|
|
||||||
}
|
|
||||||
// Set the query on the source
|
// Set the query on the source
|
||||||
offsetContext.setQuery(query);
|
offsetContext.setQuery(connectorAdapter.getRecordingQueryFromEvent(unwrapData(event)));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -975,16 +834,8 @@ public void execute(ChangeEventSourceContext context, MySqlPartition partition,
|
|||||||
|
|
||||||
// Conditionally register ROWS_QUERY handler to parse SQL statements.
|
// Conditionally register ROWS_QUERY handler to parse SQL statements.
|
||||||
if (connectorConfig.includeSqlQuery()) {
|
if (connectorConfig.includeSqlQuery()) {
|
||||||
if (!connection.isMariaDb()) {
|
final EventType eventType = connectorAdapter.getBinaryLogClientConfigurator().getIncludeSqlQueryEventType();
|
||||||
eventHandlers.put(EventType.ROWS_QUERY, (event) -> handleRecordingQuery(effectiveOffsetContext, event));
|
eventHandlers.put(eventType, (event) -> handleRecordingQuery(effectiveOffsetContext, event));
|
||||||
}
|
|
||||||
else {
|
|
||||||
// Binlog client explicitly needs to be told to enable ANNOTATE_ROWS events, which is the
|
|
||||||
// MariaDB equivalent of ROWS_QUERY. This must be done ahead of the connection to make
|
|
||||||
// sure that the right negotiation bits are set during handshake.
|
|
||||||
client.setUseSendAnnotateRowsEvent(true);
|
|
||||||
eventHandlers.put(EventType.ANNOTATE_ROWS, (event) -> handleRecordingQuery(effectiveOffsetContext, event));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
BinaryLogClient.EventListener listener;
|
BinaryLogClient.EventListener listener;
|
||||||
@ -1007,19 +858,19 @@ public void execute(ChangeEventSourceContext context, MySqlPartition partition,
|
|||||||
metrics.setIsGtidModeEnabled(isGtidModeEnabled);
|
metrics.setIsGtidModeEnabled(isGtidModeEnabled);
|
||||||
|
|
||||||
// Get the current GtidSet from MySQL so we can get a filtered/merged GtidSet based off of the last Debezium checkpoint.
|
// Get the current GtidSet from MySQL so we can get a filtered/merged GtidSet based off of the last Debezium checkpoint.
|
||||||
String availableServerGtidStr = connection.knownGtidSet();
|
|
||||||
if (isGtidModeEnabled) {
|
if (isGtidModeEnabled) {
|
||||||
// The server is using GTIDs, so enable the handler ...
|
// The server is using GTIDs, so enable the handler ...
|
||||||
eventHandlers.put(EventType.GTID, (event) -> handleGtidEvent(effectiveOffsetContext, event));
|
eventHandlers.put(EventType.GTID, (event) -> handleGtidEvent(effectiveOffsetContext, event));
|
||||||
|
|
||||||
// Now look at the GTID set from the server and what we've previously seen ...
|
// Now look at the GTID set from the server and what we've previously seen ...
|
||||||
GtidSet availableServerGtidSet = new GtidSet(availableServerGtidStr);
|
GtidSet availableServerGtidSet = connection.knownGtidSet();
|
||||||
|
|
||||||
// also take into account purged GTID logs
|
// also take into account purged GTID logs
|
||||||
GtidSet purgedServerGtidSet = connection.purgedGtidSet();
|
GtidSet purgedServerGtidSet = connection.purgedGtidSet();
|
||||||
LOGGER.info("GTID set purged on server: {}", purgedServerGtidSet);
|
LOGGER.info("GTID set purged on server: {}", purgedServerGtidSet);
|
||||||
|
|
||||||
GtidSet filteredGtidSet = filterGtidSet(effectiveOffsetContext, availableServerGtidSet, purgedServerGtidSet);
|
GtidSet filteredGtidSet = connection.filterGtidSet(connectorConfig.gtidSourceFilter(),
|
||||||
|
effectiveOffsetContext.gtidSet(), availableServerGtidSet, purgedServerGtidSet);
|
||||||
if (filteredGtidSet != null) {
|
if (filteredGtidSet != null) {
|
||||||
// We've seen at least some GTIDs, so start reading from the filtered GTID set ...
|
// We've seen at least some GTIDs, so start reading from the filtered GTID set ...
|
||||||
LOGGER.info("Registering binlog reader with GTID set: {}", filteredGtidSet);
|
LOGGER.info("Registering binlog reader with GTID set: {}", filteredGtidSet);
|
||||||
@ -1130,82 +981,6 @@ public MySqlOffsetContext getOffsetContext() {
|
|||||||
return effectiveOffsetContext;
|
return effectiveOffsetContext;
|
||||||
}
|
}
|
||||||
|
|
||||||
private SSLSocketFactory getBinlogSslSocketFactory(MySqlConnectorConfig connectorConfig, MySqlConnection connection) {
|
|
||||||
String acceptedTlsVersion = connection.getSessionVariableForSslVersion();
|
|
||||||
if (!isNullOrEmpty(acceptedTlsVersion)) {
|
|
||||||
SSLMode sslMode = sslModeFor(connectorConfig.sslMode());
|
|
||||||
LOGGER.info("Enable ssl " + sslMode + " mode for connector " + connectorConfig.getLogicalName());
|
|
||||||
|
|
||||||
final char[] keyPasswordArray = connection.connectionConfig().sslKeyStorePassword();
|
|
||||||
final String keyFilename = connection.connectionConfig().sslKeyStore();
|
|
||||||
final char[] trustPasswordArray = connection.connectionConfig().sslTrustStorePassword();
|
|
||||||
final String trustFilename = connection.connectionConfig().sslTrustStore();
|
|
||||||
KeyManager[] keyManagers = null;
|
|
||||||
if (keyFilename != null) {
|
|
||||||
try {
|
|
||||||
KeyStore ks = connection.loadKeyStore(keyFilename, keyPasswordArray);
|
|
||||||
|
|
||||||
KeyManagerFactory kmf = KeyManagerFactory.getInstance("NewSunX509");
|
|
||||||
kmf.init(ks, keyPasswordArray);
|
|
||||||
|
|
||||||
keyManagers = kmf.getKeyManagers();
|
|
||||||
}
|
|
||||||
catch (KeyStoreException | NoSuchAlgorithmException | UnrecoverableKeyException e) {
|
|
||||||
throw new DebeziumException("Could not load keystore", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TrustManager[] trustManagers;
|
|
||||||
try {
|
|
||||||
KeyStore ks = null;
|
|
||||||
if (trustFilename != null) {
|
|
||||||
ks = connection.loadKeyStore(trustFilename, trustPasswordArray);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ks == null && (sslMode == SSLMode.PREFERRED || sslMode == SSLMode.REQUIRED)) {
|
|
||||||
trustManagers = new TrustManager[]{
|
|
||||||
new X509TrustManager() {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void checkClientTrusted(X509Certificate[] x509Certificates, String s)
|
|
||||||
throws CertificateException {
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void checkServerTrusted(X509Certificate[] x509Certificates, String s)
|
|
||||||
throws CertificateException {
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public X509Certificate[] getAcceptedIssuers() {
|
|
||||||
return new X509Certificate[0];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
|
|
||||||
tmf.init(ks);
|
|
||||||
trustManagers = tmf.getTrustManagers();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (KeyStoreException | NoSuchAlgorithmException e) {
|
|
||||||
throw new DebeziumException("Could not load truststore", e);
|
|
||||||
}
|
|
||||||
// DBZ-1208 Resembles the logic from the upstream BinaryLogClient, only that
|
|
||||||
// the accepted TLS version is passed to the constructed factory
|
|
||||||
final KeyManager[] finalKMS = keyManagers;
|
|
||||||
return new DefaultSSLSocketFactory(acceptedTlsVersion) {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void initSSLContext(SSLContext sc) throws GeneralSecurityException {
|
|
||||||
sc.init(finalKMS, trustManagers, null);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void logStreamingSourceState() {
|
private void logStreamingSourceState() {
|
||||||
logStreamingSourceState(Level.ERROR);
|
logStreamingSourceState(Level.ERROR);
|
||||||
}
|
}
|
||||||
@ -1229,54 +1004,6 @@ private void logStreamingSourceState(Level severity) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Apply the include/exclude GTID source filters to the current {@link MySqlOffsetContext#gtidSet() GTID set} and merge them onto the
|
|
||||||
* currently available GTID set from a MySQL server.
|
|
||||||
*
|
|
||||||
* The merging behavior of this method might seem a bit strange at first. It's required in order for Debezium to consume a
|
|
||||||
* MySQL binlog that has multi-source replication enabled, if a failover has to occur. In such a case, the server that
|
|
||||||
* Debezium is failed over to might have a different set of sources, but still include the sources required for Debezium
|
|
||||||
* to continue to function. MySQL does not allow downstream replicas to connect if the GTID set does not contain GTIDs for
|
|
||||||
* all channels that the server is replicating from, even if the server does have the data needed by the client. To get
|
|
||||||
* around this, we can have Debezium merge its GTID set with whatever is on the server, so that MySQL will allow it to
|
|
||||||
* connect. See <a href="https://issues.jboss.org/browse/DBZ-143">DBZ-143</a> for details.
|
|
||||||
*
|
|
||||||
* This method does not mutate any state in the context.
|
|
||||||
*
|
|
||||||
* @param availableServerGtidSet the GTID set currently available in the MySQL server
|
|
||||||
* @param purgedServerGtid the GTID set already purged by the MySQL server
|
|
||||||
* @return A GTID set meant for consuming from a MySQL binlog; may return null if the SourceInfo has no GTIDs and therefore
|
|
||||||
* none were filtered
|
|
||||||
*/
|
|
||||||
public GtidSet filterGtidSet(MySqlOffsetContext offsetContext, GtidSet availableServerGtidSet, GtidSet purgedServerGtid) {
|
|
||||||
String gtidStr = offsetContext.gtidSet();
|
|
||||||
if (gtidStr == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
LOGGER.info("Attempting to generate a filtered GTID set");
|
|
||||||
LOGGER.info("GTID set from previous recorded offset: {}", gtidStr);
|
|
||||||
GtidSet filteredGtidSet = new GtidSet(gtidStr);
|
|
||||||
Predicate<String> gtidSourceFilter = connectorConfig.gtidSourceFilter();
|
|
||||||
if (gtidSourceFilter != null) {
|
|
||||||
filteredGtidSet = filteredGtidSet.retainAll(gtidSourceFilter);
|
|
||||||
LOGGER.info("GTID set after applying GTID source includes/excludes to previous recorded offset: {}", filteredGtidSet);
|
|
||||||
}
|
|
||||||
LOGGER.info("GTID set available on server: {}", availableServerGtidSet);
|
|
||||||
|
|
||||||
final GtidSet knownGtidSet = filteredGtidSet;
|
|
||||||
LOGGER.info("Using first available positions for new GTID channels");
|
|
||||||
final GtidSet relevantAvailableServerGtidSet = (gtidSourceFilter != null) ? availableServerGtidSet.retainAll(gtidSourceFilter) : availableServerGtidSet;
|
|
||||||
LOGGER.info("Relevant GTID set available on server: {}", relevantAvailableServerGtidSet);
|
|
||||||
|
|
||||||
GtidSet mergedGtidSet = relevantAvailableServerGtidSet
|
|
||||||
.retainAll(uuid -> knownGtidSet.forServerWithId(uuid) != null)
|
|
||||||
.with(purgedServerGtid)
|
|
||||||
.with(filteredGtidSet);
|
|
||||||
|
|
||||||
LOGGER.info("Final merged GTID set to use when connecting to MySQL: {}", mergedGtidSet);
|
|
||||||
return mergedGtidSet;
|
|
||||||
}
|
|
||||||
|
|
||||||
MySqlStreamingChangeEventSourceMetrics getMetrics() {
|
MySqlStreamingChangeEventSourceMetrics getMetrics() {
|
||||||
return metrics;
|
return metrics;
|
||||||
}
|
}
|
||||||
|
@ -39,8 +39,11 @@
|
|||||||
|
|
||||||
import io.debezium.DebeziumException;
|
import io.debezium.DebeziumException;
|
||||||
import io.debezium.annotation.Immutable;
|
import io.debezium.annotation.Immutable;
|
||||||
|
import io.debezium.annotation.VisibleForTesting;
|
||||||
import io.debezium.config.CommonConnectorConfig.BinaryHandlingMode;
|
import io.debezium.config.CommonConnectorConfig.BinaryHandlingMode;
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
import io.debezium.connector.mysql.antlr.MySqlAntlrDdlParser;
|
import io.debezium.connector.mysql.antlr.MySqlAntlrDdlParser;
|
||||||
|
import io.debezium.connector.mysql.strategy.ConnectorAdapter;
|
||||||
import io.debezium.data.Json;
|
import io.debezium.data.Json;
|
||||||
import io.debezium.data.SpecialValueDecimal;
|
import io.debezium.data.SpecialValueDecimal;
|
||||||
import io.debezium.jdbc.JdbcValueConverters;
|
import io.debezium.jdbc.JdbcValueConverters;
|
||||||
@ -115,6 +118,7 @@ else if (70 <= year && year <= 99) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private final ParsingErrorHandler parsingErrorHandler;
|
private final ParsingErrorHandler parsingErrorHandler;
|
||||||
|
private final ConnectorAdapter connectorAdapter;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a new instance that always uses UTC for the default time zone when_needed converting values without timezone information
|
* Create a new instance that always uses UTC for the default time zone when_needed converting values without timezone information
|
||||||
@ -128,9 +132,16 @@ else if (70 <= year && year <= 99) {
|
|||||||
* {@link io.debezium.jdbc.JdbcValueConverters.BigIntUnsignedMode#PRECISE} is to be used
|
* {@link io.debezium.jdbc.JdbcValueConverters.BigIntUnsignedMode#PRECISE} is to be used
|
||||||
* @param binaryMode how binary columns should be represented
|
* @param binaryMode how binary columns should be represented
|
||||||
*/
|
*/
|
||||||
|
@VisibleForTesting
|
||||||
public MySqlValueConverters(DecimalMode decimalMode, TemporalPrecisionMode temporalPrecisionMode, BigIntUnsignedMode bigIntUnsignedMode,
|
public MySqlValueConverters(DecimalMode decimalMode, TemporalPrecisionMode temporalPrecisionMode, BigIntUnsignedMode bigIntUnsignedMode,
|
||||||
BinaryHandlingMode binaryMode) {
|
BinaryHandlingMode binaryMode) {
|
||||||
this(decimalMode, temporalPrecisionMode, bigIntUnsignedMode, binaryMode, x -> x, MySqlValueConverters::defaultParsingErrorHandler);
|
this(decimalMode, temporalPrecisionMode, bigIntUnsignedMode, binaryMode, x -> x, MySqlValueConverters::defaultParsingErrorHandler, resolveDefaultAdapter());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ConnectorAdapter resolveDefaultAdapter() {
|
||||||
|
Configuration config = Configuration.empty();
|
||||||
|
MySqlConnectorConfig connectorConfig = new MySqlConnectorConfig(config);
|
||||||
|
return MySqlConnectorConfig.ConnectorAdapterMode.MYSQL.getAdapter(connectorConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -145,12 +156,14 @@ public MySqlValueConverters(DecimalMode decimalMode, TemporalPrecisionMode tempo
|
|||||||
* {@link io.debezium.jdbc.JdbcValueConverters.BigIntUnsignedMode#PRECISE} is to be used
|
* {@link io.debezium.jdbc.JdbcValueConverters.BigIntUnsignedMode#PRECISE} is to be used
|
||||||
* @param binaryMode how binary columns should be represented
|
* @param binaryMode how binary columns should be represented
|
||||||
* @param adjuster a temporal adjuster to make a database specific time modification before conversion
|
* @param adjuster a temporal adjuster to make a database specific time modification before conversion
|
||||||
|
* @param connectorAdapter the connector adapter
|
||||||
*/
|
*/
|
||||||
public MySqlValueConverters(DecimalMode decimalMode, TemporalPrecisionMode temporalPrecisionMode, BigIntUnsignedMode bigIntUnsignedMode,
|
public MySqlValueConverters(DecimalMode decimalMode, TemporalPrecisionMode temporalPrecisionMode, BigIntUnsignedMode bigIntUnsignedMode,
|
||||||
BinaryHandlingMode binaryMode,
|
BinaryHandlingMode binaryMode,
|
||||||
TemporalAdjuster adjuster, ParsingErrorHandler parsingErrorHandler) {
|
TemporalAdjuster adjuster, ParsingErrorHandler parsingErrorHandler, ConnectorAdapter connectorAdapter) {
|
||||||
super(decimalMode, temporalPrecisionMode, ZoneOffset.UTC, adjuster, bigIntUnsignedMode, binaryMode);
|
super(decimalMode, temporalPrecisionMode, ZoneOffset.UTC, adjuster, bigIntUnsignedMode, binaryMode);
|
||||||
this.parsingErrorHandler = parsingErrorHandler;
|
this.parsingErrorHandler = parsingErrorHandler;
|
||||||
|
this.connectorAdapter = connectorAdapter;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -331,10 +344,10 @@ protected Charset charsetFor(Column column) {
|
|||||||
logger.warn("Column is missing a character set: {}", column);
|
logger.warn("Column is missing a character set: {}", column);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
String encoding = MySqlConnection.getJavaEncodingForMysqlCharSet(mySqlCharsetName);
|
String encoding = connectorAdapter.getJavaEncodingForCharSet(mySqlCharsetName);
|
||||||
if (encoding == null) {
|
if (encoding == null) {
|
||||||
logger.debug("Column uses MySQL character set '{}', which has no mapping to a Java character set, will try it in lowercase", mySqlCharsetName);
|
logger.debug("Column uses MySQL character set '{}', which has no mapping to a Java character set, will try it in lowercase", mySqlCharsetName);
|
||||||
encoding = MySqlConnection.getJavaEncodingForMysqlCharSet(mySqlCharsetName.toLowerCase());
|
encoding = connectorAdapter.getJavaEncodingForCharSet(mySqlCharsetName.toLowerCase());
|
||||||
}
|
}
|
||||||
if (encoding == null) {
|
if (encoding == null) {
|
||||||
logger.warn("Column uses MySQL character set '{}', which has no mapping to a Java character set", mySqlCharsetName);
|
logger.warn("Column uses MySQL character set '{}', which has no mapping to a Java character set", mySqlCharsetName);
|
||||||
|
@ -0,0 +1,295 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy;
|
||||||
|
|
||||||
|
import static io.debezium.util.Strings.isNullOrEmpty;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.security.GeneralSecurityException;
|
||||||
|
import java.security.KeyStore;
|
||||||
|
import java.security.KeyStoreException;
|
||||||
|
import java.security.NoSuchAlgorithmException;
|
||||||
|
import java.security.UnrecoverableKeyException;
|
||||||
|
import java.security.cert.CertificateException;
|
||||||
|
import java.security.cert.X509Certificate;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.ThreadFactory;
|
||||||
|
|
||||||
|
import javax.net.ssl.KeyManager;
|
||||||
|
import javax.net.ssl.KeyManagerFactory;
|
||||||
|
import javax.net.ssl.SSLContext;
|
||||||
|
import javax.net.ssl.TrustManager;
|
||||||
|
import javax.net.ssl.TrustManagerFactory;
|
||||||
|
import javax.net.ssl.X509TrustManager;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.github.shyiko.mysql.binlog.BinaryLogClient;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.Event;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.EventData;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.EventHeaderV4;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.EventType;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.TableMapEventData;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.TransactionPayloadEventData;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializationException;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.deserialization.GtidEventDataDeserializer;
|
||||||
|
import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
|
||||||
|
import com.github.shyiko.mysql.binlog.network.DefaultSSLSocketFactory;
|
||||||
|
import com.github.shyiko.mysql.binlog.network.SSLMode;
|
||||||
|
import com.github.shyiko.mysql.binlog.network.SSLSocketFactory;
|
||||||
|
|
||||||
|
import io.debezium.DebeziumException;
|
||||||
|
import io.debezium.config.CommonConnectorConfig.EventProcessingFailureHandlingMode;
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
|
import io.debezium.connector.mysql.EventDataDeserializationExceptionData;
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig.SecureConnectionMode;
|
||||||
|
import io.debezium.connector.mysql.RowDeserializers;
|
||||||
|
import io.debezium.connector.mysql.StopEventDataDeserializer;
|
||||||
|
import io.debezium.connector.mysql.TransactionPayloadDeserializer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public abstract class AbstractBinaryLogClientConfigurator implements BinaryLogClientConfigurator {
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractBinaryLogClientConfigurator.class);
|
||||||
|
|
||||||
|
private final MySqlConnectorConfig connectorConfig;
|
||||||
|
private final float heartbeatIntervalFactor = 0.8f;
|
||||||
|
private final EventProcessingFailureHandlingMode eventDeserializationFailureHandlingMode;
|
||||||
|
|
||||||
|
public AbstractBinaryLogClientConfigurator(MySqlConnectorConfig connectorConfig) {
|
||||||
|
this.connectorConfig = connectorConfig;
|
||||||
|
this.eventDeserializationFailureHandlingMode = connectorConfig.getEventProcessingFailureHandlingMode();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BinaryLogClient configure(BinaryLogClient client, ThreadFactory threadFactory, AbstractConnectorConnection connection) {
|
||||||
|
client.setThreadFactory(threadFactory);
|
||||||
|
client.setServerId(connectorConfig.serverId());
|
||||||
|
client.setSSLMode(sslModeFor(connectorConfig.sslMode()));
|
||||||
|
|
||||||
|
if (connectorConfig.sslModeEnabled()) {
|
||||||
|
SSLSocketFactory sslSocketFactory = getBinlogSslSocketFactory(connectorConfig, connection);
|
||||||
|
if (sslSocketFactory != null) {
|
||||||
|
client.setSslSocketFactory(sslSocketFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
configureReplicaCompatibility(client);
|
||||||
|
|
||||||
|
Configuration configuration = connectorConfig.getConfig();
|
||||||
|
client.setKeepAlive(configuration.getBoolean(MySqlConnectorConfig.KEEP_ALIVE));
|
||||||
|
final long keepAliveInterval = configuration.getLong(MySqlConnectorConfig.KEEP_ALIVE_INTERVAL_MS);
|
||||||
|
client.setKeepAliveInterval(keepAliveInterval);
|
||||||
|
// Considering heartbeatInterval should be less than keepAliveInterval, we use the heartbeatIntervalFactor
|
||||||
|
// multiply by keepAliveInterval and set the result value to heartbeatInterval.The default value of heartbeatIntervalFactor
|
||||||
|
// is 0.8, and we believe the left time (0.2 * keepAliveInterval) is enough to process the packet received from the MySQL server.
|
||||||
|
client.setHeartbeatInterval((long) (keepAliveInterval * heartbeatIntervalFactor));
|
||||||
|
|
||||||
|
client.setEventDeserializer(createEventDeserializer());
|
||||||
|
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected EventDeserializer createEventDeserializer() {
|
||||||
|
// Set up the event deserializer with additional type(s) ...
|
||||||
|
final Map<Long, TableMapEventData> tableMapEventByTableId = new HashMap<>();
|
||||||
|
EventDeserializer eventDeserializer = new EventDeserializer() {
|
||||||
|
@Override
|
||||||
|
public Event nextEvent(ByteArrayInputStream inputStream) throws IOException {
|
||||||
|
try {
|
||||||
|
// Delegate to the superclass ...
|
||||||
|
Event event = super.nextEvent(inputStream);
|
||||||
|
|
||||||
|
// We have to record the most recent TableMapEventData for each table number for our custom deserializers ...
|
||||||
|
if (event.getHeader().getEventType() == EventType.TABLE_MAP) {
|
||||||
|
TableMapEventData tableMapEvent = event.getData();
|
||||||
|
tableMapEventByTableId.put(tableMapEvent.getTableId(), tableMapEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
// DBZ-2663 Handle for transaction payload and capture the table map event and add it to the map
|
||||||
|
if (event.getHeader().getEventType() == EventType.TRANSACTION_PAYLOAD) {
|
||||||
|
TransactionPayloadEventData transactionPayloadEventData = (TransactionPayloadEventData) event.getData();
|
||||||
|
/**
|
||||||
|
* Loop over the uncompressed events in the transaction payload event and add the table map
|
||||||
|
* event in the map of table events
|
||||||
|
**/
|
||||||
|
for (Event uncompressedEvent : transactionPayloadEventData.getUncompressedEvents()) {
|
||||||
|
if (uncompressedEvent.getHeader().getEventType() == EventType.TABLE_MAP
|
||||||
|
&& uncompressedEvent.getData() != null) {
|
||||||
|
TableMapEventData tableMapEvent = (TableMapEventData) uncompressedEvent.getData();
|
||||||
|
tableMapEventByTableId.put(tableMapEvent.getTableId(), tableMapEvent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DBZ-5126 Clean cache on rotate event to prevent it from growing indefinitely.
|
||||||
|
if (event.getHeader().getEventType() == EventType.ROTATE && event.getHeader().getTimestamp() != 0) {
|
||||||
|
tableMapEventByTableId.clear();
|
||||||
|
}
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
// DBZ-217 In case an event couldn't be read we create a pseudo-event for the sake of logging
|
||||||
|
catch (EventDataDeserializationException edde) {
|
||||||
|
// DBZ-3095 As of Java 15, when reaching EOF in the binlog stream, the polling loop in
|
||||||
|
// BinaryLogClient#listenForEventPackets() keeps returning values != -1 from peek();
|
||||||
|
// this causes the loop to never finish
|
||||||
|
// Propagating the exception (either EOF or socket closed) causes the loop to be aborted
|
||||||
|
// in this case
|
||||||
|
if (edde.getCause() instanceof IOException) {
|
||||||
|
throw edde;
|
||||||
|
}
|
||||||
|
|
||||||
|
EventHeaderV4 header = new EventHeaderV4();
|
||||||
|
header.setEventType(EventType.INCIDENT);
|
||||||
|
header.setTimestamp(edde.getEventHeader().getTimestamp());
|
||||||
|
header.setServerId(edde.getEventHeader().getServerId());
|
||||||
|
|
||||||
|
if (edde.getEventHeader() instanceof EventHeaderV4) {
|
||||||
|
header.setEventLength(((EventHeaderV4) edde.getEventHeader()).getEventLength());
|
||||||
|
header.setNextPosition(((EventHeaderV4) edde.getEventHeader()).getNextPosition());
|
||||||
|
header.setFlags(((EventHeaderV4) edde.getEventHeader()).getFlags());
|
||||||
|
}
|
||||||
|
|
||||||
|
EventData data = new EventDataDeserializationExceptionData(edde);
|
||||||
|
return new Event(header, data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Add our custom deserializers ...
|
||||||
|
eventDeserializer.setEventDataDeserializer(EventType.STOP, new StopEventDataDeserializer());
|
||||||
|
eventDeserializer.setEventDataDeserializer(EventType.GTID, new GtidEventDataDeserializer());
|
||||||
|
eventDeserializer.setEventDataDeserializer(EventType.WRITE_ROWS,
|
||||||
|
new RowDeserializers.WriteRowsDeserializer(tableMapEventByTableId, eventDeserializationFailureHandlingMode));
|
||||||
|
eventDeserializer.setEventDataDeserializer(EventType.UPDATE_ROWS,
|
||||||
|
new RowDeserializers.UpdateRowsDeserializer(tableMapEventByTableId, eventDeserializationFailureHandlingMode));
|
||||||
|
eventDeserializer.setEventDataDeserializer(EventType.DELETE_ROWS,
|
||||||
|
new RowDeserializers.DeleteRowsDeserializer(tableMapEventByTableId, eventDeserializationFailureHandlingMode));
|
||||||
|
eventDeserializer.setEventDataDeserializer(EventType.EXT_WRITE_ROWS,
|
||||||
|
new RowDeserializers.WriteRowsDeserializer(
|
||||||
|
tableMapEventByTableId, eventDeserializationFailureHandlingMode).setMayContainExtraInformation(true));
|
||||||
|
eventDeserializer.setEventDataDeserializer(EventType.EXT_UPDATE_ROWS,
|
||||||
|
new RowDeserializers.UpdateRowsDeserializer(
|
||||||
|
tableMapEventByTableId, eventDeserializationFailureHandlingMode).setMayContainExtraInformation(true));
|
||||||
|
eventDeserializer.setEventDataDeserializer(EventType.EXT_DELETE_ROWS,
|
||||||
|
new RowDeserializers.DeleteRowsDeserializer(
|
||||||
|
tableMapEventByTableId, eventDeserializationFailureHandlingMode).setMayContainExtraInformation(true));
|
||||||
|
eventDeserializer.setEventDataDeserializer(EventType.TRANSACTION_PAYLOAD,
|
||||||
|
new TransactionPayloadDeserializer(tableMapEventByTableId, eventDeserializationFailureHandlingMode));
|
||||||
|
|
||||||
|
return eventDeserializer;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventType getIncludeSqlQueryEventType() {
|
||||||
|
return EventType.ROWS_QUERY;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected MySqlConnectorConfig getConnectorConfig() {
|
||||||
|
return connectorConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void configureReplicaCompatibility(BinaryLogClient client) {
|
||||||
|
// default is a no-op
|
||||||
|
}
|
||||||
|
|
||||||
|
private SSLMode sslModeFor(SecureConnectionMode mode) {
|
||||||
|
switch (mode) {
|
||||||
|
case DISABLED:
|
||||||
|
return SSLMode.DISABLED;
|
||||||
|
case PREFERRED:
|
||||||
|
return SSLMode.PREFERRED;
|
||||||
|
case REQUIRED:
|
||||||
|
return SSLMode.REQUIRED;
|
||||||
|
case VERIFY_CA:
|
||||||
|
return SSLMode.VERIFY_CA;
|
||||||
|
case VERIFY_IDENTITY:
|
||||||
|
return SSLMode.VERIFY_IDENTITY;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SSLSocketFactory getBinlogSslSocketFactory(MySqlConnectorConfig connectorConfig, AbstractConnectorConnection connection) {
|
||||||
|
String acceptedTlsVersion = connection.getSessionVariableForSslVersion();
|
||||||
|
if (!isNullOrEmpty(acceptedTlsVersion)) {
|
||||||
|
SSLMode sslMode = sslModeFor(connectorConfig.sslMode());
|
||||||
|
LOGGER.info("Enable ssl " + sslMode + " mode for connector " + connectorConfig.getLogicalName());
|
||||||
|
|
||||||
|
final char[] keyPasswordArray = connection.connectionConfig().sslKeyStorePassword();
|
||||||
|
final String keyFilename = connection.connectionConfig().sslKeyStore();
|
||||||
|
final char[] trustPasswordArray = connection.connectionConfig().sslTrustStorePassword();
|
||||||
|
final String trustFilename = connection.connectionConfig().sslTrustStore();
|
||||||
|
KeyManager[] keyManagers = null;
|
||||||
|
if (keyFilename != null) {
|
||||||
|
try {
|
||||||
|
KeyStore ks = connection.loadKeyStore(keyFilename, keyPasswordArray);
|
||||||
|
|
||||||
|
KeyManagerFactory kmf = KeyManagerFactory.getInstance("NewSunX509");
|
||||||
|
kmf.init(ks, keyPasswordArray);
|
||||||
|
|
||||||
|
keyManagers = kmf.getKeyManagers();
|
||||||
|
}
|
||||||
|
catch (KeyStoreException | NoSuchAlgorithmException | UnrecoverableKeyException e) {
|
||||||
|
throw new DebeziumException("Could not load keystore", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TrustManager[] trustManagers;
|
||||||
|
try {
|
||||||
|
KeyStore ks = null;
|
||||||
|
if (trustFilename != null) {
|
||||||
|
ks = connection.loadKeyStore(trustFilename, trustPasswordArray);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ks == null && (sslMode == SSLMode.PREFERRED || sslMode == SSLMode.REQUIRED)) {
|
||||||
|
trustManagers = new TrustManager[]{
|
||||||
|
new X509TrustManager() {
|
||||||
|
@Override
|
||||||
|
public void checkClientTrusted(X509Certificate[] x509Certificates, String s)
|
||||||
|
throws CertificateException {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void checkServerTrusted(X509Certificate[] x509Certificates, String s)
|
||||||
|
throws CertificateException {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public X509Certificate[] getAcceptedIssuers() {
|
||||||
|
return new X509Certificate[0];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
|
||||||
|
tmf.init(ks);
|
||||||
|
trustManagers = tmf.getTrustManagers();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (KeyStoreException | NoSuchAlgorithmException e) {
|
||||||
|
throw new DebeziumException("Could not load truststore", e);
|
||||||
|
}
|
||||||
|
// DBZ-1208 Resembles the logic from the upstream BinaryLogClient, only that
|
||||||
|
// the accepted TLS version is passed to the constructed factory
|
||||||
|
final KeyManager[] finalKMS = keyManagers;
|
||||||
|
return new DefaultSSLSocketFactory(acceptedTlsVersion) {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void initSSLContext(SSLContext sc) throws GeneralSecurityException {
|
||||||
|
sc.init(finalKMS, trustManagers, null);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,167 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy;
|
||||||
|
|
||||||
|
import static io.debezium.config.CommonConnectorConfig.DATABASE_CONFIG_PREFIX;
|
||||||
|
import static io.debezium.config.CommonConnectorConfig.DRIVER_CONFIG_PREFIX;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
|
||||||
|
import io.debezium.config.CommonConnectorConfig;
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
|
import io.debezium.config.Field;
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
|
import io.debezium.jdbc.JdbcConfiguration;
|
||||||
|
import io.debezium.jdbc.JdbcConnection;
|
||||||
|
import io.debezium.util.Strings;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public abstract class AbstractConnectionConfiguration implements ConnectionConfiguration {
|
||||||
|
|
||||||
|
public static final String URL_PATTERN = "${protocol}://${hostname}:${port}/?useInformationSchema=true&nullCatalogMeansCurrent=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=CONVERT_TO_NULL&connectTimeout=${connectTimeout}";
|
||||||
|
|
||||||
|
private final JdbcConfiguration jdbcConfig;
|
||||||
|
private final JdbcConnection.ConnectionFactory factory;
|
||||||
|
private final Configuration config;
|
||||||
|
|
||||||
|
public AbstractConnectionConfiguration(Configuration config) {
|
||||||
|
// Set up the JDBC connection without actually connecting, with extra MySQL-specific properties
|
||||||
|
// to give us better JDBC database metadata behavior, including using UTF-8 for the client-side character encoding
|
||||||
|
// per https://dev.mysql.com/doc/connector-j/5.1/en/connector-j-reference-charsets.html
|
||||||
|
this.config = config;
|
||||||
|
final boolean useSSL = sslModeEnabled();
|
||||||
|
final Configuration dbConfig = config
|
||||||
|
.edit()
|
||||||
|
.withDefault(MySqlConnectorConfig.PORT, MySqlConnectorConfig.PORT.defaultValue())
|
||||||
|
.withDefault(MySqlConnectorConfig.JDBC_PROTOCOL, MySqlConnectorConfig.JDBC_PROTOCOL.defaultValue())
|
||||||
|
.build()
|
||||||
|
.subset(DATABASE_CONFIG_PREFIX, true)
|
||||||
|
.merge(config.subset(DRIVER_CONFIG_PREFIX, true));
|
||||||
|
|
||||||
|
final Configuration.Builder jdbcConfigBuilder = dbConfig
|
||||||
|
.edit()
|
||||||
|
.with("connectTimeout", Long.toString(getConnectionTimeout().toMillis()))
|
||||||
|
.with("sslMode", sslMode().getValue());
|
||||||
|
|
||||||
|
if (useSSL) {
|
||||||
|
if (!Strings.isNullOrBlank(sslTrustStore())) {
|
||||||
|
jdbcConfigBuilder.with("trustCertificateKeyStoreUrl", "file:" + sslTrustStore());
|
||||||
|
}
|
||||||
|
if (sslTrustStorePassword() != null) {
|
||||||
|
jdbcConfigBuilder.with("trustCertificateKeyStorePassword", String.valueOf(sslTrustStorePassword()));
|
||||||
|
}
|
||||||
|
if (!Strings.isNullOrBlank(sslKeyStore())) {
|
||||||
|
jdbcConfigBuilder.with("clientCertificateKeyStoreUrl", "file:" + sslKeyStore());
|
||||||
|
}
|
||||||
|
if (sslKeyStorePassword() != null) {
|
||||||
|
jdbcConfigBuilder.with("clientCertificateKeyStorePassword", String.valueOf(sslKeyStorePassword()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
jdbcConfigBuilder.with(getConnectionTimeZonePropertyName(), resolveConnectionTimeZone(dbConfig));
|
||||||
|
|
||||||
|
// Set and remove options to prevent potential vulnerabilities
|
||||||
|
jdbcConfigBuilder
|
||||||
|
.with("allowLoadLocalInfile", "false")
|
||||||
|
.with("allowUrlInLocalInfile", "false")
|
||||||
|
.with("autoDeserialize", false)
|
||||||
|
.without("queryInterceptors");
|
||||||
|
|
||||||
|
this.jdbcConfig = JdbcConfiguration.adapt(jdbcConfigBuilder.build());
|
||||||
|
String driverClassName = this.config.getString(MySqlConnectorConfig.JDBC_DRIVER);
|
||||||
|
Field protocol = MySqlConnectorConfig.JDBC_PROTOCOL;
|
||||||
|
|
||||||
|
factory = JdbcConnection.patternBasedFactory(URL_PATTERN, driverClassName, getClass().getClassLoader(), protocol);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public JdbcConfiguration config() {
|
||||||
|
return jdbcConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Configuration originalConfig() {
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public JdbcConnection.ConnectionFactory factory() {
|
||||||
|
return factory;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String username() {
|
||||||
|
return config.getString(MySqlConnectorConfig.USER);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String password() {
|
||||||
|
return config.getString(MySqlConnectorConfig.PASSWORD);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String hostname() {
|
||||||
|
return config.getString(MySqlConnectorConfig.HOSTNAME);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int port() {
|
||||||
|
return config.getInteger(MySqlConnectorConfig.PORT);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MySqlConnectorConfig.SecureConnectionMode sslMode() {
|
||||||
|
String mode = config.getString(MySqlConnectorConfig.SSL_MODE);
|
||||||
|
return MySqlConnectorConfig.SecureConnectionMode.parse(mode);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean sslModeEnabled() {
|
||||||
|
return sslMode() != MySqlConnectorConfig.SecureConnectionMode.DISABLED;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String sslKeyStore() {
|
||||||
|
return config.getString(MySqlConnectorConfig.SSL_KEYSTORE);
|
||||||
|
}
|
||||||
|
|
||||||
|
public char[] sslKeyStorePassword() {
|
||||||
|
String password = config.getString(MySqlConnectorConfig.SSL_KEYSTORE_PASSWORD);
|
||||||
|
return Strings.isNullOrBlank(password) ? null : password.toCharArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String sslTrustStore() {
|
||||||
|
return config.getString(MySqlConnectorConfig.SSL_TRUSTSTORE);
|
||||||
|
}
|
||||||
|
|
||||||
|
public char[] sslTrustStorePassword() {
|
||||||
|
String password = config.getString(MySqlConnectorConfig.SSL_TRUSTSTORE_PASSWORD);
|
||||||
|
return Strings.isNullOrBlank(password) ? null : password.toCharArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Duration getConnectionTimeout() {
|
||||||
|
return Duration.ofMillis(config.getLong(MySqlConnectorConfig.CONNECTION_TIMEOUT_MS));
|
||||||
|
}
|
||||||
|
|
||||||
|
public CommonConnectorConfig.EventProcessingFailureHandlingMode eventProcessingFailureHandlingMode() {
|
||||||
|
String mode = config.getString(CommonConnectorConfig.EVENT_PROCESSING_FAILURE_HANDLING_MODE);
|
||||||
|
if (mode == null) {
|
||||||
|
mode = config.getString(MySqlConnectorConfig.EVENT_DESERIALIZATION_FAILURE_HANDLING_MODE);
|
||||||
|
}
|
||||||
|
return CommonConnectorConfig.EventProcessingFailureHandlingMode.parse(mode);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CommonConnectorConfig.EventProcessingFailureHandlingMode inconsistentSchemaHandlingMode() {
|
||||||
|
String mode = config.getString(MySqlConnectorConfig.INCONSISTENT_SCHEMA_HANDLING_MODE);
|
||||||
|
return CommonConnectorConfig.EventProcessingFailureHandlingMode.parse(mode);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract String getConnectionTimeZonePropertyName();
|
||||||
|
|
||||||
|
protected abstract String resolveConnectionTimeZone(Configuration dbConfig);
|
||||||
|
}
|
@ -0,0 +1,502 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy;
|
||||||
|
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.OptionalLong;
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.DebeziumException;
|
||||||
|
import io.debezium.connector.mysql.GtidSet;
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
|
import io.debezium.connector.mysql.MySqlFieldReader;
|
||||||
|
import io.debezium.connector.mysql.MySqlOffsetContext;
|
||||||
|
import io.debezium.connector.mysql.MySqlSystemVariables;
|
||||||
|
import io.debezium.jdbc.JdbcConnection;
|
||||||
|
import io.debezium.relational.Column;
|
||||||
|
import io.debezium.relational.Table;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
import io.debezium.util.Strings;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An abstract common implementation of {@link JdbcConnection} for MySQL and MariaDB.
|
||||||
|
*
|
||||||
|
* @author Jiri Pechanec, Randall Hauch, Chris Cranford
|
||||||
|
*/
|
||||||
|
public abstract class AbstractConnectorConnection extends JdbcConnection {
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractConnectorConnection.class);
|
||||||
|
|
||||||
|
private static final String SQL_SHOW_SYSTEM_VARIABLES = "SHOW VARIABLES";
|
||||||
|
private static final String SQL_SHOW_SYSTEM_VARIABLES_CHARACTER_SET = "SHOW VARIABLES WHERE Variable_name IN ('character_set_server','collation_server')";
|
||||||
|
private static final String SQL_SHOW_SESSION_VARIABLE_SSL_VERSION = "SHOW SESSION STATUS LIKE 'Ssl_version'";
|
||||||
|
private static final String QUOTED_CHARACTER = "`";
|
||||||
|
|
||||||
|
private final ConnectionConfiguration connectionConfig;
|
||||||
|
// todo: rename to drop the prefix on the interface??
|
||||||
|
private final MySqlFieldReader fieldReader;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new connection using the supplied configuration.
|
||||||
|
*
|
||||||
|
* @param configuration the connection configuration instance, may not be null
|
||||||
|
* @param fieldReader the configured snapshot fetch size
|
||||||
|
*/
|
||||||
|
public AbstractConnectorConnection(ConnectionConfiguration configuration, MySqlFieldReader fieldReader) {
|
||||||
|
super(configuration.config(), configuration.factory(), QUOTED_CHARACTER, QUOTED_CHARACTER);
|
||||||
|
this.connectionConfig = configuration;
|
||||||
|
this.fieldReader = fieldReader;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object getColumnValue(ResultSet rs, int columnIndex, Column column, Table table) throws SQLException {
|
||||||
|
return fieldReader.readField(rs, columnIndex, column, table);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String quotedTableIdString(TableId tableId) {
|
||||||
|
return tableId.toQuotedString('`');
|
||||||
|
}
|
||||||
|
|
||||||
|
public String connectionString() {
|
||||||
|
return connectionString(AbstractConnectionConfiguration.URL_PATTERN);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ConnectionConfiguration connectionConfig() {
|
||||||
|
return connectionConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine whether the current user has the named privilege. If the user has the "ALL" privilege, this
|
||||||
|
* method will always return {@code true}.
|
||||||
|
*
|
||||||
|
* @param grantName the name of the database privilege; may not be null
|
||||||
|
* @return {@code true} if the user has the named privilege; {@code false} otherwise
|
||||||
|
*/
|
||||||
|
public boolean userHasPrivileges(String grantName) {
|
||||||
|
try {
|
||||||
|
return queryAndMap("SHOW GRANTS FOR CURRENT_USER", rs -> {
|
||||||
|
while (rs.next()) {
|
||||||
|
String grants = rs.getString(1);
|
||||||
|
LOGGER.debug(grants);
|
||||||
|
if (grants == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
grants = grants.toUpperCase();
|
||||||
|
if (grants.contains("ALL") || grants.contains(grantName.toUpperCase())) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Unexpected error while connecting to database and looking at privileges for current user: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine the earliest binlog filename that is still available in the server.
|
||||||
|
*
|
||||||
|
* @return the name of the earliest binlog filename, or null if there are none
|
||||||
|
*/
|
||||||
|
public String earliestBinlogFilename() {
|
||||||
|
// Accumulate the available binlog filenames ...
|
||||||
|
List<String> logNames = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
LOGGER.info("Checking all known binlogs from the database");
|
||||||
|
query("SHOW BINARY LOGS", rs -> {
|
||||||
|
while (rs.next()) {
|
||||||
|
logNames.add(rs.getString(1));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Unexpected error while connecting to the database and looking for binary logs: ", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (logNames.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return logNames.get(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Query the database server and get the list of binlog files that are currently available.
|
||||||
|
*
|
||||||
|
* @return list of binlog files
|
||||||
|
*/
|
||||||
|
public List<String> availableBinlogFiles() {
|
||||||
|
List<String> logNames = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
LOGGER.info("Get all known binlogs");
|
||||||
|
query("SHOW BINARY LOGS", rs -> {
|
||||||
|
while (rs.next()) {
|
||||||
|
logNames.add(rs.getString(1));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return logNames;
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Unexpected error while connecting to the database and looking for binary logs: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the estimated table size, aka number of rows.
|
||||||
|
*
|
||||||
|
* @param tableId the table identifier; should never be null
|
||||||
|
* @return an optional long-value that may be empty if no data is available or an exception occurred
|
||||||
|
*/
|
||||||
|
public OptionalLong getEstimatedTableSize(TableId tableId) {
|
||||||
|
try {
|
||||||
|
// Choose how we create statements based on the # of rows.
|
||||||
|
// This is approximate and less accurate then COUNT(*),
|
||||||
|
// but far more efficient for large InnoDB tables.
|
||||||
|
execute("USE `" + tableId.catalog() + "`;");
|
||||||
|
return queryAndMap("SHOW TABLE STATUS LIKE '" + tableId.table() + "';", rs -> {
|
||||||
|
if (rs.next()) {
|
||||||
|
return OptionalLong.of((rs.getLong(5)));
|
||||||
|
}
|
||||||
|
return OptionalLong.empty();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
LOGGER.debug("Error while getting number of rows in table {}: {}", tableId, e.getMessage(), e);
|
||||||
|
}
|
||||||
|
return OptionalLong.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the charset-related system variables.
|
||||||
|
*
|
||||||
|
* @return the system variables that are related to server character sets; never null
|
||||||
|
*/
|
||||||
|
public Map<String, String> readCharsetSystemVariables() {
|
||||||
|
// Read the system variables from the MySQL instance and get the current database name ...
|
||||||
|
LOGGER.debug("Reading charset-related system variables before parsing DDL history.");
|
||||||
|
return querySystemVariables(SQL_SHOW_SYSTEM_VARIABLES_CHARACTER_SET);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Executes a {@code SET} statement, setting each variable with it's specified value.
|
||||||
|
*
|
||||||
|
* @param variables key/value variable names as keys and the value(s) to be set
|
||||||
|
* @return the constructed {@code SET} database statement; never null
|
||||||
|
*/
|
||||||
|
public String setStatementFor(Map<String, String> variables) {
|
||||||
|
StringBuilder sb = new StringBuilder("SET ");
|
||||||
|
boolean first = true;
|
||||||
|
List<String> varNames = new ArrayList<>(variables.keySet());
|
||||||
|
Collections.sort(varNames);
|
||||||
|
for (String varName : varNames) {
|
||||||
|
if (first) {
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
sb.append(", ");
|
||||||
|
}
|
||||||
|
sb.append(varName).append("=");
|
||||||
|
String value = variables.get(varName);
|
||||||
|
if (value == null) {
|
||||||
|
value = "";
|
||||||
|
}
|
||||||
|
if (value.contains(",") || value.contains(";")) {
|
||||||
|
value = "'" + value + "'";
|
||||||
|
}
|
||||||
|
sb.append(value);
|
||||||
|
}
|
||||||
|
return sb.append(";").toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determines whether the binlog format used by the database server is {@code binlog_row_image='FULL'}.
|
||||||
|
*
|
||||||
|
* @return {@code true} if the {@code binlog_row_image} is set to {@code FULL}, {@code false} otherwise
|
||||||
|
*/
|
||||||
|
public boolean isBinlogRowImageFull() {
|
||||||
|
try {
|
||||||
|
final String rowImage = queryAndMap("SHOW GLOBAL VARIABLES LIKE 'binlog_row_image'", rs -> {
|
||||||
|
if (rs.next()) {
|
||||||
|
return rs.getString(2);
|
||||||
|
}
|
||||||
|
// This setting was introduced in MySQL 5.6+ with default of 'FULL'.
|
||||||
|
// For older versions, assume 'FULL'.
|
||||||
|
return "FULL";
|
||||||
|
});
|
||||||
|
LOGGER.debug("binlog_row_image={}", rowImage);
|
||||||
|
return "FULL".equalsIgnoreCase(rowImage);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Unexpected error while connecting to the database and looking at BINLOG_ROW_IMAGE mode: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine whether the database server has the row-level binlog enabled.
|
||||||
|
*
|
||||||
|
* @return {@code true} if the server's {@code binlog_format} is set to {@code ROW}, {@code false} otherwise
|
||||||
|
*/
|
||||||
|
public boolean isBinlogFormatRow() {
|
||||||
|
try {
|
||||||
|
final String mode = queryAndMap("SHOW GLOBAL VARIABLES LIKE 'binlog_format'", rs -> rs.next() ? rs.getString(2) : "");
|
||||||
|
LOGGER.debug("binlog_format={}", mode);
|
||||||
|
return "ROW".equalsIgnoreCase(mode);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Unexpected error while connecting to the database and looking at BINLOG_FORMAT mode: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the database server's default character sets for existing databases.
|
||||||
|
*
|
||||||
|
* @return the map of database names and their default character sets; never null
|
||||||
|
*/
|
||||||
|
public Map<String, DatabaseLocales> readDatabaseCollations() {
|
||||||
|
LOGGER.debug("Reading default database charsets");
|
||||||
|
try {
|
||||||
|
return queryAndMap("SELECT schema_name, default_character_set_name, default_collation_name FROM information_schema.schemata", rs -> {
|
||||||
|
final Map<String, DatabaseLocales> charsets = new HashMap<>();
|
||||||
|
while (rs.next()) {
|
||||||
|
String dbName = rs.getString(1);
|
||||||
|
String charset = rs.getString(2);
|
||||||
|
String collation = rs.getString(3);
|
||||||
|
if (dbName != null && (charset != null || collation != null)) {
|
||||||
|
charsets.put(dbName, new DatabaseLocales(charset, collation));
|
||||||
|
LOGGER.debug("\t{} = {}, {}",
|
||||||
|
Strings.pad(dbName, 45, ' '),
|
||||||
|
Strings.pad(charset, 45, ' '),
|
||||||
|
Strings.pad(collation, 45, ' '));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return charsets;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Error reading default database charsets: " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return whether the table identifiers are case-sensitive.
|
||||||
|
*
|
||||||
|
* @return {@code true} if the table identifiers are case-sensitive, {@code false} otherwise
|
||||||
|
*/
|
||||||
|
public boolean isTableIdCaseSensitive() {
|
||||||
|
return !"0".equals(readSystemVariables().get(MySqlSystemVariables.LOWER_CASE_TABLE_NAMES));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine whether the binlog position as set in the offset details is available on the server.
|
||||||
|
*
|
||||||
|
* @param config the connector configuration; should not be null
|
||||||
|
* @param gtid the GTID from the connector offsets; may be null
|
||||||
|
* @param binlogFileName the binlog file name from the connector offsets; may be null
|
||||||
|
* @return {@code true} if the binlog position is available, {@code false} otherwise
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("BooleanMethodIsAlwaysInverted")
|
||||||
|
public boolean isBinlogPositionAvailable(MySqlConnectorConfig config, String gtid, String binlogFileName) {
|
||||||
|
if (gtid != null) {
|
||||||
|
if (gtid.trim().isEmpty()) {
|
||||||
|
// Start at the beginning
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
GtidSet availableGtidSet = knownGtidSet();
|
||||||
|
if (availableGtidSet.isEmpty()) {
|
||||||
|
// Last offsets had GTIDs but the server does not use them
|
||||||
|
LOGGER.info("Connector used GTIDs previously, but server does not know of any GTIDs or they are not enabled");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// GTIDs are enabled, used previously, retain only the ranges allowed
|
||||||
|
GtidSet gtidSet = createGtidSet(gtid).retainAll(config.gtidSourceFilter());
|
||||||
|
|
||||||
|
// Get the GTID set that is available on the server
|
||||||
|
if (gtidSet.isContainedWithin(availableGtidSet)) {
|
||||||
|
LOGGER.info("The current GTID set {} does not contain the GTID set required by the connector {}",
|
||||||
|
availableGtidSet, gtidSet);
|
||||||
|
|
||||||
|
final GtidSet knownServerSet = availableGtidSet.retainAll(config.gtidSourceFilter());
|
||||||
|
final GtidSet gtidSetToReplicate = subtractGtidSet(knownServerSet, gtidSet);
|
||||||
|
final GtidSet purgedGtidSet = purgedGtidSet();
|
||||||
|
LOGGER.info("Serer has already purged {} GTIDs", purgedGtidSet);
|
||||||
|
|
||||||
|
final GtidSet nonPurgedGtidSetTemplate = subtractGtidSet(gtidSetToReplicate, purgedGtidSet);
|
||||||
|
LOGGER.info("GTIDs known by the server but not processed yet {}, for replication are available only {}",
|
||||||
|
gtidSetToReplicate, nonPurgedGtidSetTemplate);
|
||||||
|
|
||||||
|
if (!gtidSetToReplicate.equals(nonPurgedGtidSetTemplate)) {
|
||||||
|
LOGGER.info("Some of the GTIDs needed to replicate have been already purged");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
LOGGER.info("Connector last known GTIDs are {}, but server has {}", gtidSet, availableGtidSet);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Strings.isNullOrBlank(binlogFileName)) {
|
||||||
|
// Start at the current position
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Accumulate the available binlog filenames, and compare with the one we're supposed to use
|
||||||
|
List<String> logNames = availableBinlogFiles();
|
||||||
|
boolean found = logNames.stream().anyMatch(binlogFileName::equals);
|
||||||
|
if (!found && LOGGER.isInfoEnabled()) {
|
||||||
|
LOGGER.info("Connector requires binlog file '{}', but server only has {}", binlogFileName, String.join(", ", logNames));
|
||||||
|
}
|
||||||
|
else if (found && LOGGER.isInfoEnabled()) {
|
||||||
|
LOGGER.info("Server has the binlog file '{}' required by the connector", binlogFileName);
|
||||||
|
}
|
||||||
|
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine whether the server has enabled GTID support.
|
||||||
|
*
|
||||||
|
* @return {@code false} if the server has not enabled GTIDs, {@code true} otherwise
|
||||||
|
*/
|
||||||
|
public abstract boolean isGtidModeEnabled();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the most recent executed GTID set or position.
|
||||||
|
*
|
||||||
|
* @return the string representation of the most recent executed GTID set or position; never null but
|
||||||
|
* will be empty if the server does not support or has not processed any GTID
|
||||||
|
*/
|
||||||
|
public abstract GtidSet knownGtidSet();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determines the difference between two GTID sets.
|
||||||
|
*
|
||||||
|
* @param set1 the first set; should never be null
|
||||||
|
* @param set2 the second set; should never be null
|
||||||
|
* @return the subtraction of the two sets in a new GtidSet instance; never null
|
||||||
|
*/
|
||||||
|
public abstract GtidSet subtractGtidSet(GtidSet set1, GtidSet set2);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the purged GTID values from the server.
|
||||||
|
*
|
||||||
|
* @return A GTID set; may be empty of GTID support is not enabled or if none have been purged
|
||||||
|
*/
|
||||||
|
public abstract GtidSet purgedGtidSet();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply the include/exclude GTID source filters to the current {@link MySqlOffsetContext#gtidSet() GTID set} and merge them onto the
|
||||||
|
* currently available GTID set from a MySQL server.
|
||||||
|
*
|
||||||
|
* The merging behavior of this method might seem a bit strange at first. It's required in order for Debezium to consume a
|
||||||
|
* MySQL binlog that has multi-source replication enabled, if a failover has to occur. In such a case, the server that
|
||||||
|
* Debezium is failed over to might have a different set of sources, but still include the sources required for Debezium
|
||||||
|
* to continue to function. MySQL does not allow downstream replicas to connect if the GTID set does not contain GTIDs for
|
||||||
|
* all channels that the server is replicating from, even if the server does have the data needed by the client. To get
|
||||||
|
* around this, we can have Debezium merge its GTID set with whatever is on the server, so that MySQL will allow it to
|
||||||
|
* connect. See <a href="https://issues.jboss.org/browse/DBZ-143">DBZ-143</a> for details.
|
||||||
|
*
|
||||||
|
* This method does not mutate any state in the context.
|
||||||
|
*
|
||||||
|
* @param availableServerGtidSet the GTID set currently available in the MySQL server
|
||||||
|
* @param purgedServerGtid the GTID set already purged by the MySQL server
|
||||||
|
* @return A GTID set meant for consuming from a MySQL binlog; may return null if the SourceInfo has no GTIDs and therefore
|
||||||
|
* none were filtered
|
||||||
|
*/
|
||||||
|
|
||||||
|
public abstract GtidSet filterGtidSet(Predicate<String> gtidSourceFilter, String offsetGtids, GtidSet availableServerGtidSet, GtidSet purgedServerGtidSet);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the system variables.
|
||||||
|
*
|
||||||
|
* @return all the system variables; never null
|
||||||
|
*/
|
||||||
|
protected Map<String, String> readSystemVariables() {
|
||||||
|
// Read the system variables from the MySQL instance and get the current database name ...
|
||||||
|
LOGGER.debug("Reading system variables");
|
||||||
|
return querySystemVariables(SQL_SHOW_SYSTEM_VARIABLES);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the SSL version session variable.
|
||||||
|
*
|
||||||
|
* @return the session variable value related to the session SSL version
|
||||||
|
*/
|
||||||
|
protected String getSessionVariableForSslVersion() {
|
||||||
|
final String SSL_VERSION = "Ssl_version";
|
||||||
|
LOGGER.debug("Reading session variable for Ssl Version");
|
||||||
|
Map<String, String> sessionVariables = querySystemVariables(SQL_SHOW_SESSION_VARIABLE_SSL_VERSION);
|
||||||
|
if (!sessionVariables.isEmpty() && sessionVariables.containsKey(SSL_VERSION)) {
|
||||||
|
return sessionVariables.get(SSL_VERSION);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract GtidSet createGtidSet(String gtids);
|
||||||
|
|
||||||
|
private Map<String, String> querySystemVariables(String statement) {
|
||||||
|
final Map<String, String> variables = new HashMap<>();
|
||||||
|
try {
|
||||||
|
query(statement, rs -> {
|
||||||
|
while (rs.next()) {
|
||||||
|
String varName = rs.getString(1);
|
||||||
|
String value = rs.getString(2);
|
||||||
|
if (varName != null && value != null) {
|
||||||
|
variables.put(varName, value);
|
||||||
|
LOGGER.debug("\t{} = {}",
|
||||||
|
Strings.pad(varName, 45, ' '),
|
||||||
|
Strings.pad(value, 45, ' '));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Error reading MySQL variables: " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
|
||||||
|
return variables;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class DatabaseLocales {
|
||||||
|
private final String charset;
|
||||||
|
private final String collation;
|
||||||
|
|
||||||
|
public DatabaseLocales(String charset, String collation) {
|
||||||
|
this.charset = charset;
|
||||||
|
this.collation = collation;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void appendToDdlStatement(String dbName, StringBuilder ddl) {
|
||||||
|
if (charset != null) {
|
||||||
|
LOGGER.debug("Setting default charset '{}' for database '{}'", charset, dbName);
|
||||||
|
ddl.append(" CHARSET ").append(charset);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
LOGGER.info("Default database charset for '{}' not found", dbName);
|
||||||
|
}
|
||||||
|
if (collation != null) {
|
||||||
|
LOGGER.debug("Setting default collation '{}' for database '{}'", collation, dbName);
|
||||||
|
ddl.append(" COLLATE ").append(collation);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
LOGGER.info("Default database collation for '{}' not found", dbName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -3,22 +3,30 @@
|
|||||||
*
|
*
|
||||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
*/
|
*/
|
||||||
package io.debezium.connector.mysql;
|
package io.debezium.connector.mysql.strategy;
|
||||||
|
|
||||||
import java.util.function.Predicate;
|
import java.util.function.Predicate;
|
||||||
|
|
||||||
|
import io.debezium.annotation.VisibleForTesting;
|
||||||
|
import io.debezium.connector.mysql.GtidSet;
|
||||||
|
import io.debezium.connector.mysql.MySqlOffsetContext;
|
||||||
|
import io.debezium.connector.mysql.SourceInfo;
|
||||||
import io.debezium.document.Document;
|
import io.debezium.document.Document;
|
||||||
import io.debezium.relational.history.HistoryRecordComparator;
|
import io.debezium.relational.history.HistoryRecordComparator;
|
||||||
|
|
||||||
final class MySqlHistoryRecordComparator extends HistoryRecordComparator {
|
/**
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public abstract class AbstractHistoryRecordComparator extends HistoryRecordComparator {
|
||||||
|
|
||||||
private final Predicate<String> gtidSourceFilter;
|
private final Predicate<String> gtidSourceFilter;
|
||||||
|
|
||||||
MySqlHistoryRecordComparator(Predicate<String> gtidSourceFilter) {
|
public AbstractHistoryRecordComparator(Predicate<String> gtidSourceFilter) {
|
||||||
super();
|
|
||||||
this.gtidSourceFilter = gtidSourceFilter;
|
this.gtidSourceFilter = gtidSourceFilter;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected abstract GtidSet createGtidSet(String gtidSet);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Determine whether the first offset is at or before the point in time of the second
|
* Determine whether the first offset is at or before the point in time of the second
|
||||||
* offset, where the offsets are given in JSON representation of the maps returned by {@link MySqlOffsetContext#getOffset()}.
|
* offset, where the offsets are given in JSON representation of the maps returned by {@link MySqlOffsetContext#getOffset()}.
|
||||||
@ -36,15 +44,16 @@ final class MySqlHistoryRecordComparator extends HistoryRecordComparator {
|
|||||||
* @return {@code true} if the recorded position is at or before the desired position; or {@code false} otherwise
|
* @return {@code true} if the recorded position is at or before the desired position; or {@code false} otherwise
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
protected boolean isPositionAtOrBefore(Document recorded, Document desired) {
|
@VisibleForTesting
|
||||||
|
public boolean isPositionAtOrBefore(Document recorded, Document desired) {
|
||||||
String recordedGtidSetStr = recorded.getString(MySqlOffsetContext.GTID_SET_KEY);
|
String recordedGtidSetStr = recorded.getString(MySqlOffsetContext.GTID_SET_KEY);
|
||||||
String desiredGtidSetStr = desired.getString(MySqlOffsetContext.GTID_SET_KEY);
|
String desiredGtidSetStr = desired.getString(MySqlOffsetContext.GTID_SET_KEY);
|
||||||
if (desiredGtidSetStr != null) {
|
if (desiredGtidSetStr != null) {
|
||||||
// The desired position uses GTIDs, so we ideally compare using GTIDs ...
|
// The desired position uses GTIDs, so we ideally compare using GTIDs ...
|
||||||
if (recordedGtidSetStr != null) {
|
if (recordedGtidSetStr != null) {
|
||||||
// Both have GTIDs, so base the comparison entirely on the GTID sets.
|
// Both have GTIDs, so base the comparison entirely on the GTID sets.
|
||||||
GtidSet recordedGtidSet = new GtidSet(recordedGtidSetStr);
|
GtidSet recordedGtidSet = createGtidSet(recordedGtidSetStr);
|
||||||
GtidSet desiredGtidSet = new GtidSet(desiredGtidSetStr);
|
GtidSet desiredGtidSet = createGtidSet(desiredGtidSetStr);
|
||||||
if (gtidSourceFilter != null) {
|
if (gtidSourceFilter != null) {
|
||||||
// Apply the GTID source filter before we do any comparisons ...
|
// Apply the GTID source filter before we do any comparisons ...
|
||||||
recordedGtidSet = recordedGtidSet.retainAll(gtidSourceFilter);
|
recordedGtidSet = recordedGtidSet.retainAll(gtidSourceFilter);
|
@ -0,0 +1,29 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy;
|
||||||
|
|
||||||
|
import java.util.concurrent.ThreadFactory;
|
||||||
|
|
||||||
|
import com.github.shyiko.mysql.binlog.BinaryLogClient;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.EventType;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public interface BinaryLogClientConfigurator {
|
||||||
|
/**
|
||||||
|
* Configures the provided Binary Log Client instance.
|
||||||
|
*
|
||||||
|
* @param client the client instance ot be configured; should not be null
|
||||||
|
* @param threadFactory the thread factory to be used; should not be null
|
||||||
|
* @param connection the connector's JDBC connection; should not be null
|
||||||
|
*
|
||||||
|
* @return the configured binary log client instance
|
||||||
|
*/
|
||||||
|
BinaryLogClient configure(BinaryLogClient client, ThreadFactory threadFactory, AbstractConnectorConnection connection);
|
||||||
|
|
||||||
|
EventType getIncludeSqlQueryEventType();
|
||||||
|
}
|
@ -0,0 +1,42 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy;
|
||||||
|
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
|
import io.debezium.jdbc.JdbcConfiguration;
|
||||||
|
import io.debezium.jdbc.JdbcConnection;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public interface ConnectionConfiguration {
|
||||||
|
JdbcConfiguration config();
|
||||||
|
|
||||||
|
Configuration originalConfig();
|
||||||
|
|
||||||
|
JdbcConnection.ConnectionFactory factory();
|
||||||
|
|
||||||
|
String username();
|
||||||
|
|
||||||
|
String password();
|
||||||
|
|
||||||
|
String hostname();
|
||||||
|
|
||||||
|
int port();
|
||||||
|
|
||||||
|
MySqlConnectorConfig.SecureConnectionMode sslMode();
|
||||||
|
|
||||||
|
boolean sslModeEnabled();
|
||||||
|
|
||||||
|
String sslKeyStore();
|
||||||
|
|
||||||
|
char[] sslKeyStorePassword();
|
||||||
|
|
||||||
|
String sslTrustStore();
|
||||||
|
|
||||||
|
char[] sslTrustStorePassword();
|
||||||
|
}
|
@ -0,0 +1,63 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy;
|
||||||
|
|
||||||
|
import com.github.shyiko.mysql.binlog.event.EventData;
|
||||||
|
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
|
import io.debezium.connector.mysql.MySqlDatabaseSchema;
|
||||||
|
import io.debezium.connector.mysql.MySqlOffsetContext;
|
||||||
|
import io.debezium.connector.mysql.MySqlPartition;
|
||||||
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
|
import io.debezium.pipeline.notification.NotificationService;
|
||||||
|
import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotChangeEventSource;
|
||||||
|
import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotContext;
|
||||||
|
import io.debezium.pipeline.source.spi.DataChangeEventListener;
|
||||||
|
import io.debezium.pipeline.source.spi.SnapshotProgressListener;
|
||||||
|
import io.debezium.spi.schema.DataCollectionId;
|
||||||
|
import io.debezium.util.Clock;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides the MySQL connector with an adapter pattern to support varied configurations
|
||||||
|
* between MySQL and MariaDB and their drivers.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public interface ConnectorAdapter {
|
||||||
|
|
||||||
|
// todo: we had to introduce the configuration argument because of the extra database properties
|
||||||
|
// that are set in the task; we would get a test failure because of those missing if we
|
||||||
|
// simply created the connection configuration the base data in the MySqlConnectorConfig ctor
|
||||||
|
AbstractConnectorConnection createConnection(Configuration configuration);
|
||||||
|
|
||||||
|
BinaryLogClientConfigurator getBinaryLogClientConfigurator();
|
||||||
|
|
||||||
|
// todo: should we consider splitting value converters, it may prove useful in the future
|
||||||
|
// doing so would imply we won't likely need this method as it can be encapsulated?
|
||||||
|
String getJavaEncodingForCharSet(String charSetName);
|
||||||
|
|
||||||
|
// todo: for the moment we only expose the few handler deviations
|
||||||
|
// we may want to simply implement an abstract and concrete streaming impls
|
||||||
|
|
||||||
|
String getRecordingQueryFromEvent(EventData event);
|
||||||
|
|
||||||
|
AbstractHistoryRecordComparator getHistoryRecordComparator();
|
||||||
|
|
||||||
|
<T> IncrementalSnapshotContext<T> getIncrementalSnapshotContext();
|
||||||
|
|
||||||
|
Long getReadOnlyIncrementalSnapshotSignalOffset(MySqlOffsetContext previousOffsets);
|
||||||
|
|
||||||
|
IncrementalSnapshotChangeEventSource<MySqlPartition, ? extends DataCollectionId> createIncrementalSnapshotChangeEventSource(
|
||||||
|
MySqlConnectorConfig connectorConfig,
|
||||||
|
AbstractConnectorConnection connection,
|
||||||
|
EventDispatcher<MySqlPartition, ? extends DataCollectionId> dispatcher,
|
||||||
|
MySqlDatabaseSchema schema,
|
||||||
|
Clock clock,
|
||||||
|
SnapshotProgressListener<MySqlPartition> snapshotProgressListener,
|
||||||
|
DataChangeEventListener<MySqlPartition> dataChangeEventListener,
|
||||||
|
NotificationService<MySqlPartition, MySqlOffsetContext> notificationService);
|
||||||
|
}
|
@ -0,0 +1,59 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mariadb;
|
||||||
|
|
||||||
|
import java.util.concurrent.ThreadFactory;
|
||||||
|
|
||||||
|
import com.github.shyiko.mysql.binlog.BinaryLogClient;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.EventType;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
|
||||||
|
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractBinaryLogClientConfigurator;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An {@link AbstractBinaryLogClientConfigurator} implementation for MariaDB.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MariaDbBinaryLogClientConfigurator extends AbstractBinaryLogClientConfigurator {
|
||||||
|
|
||||||
|
public MariaDbBinaryLogClientConfigurator(MySqlConnectorConfig connectorConfig) {
|
||||||
|
super(connectorConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BinaryLogClient configure(BinaryLogClient client, ThreadFactory threadFactory, AbstractConnectorConnection connection) {
|
||||||
|
BinaryLogClient result = super.configure(client, threadFactory, connection);
|
||||||
|
if (getConnectorConfig().includeSqlQuery()) {
|
||||||
|
// Binlog client explicitly needs to be told to enable ANNOTATE_ROWS events, which is the
|
||||||
|
// MariaDB equivalent of ROWS_QUERY. This must be done ahead of the connection to make
|
||||||
|
// sure that the right negotiation bits are set during handshake.
|
||||||
|
result.setUseSendAnnotateRowsEvent(true);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventType getIncludeSqlQueryEventType() {
|
||||||
|
return EventType.ANNOTATE_ROWS;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void configureReplicaCompatibility(BinaryLogClient client) {
|
||||||
|
// This makes sure BEGIN events are emitted via QUERY events rather than GTIDs.
|
||||||
|
client.setMariaDbSlaveCapability(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected EventDeserializer createEventDeserializer() {
|
||||||
|
EventDeserializer eventDeserializer = super.createEventDeserializer();
|
||||||
|
eventDeserializer.setCompatibilityMode(EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY);
|
||||||
|
return eventDeserializer;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,87 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mariadb;
|
||||||
|
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.DebeziumException;
|
||||||
|
import io.debezium.connector.mysql.GtidSet;
|
||||||
|
import io.debezium.connector.mysql.MySqlFieldReader;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An {@link AbstractConnectorConnection} for MariaDB.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MariaDbConnection extends AbstractConnectorConnection {
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(MariaDbConnection.class);
|
||||||
|
|
||||||
|
public MariaDbConnection(MariaDbConnectionConfiguration connectionConfig, MySqlFieldReader fieldReader) {
|
||||||
|
super(connectionConfig, fieldReader);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isGtidModeEnabled() {
|
||||||
|
// MariaDB always has GTID enabled; however, GTID_STRICT_MODE can be enabled or disabled.
|
||||||
|
// For now we don't enforce this, so it can be a mixture
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet knownGtidSet() {
|
||||||
|
// MariaDB does not store the executed GTID details in the SHOW MASTER STATUS output like MySQL;
|
||||||
|
// however, instead makes this information available as a variable. The GTID_BINLOG_POS gives
|
||||||
|
// the current GTID position of the binary log and can therefore be considered the equivalent to
|
||||||
|
// MySQL's executed GTID set.
|
||||||
|
try {
|
||||||
|
return queryAndMap("SHOW GLOBAL VARIABLES LIKE 'GTID_BINLOG_POS'", rs -> {
|
||||||
|
if (rs.next()) {
|
||||||
|
LOGGER.info("knownGtidSet = {}", rs.getString(2));
|
||||||
|
return new MariaDbGtidSet(rs.getString(2));
|
||||||
|
}
|
||||||
|
return new MariaDbGtidSet("");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Unexpected error while looking at GTID_BINLOG_POS: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet subtractGtidSet(GtidSet set1, GtidSet set2) {
|
||||||
|
throw new DebeziumException("GtidSet subtraction not yet implemented by MariaDB");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet purgedGtidSet() {
|
||||||
|
// MariaDB does not store purged GTID details in a variable like MySQL; however, it stores the
|
||||||
|
// information in the `gtid_slave_pos` table in the `mysql` database, but this information has
|
||||||
|
// slightly different semantics. The purging is handled by MariaDB through the binary log's
|
||||||
|
// expiration settings and the `RESET MASTER` or `PURGE BINARY LOGS` statements.
|
||||||
|
//
|
||||||
|
// In order to calculate the purged state, we would need to get the `gtid_binlog_pos` variable
|
||||||
|
// that shows the current position of the GTID in the binary log, used by the primary, and
|
||||||
|
// compare this with the `gtid_slave_pos` variable on the replica server, which indicates the
|
||||||
|
// position of the GTIDs that have been applied.
|
||||||
|
throw new DebeziumException("Fetching purged GtidSet details is not yet supported");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet filterGtidSet(Predicate<String> gtidSourceFilter, String offsetGtids, GtidSet availableServerGtidSet, GtidSet purgedServerGtidSet) {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected GtidSet createGtidSet(String gtids) {
|
||||||
|
return new MariaDbGtidSet(gtids);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,37 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mariadb;
|
||||||
|
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectionConfiguration;
|
||||||
|
import io.debezium.util.Strings;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An {@link AbstractConnectionConfiguration} for MariaDB.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MariaDbConnectionConfiguration extends AbstractConnectionConfiguration {
|
||||||
|
|
||||||
|
private static final String JDBC_PROPERTY_MARIADB_TIME_ZONE = "timezone";
|
||||||
|
|
||||||
|
public MariaDbConnectionConfiguration(Configuration config) {
|
||||||
|
super(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String getConnectionTimeZonePropertyName() {
|
||||||
|
return JDBC_PROPERTY_MARIADB_TIME_ZONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String resolveConnectionTimeZone(Configuration dbConfig) {
|
||||||
|
// Debezium by default expected timezone data delivered in server timezone
|
||||||
|
String timezone = dbConfig.getString(JDBC_PROPERTY_MARIADB_TIME_ZONE);
|
||||||
|
return !Strings.isNullOrBlank(timezone) ? timezone : "auto";
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,101 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mariadb;
|
||||||
|
|
||||||
|
import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.EventData;
|
||||||
|
|
||||||
|
import io.debezium.DebeziumException;
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
|
import io.debezium.connector.mysql.MariaDbProtocolFieldReader;
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
|
import io.debezium.connector.mysql.MySqlDatabaseSchema;
|
||||||
|
import io.debezium.connector.mysql.MySqlOffsetContext;
|
||||||
|
import io.debezium.connector.mysql.MySqlPartition;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractHistoryRecordComparator;
|
||||||
|
import io.debezium.connector.mysql.strategy.BinaryLogClientConfigurator;
|
||||||
|
import io.debezium.connector.mysql.strategy.ConnectorAdapter;
|
||||||
|
import io.debezium.connector.mysql.strategy.mysql.MySqlConnection;
|
||||||
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
|
import io.debezium.pipeline.notification.NotificationService;
|
||||||
|
import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotChangeEventSource;
|
||||||
|
import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotContext;
|
||||||
|
import io.debezium.pipeline.source.snapshot.incremental.SignalBasedIncrementalSnapshotContext;
|
||||||
|
import io.debezium.pipeline.source.spi.DataChangeEventListener;
|
||||||
|
import io.debezium.pipeline.source.spi.SnapshotProgressListener;
|
||||||
|
import io.debezium.spi.schema.DataCollectionId;
|
||||||
|
import io.debezium.util.Clock;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This connector adapter provides a complete implementation for MariaDB assuming that
|
||||||
|
* the MariaDB driver is used for connections.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MariaDbConnectorAdapter implements ConnectorAdapter {
|
||||||
|
|
||||||
|
private final MySqlConnectorConfig connectorConfig;
|
||||||
|
private final MariaDbBinaryLogClientConfigurator binaryLogClientConfigurator;
|
||||||
|
|
||||||
|
public MariaDbConnectorAdapter(MySqlConnectorConfig connectorConfig) {
|
||||||
|
this.connectorConfig = connectorConfig;
|
||||||
|
this.binaryLogClientConfigurator = new MariaDbBinaryLogClientConfigurator(connectorConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public AbstractConnectorConnection createConnection(Configuration configuration) {
|
||||||
|
MariaDbConnectionConfiguration connectionConfig = new MariaDbConnectionConfiguration(configuration);
|
||||||
|
return new MariaDbConnection(connectionConfig, new MariaDbProtocolFieldReader(connectorConfig));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BinaryLogClientConfigurator getBinaryLogClientConfigurator() {
|
||||||
|
return binaryLogClientConfigurator;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getRecordingQueryFromEvent(EventData eventData) {
|
||||||
|
return ((AnnotateRowsEventData) eventData).getRowsQuery();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getJavaEncodingForCharSet(String charSetName) {
|
||||||
|
// todo: this should use a MariaDB specific implementation
|
||||||
|
return MySqlConnection.getJavaEncodingForCharSet(charSetName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public AbstractHistoryRecordComparator getHistoryRecordComparator() {
|
||||||
|
return new MariaDbHistoryRecordComparator(connectorConfig.gtidSourceFilter());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <T> IncrementalSnapshotContext<T> getIncrementalSnapshotContext() {
|
||||||
|
if (connectorConfig.isReadOnlyConnection()) {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
return new SignalBasedIncrementalSnapshotContext<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Long getReadOnlyIncrementalSnapshotSignalOffset(MySqlOffsetContext previousOffsets) {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public IncrementalSnapshotChangeEventSource<MySqlPartition, ? extends DataCollectionId> createIncrementalSnapshotChangeEventSource(
|
||||||
|
MySqlConnectorConfig connectorConfig,
|
||||||
|
AbstractConnectorConnection connection,
|
||||||
|
EventDispatcher<MySqlPartition, ? extends DataCollectionId> dispatcher,
|
||||||
|
MySqlDatabaseSchema schema,
|
||||||
|
Clock clock,
|
||||||
|
SnapshotProgressListener<MySqlPartition> snapshotProgressListener,
|
||||||
|
DataChangeEventListener<MySqlPartition> dataChangeEventListener,
|
||||||
|
NotificationService<MySqlPartition, MySqlOffsetContext> notificationService) {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,55 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mariadb;
|
||||||
|
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
|
||||||
|
import io.debezium.DebeziumException;
|
||||||
|
import io.debezium.connector.mysql.GtidSet;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MariaDbGtidSet implements GtidSet {
|
||||||
|
|
||||||
|
public MariaDbGtidSet(String gtid) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isEmpty() {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet retainAll(Predicate<String> sourceFilter) {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isContainedWithin(GtidSet other) {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet with(GtidSet other) {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet getGtidSetBeginning() {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean contains(String gtid) {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet subtract(GtidSet other) {
|
||||||
|
throw new DebeziumException("NYI");
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,27 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mariadb;
|
||||||
|
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
|
||||||
|
import io.debezium.connector.mysql.GtidSet;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractHistoryRecordComparator;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MariaDbHistoryRecordComparator extends AbstractHistoryRecordComparator {
|
||||||
|
|
||||||
|
public MariaDbHistoryRecordComparator(Predicate<String> gtidSourceFilter) {
|
||||||
|
super(gtidSourceFilter);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected GtidSet createGtidSet(String gtidSet) {
|
||||||
|
return new MariaDbGtidSet(gtidSet);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,43 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mariadb.hybrid;
|
||||||
|
|
||||||
|
import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.EventData;
|
||||||
|
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
|
import io.debezium.connector.mysql.strategy.BinaryLogClientConfigurator;
|
||||||
|
import io.debezium.connector.mysql.strategy.mariadb.MariaDbBinaryLogClientConfigurator;
|
||||||
|
import io.debezium.connector.mysql.strategy.mysql.MySqlConnectorAdapter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This connector adapter provides a hybrid configuration where the user connects to a
|
||||||
|
* MariaDB target system; however, uses the MySQL driver.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MariaDbHybridConnectorAdapter extends MySqlConnectorAdapter {
|
||||||
|
|
||||||
|
// todo: Do we want to consider supporting this mode at all?
|
||||||
|
|
||||||
|
private final MariaDbBinaryLogClientConfigurator binaryLogClientConfigurator;
|
||||||
|
|
||||||
|
public MariaDbHybridConnectorAdapter(MySqlConnectorConfig connectorConfig) {
|
||||||
|
super(connectorConfig);
|
||||||
|
this.binaryLogClientConfigurator = new MariaDbBinaryLogClientConfigurator(connectorConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BinaryLogClientConfigurator getBinaryLogClientConfigurator() {
|
||||||
|
return binaryLogClientConfigurator;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getRecordingQueryFromEvent(EventData eventData) {
|
||||||
|
return ((AnnotateRowsEventData) eventData).getRowsQuery();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,20 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mysql;
|
||||||
|
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractBinaryLogClientConfigurator;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MySqlBinaryLogClientConfigurator extends AbstractBinaryLogClientConfigurator {
|
||||||
|
|
||||||
|
public MySqlBinaryLogClientConfigurator(MySqlConnectorConfig connectorConfig) {
|
||||||
|
super(connectorConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,146 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mysql;
|
||||||
|
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.mysql.cj.CharsetMapping;
|
||||||
|
|
||||||
|
import io.debezium.DebeziumException;
|
||||||
|
import io.debezium.connector.mysql.GtidSet;
|
||||||
|
import io.debezium.connector.mysql.MySqlFieldReader;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An {@link AbstractConnectorConnection} to be used with MySQL.
|
||||||
|
*
|
||||||
|
* @author Jiri Pechanec, Randell Hauch, Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MySqlConnection extends AbstractConnectorConnection {
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(MySqlConnection.class);
|
||||||
|
|
||||||
|
public MySqlConnection(MySqlConnectionConfiguration connectionConfig, MySqlFieldReader fieldReader) {
|
||||||
|
super(connectionConfig, fieldReader);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isGtidModeEnabled() {
|
||||||
|
try {
|
||||||
|
return queryAndMap("SHOW GLOBAL VARIABLES LIKE 'GTID_MODE'", rs -> {
|
||||||
|
if (rs.next()) {
|
||||||
|
return "ON".equalsIgnoreCase(rs.getString(2));
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Unexpected error while looking at GTID mode: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet knownGtidSet() {
|
||||||
|
try {
|
||||||
|
return queryAndMap("SHOW MASTER STATUS", rs -> {
|
||||||
|
if (rs.next() && rs.getMetaData().getColumnCount() > 4) {
|
||||||
|
return new MySqlGtidSet(rs.getString(5)); // GTID set, may be null, blank, or contain a GTID set
|
||||||
|
}
|
||||||
|
return new MySqlGtidSet("");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Unexpected error while looking at GTID mode: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet subtractGtidSet(GtidSet set1, GtidSet set2) {
|
||||||
|
try {
|
||||||
|
return prepareQueryAndMap("SELECT GTID_SUBTRACT(?, ?)",
|
||||||
|
ps -> {
|
||||||
|
ps.setString(1, set1.toString());
|
||||||
|
ps.setString(2, set2.toString());
|
||||||
|
},
|
||||||
|
rs -> {
|
||||||
|
if (rs.next()) {
|
||||||
|
return new MySqlGtidSet(rs.getString(1));
|
||||||
|
}
|
||||||
|
return new MySqlGtidSet("");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Unexpected error while executing GTID_SUBTRACT: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet purgedGtidSet() {
|
||||||
|
try {
|
||||||
|
return queryAndMap("SELECT @@global.gtid_purged", rs -> {
|
||||||
|
if (rs.next() && rs.getMetaData().getColumnCount() > 0) {
|
||||||
|
return new MySqlGtidSet(rs.getString(1)); // GTID set, may be null, blank, or contain a GTID set
|
||||||
|
}
|
||||||
|
return new MySqlGtidSet("");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new DebeziumException("Unexpected error while looking at gtid_purged variable: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet filterGtidSet(Predicate<String> gtidSourceFilter, String offsetGtids, GtidSet availableServerGtidSet, GtidSet purgedServerGtidSet) {
|
||||||
|
String gtidStr = offsetGtids;
|
||||||
|
if (gtidStr == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
LOGGER.info("Attempting to generate a filtered GTID set");
|
||||||
|
LOGGER.info("GTID set from previous recorded offset: {}", gtidStr);
|
||||||
|
GtidSet filteredGtidSet = new MySqlGtidSet(gtidStr);
|
||||||
|
if (gtidSourceFilter != null) {
|
||||||
|
filteredGtidSet = filteredGtidSet.retainAll(gtidSourceFilter);
|
||||||
|
LOGGER.info("GTID set after applying GTID source includes/excludes to previous recorded offset: {}", filteredGtidSet);
|
||||||
|
}
|
||||||
|
LOGGER.info("GTID set available on server: {}", availableServerGtidSet);
|
||||||
|
|
||||||
|
final GtidSet knownGtidSet = filteredGtidSet;
|
||||||
|
LOGGER.info("Using first available positions for new GTID channels");
|
||||||
|
final GtidSet relevantAvailableServerGtidSet = (gtidSourceFilter != null) ? availableServerGtidSet.retainAll(gtidSourceFilter) : availableServerGtidSet;
|
||||||
|
LOGGER.info("Relevant GTID set available on server: {}", relevantAvailableServerGtidSet);
|
||||||
|
|
||||||
|
GtidSet mergedGtidSet = relevantAvailableServerGtidSet
|
||||||
|
.retainAll(uuid -> ((MySqlGtidSet) knownGtidSet).forServerWithId(uuid) != null)
|
||||||
|
.with(purgedServerGtidSet)
|
||||||
|
.with(filteredGtidSet);
|
||||||
|
|
||||||
|
LOGGER.info("Final merged GTID set to use when connecting to MySQL: {}", mergedGtidSet);
|
||||||
|
return mergedGtidSet;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected GtidSet createGtidSet(String gtids) {
|
||||||
|
return new MySqlGtidSet(gtids);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getJavaEncodingForCharSet(String charSetName) {
|
||||||
|
return CharsetMappingWrapper.getJavaEncodingForMysqlCharSet(charSetName);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper to gain access to protected method
|
||||||
|
*/
|
||||||
|
private final static class CharsetMappingWrapper extends CharsetMapping {
|
||||||
|
static String getJavaEncodingForMysqlCharSet(String charSetName) {
|
||||||
|
return CharsetMapping.getStaticJavaEncodingForMysqlCharset(charSetName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,37 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mysql;
|
||||||
|
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectionConfiguration;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An {@link AbstractConnectionConfiguration} implementation for MySQL.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MySqlConnectionConfiguration extends AbstractConnectionConfiguration {
|
||||||
|
|
||||||
|
private static final String JDBC_PROPERTY_CONNECTION_TIME_ZONE = "connectionTimeZone";
|
||||||
|
|
||||||
|
public MySqlConnectionConfiguration(Configuration config) {
|
||||||
|
super(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String getConnectionTimeZonePropertyName() {
|
||||||
|
return JDBC_PROPERTY_CONNECTION_TIME_ZONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String resolveConnectionTimeZone(Configuration dbConfig) {
|
||||||
|
// Debezium by default expects time zoned data delivered in server timezone
|
||||||
|
String connectionTimeZone = dbConfig.getString(JDBC_PROPERTY_CONNECTION_TIME_ZONE);
|
||||||
|
return connectionTimeZone != null ? connectionTimeZone : "SERVER";
|
||||||
|
// return !Strings.isNullOrBlank(connectionTimeZone) ? connectionTimeZone : "SERVER";
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,118 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mysql;
|
||||||
|
|
||||||
|
import com.github.shyiko.mysql.binlog.event.EventData;
|
||||||
|
import com.github.shyiko.mysql.binlog.event.RowsQueryEventData;
|
||||||
|
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
|
import io.debezium.connector.mysql.MySqlBinaryProtocolFieldReader;
|
||||||
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
|
import io.debezium.connector.mysql.MySqlDatabaseSchema;
|
||||||
|
import io.debezium.connector.mysql.MySqlFieldReader;
|
||||||
|
import io.debezium.connector.mysql.MySqlOffsetContext;
|
||||||
|
import io.debezium.connector.mysql.MySqlPartition;
|
||||||
|
import io.debezium.connector.mysql.MySqlTextProtocolFieldReader;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractConnectorConnection;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractHistoryRecordComparator;
|
||||||
|
import io.debezium.connector.mysql.strategy.BinaryLogClientConfigurator;
|
||||||
|
import io.debezium.connector.mysql.strategy.ConnectorAdapter;
|
||||||
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
|
import io.debezium.pipeline.notification.NotificationService;
|
||||||
|
import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotChangeEventSource;
|
||||||
|
import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotContext;
|
||||||
|
import io.debezium.pipeline.source.snapshot.incremental.SignalBasedIncrementalSnapshotContext;
|
||||||
|
import io.debezium.pipeline.source.spi.DataChangeEventListener;
|
||||||
|
import io.debezium.pipeline.source.spi.SnapshotProgressListener;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
import io.debezium.spi.schema.DataCollectionId;
|
||||||
|
import io.debezium.util.Clock;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This connector adapter provides a complete implementation for MySQL assuming that the
|
||||||
|
* MySQL driver is used for connections.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MySqlConnectorAdapter implements ConnectorAdapter {
|
||||||
|
|
||||||
|
private final MySqlConnectorConfig connectorConfig;
|
||||||
|
private final MySqlBinaryLogClientConfigurator binaryLogClientConfigurator;
|
||||||
|
|
||||||
|
public MySqlConnectorAdapter(MySqlConnectorConfig connectorConfig) {
|
||||||
|
this.connectorConfig = connectorConfig;
|
||||||
|
this.binaryLogClientConfigurator = new MySqlBinaryLogClientConfigurator(connectorConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public AbstractConnectorConnection createConnection(Configuration configuration) {
|
||||||
|
final MySqlConnectionConfiguration connectionConfig = new MySqlConnectionConfiguration(configuration);
|
||||||
|
return new MySqlConnection(connectionConfig, resolveFieldReader());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BinaryLogClientConfigurator getBinaryLogClientConfigurator() {
|
||||||
|
return binaryLogClientConfigurator;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getJavaEncodingForCharSet(String charSetName) {
|
||||||
|
return MySqlConnection.getJavaEncodingForCharSet(charSetName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getRecordingQueryFromEvent(EventData eventData) {
|
||||||
|
return ((RowsQueryEventData) eventData).getQuery();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public AbstractHistoryRecordComparator getHistoryRecordComparator() {
|
||||||
|
return new MySqlHistoryRecordComparator(connectorConfig.gtidSourceFilter());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <T> IncrementalSnapshotContext<T> getIncrementalSnapshotContext() {
|
||||||
|
if (connectorConfig.isReadOnlyConnection()) {
|
||||||
|
return new MySqlReadOnlyIncrementalSnapshotContext<>();
|
||||||
|
}
|
||||||
|
return new SignalBasedIncrementalSnapshotContext<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public Long getReadOnlyIncrementalSnapshotSignalOffset(MySqlOffsetContext previousOffsets) {
|
||||||
|
return ((MySqlReadOnlyIncrementalSnapshotContext<TableId>) previousOffsets.getIncrementalSnapshotContext()).getSignalOffset();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public IncrementalSnapshotChangeEventSource<MySqlPartition, ? extends DataCollectionId> createIncrementalSnapshotChangeEventSource(
|
||||||
|
MySqlConnectorConfig connectorConfig,
|
||||||
|
AbstractConnectorConnection connection,
|
||||||
|
EventDispatcher<MySqlPartition, ? extends DataCollectionId> dispatcher,
|
||||||
|
MySqlDatabaseSchema schema,
|
||||||
|
Clock clock,
|
||||||
|
SnapshotProgressListener<MySqlPartition> snapshotProgressListener,
|
||||||
|
DataChangeEventListener<MySqlPartition> dataChangeEventListener,
|
||||||
|
NotificationService<MySqlPartition, MySqlOffsetContext> notificationService) {
|
||||||
|
return new MySqlReadOnlyIncrementalSnapshotChangeEventSource<>(
|
||||||
|
connectorConfig,
|
||||||
|
connection,
|
||||||
|
dispatcher,
|
||||||
|
schema,
|
||||||
|
clock,
|
||||||
|
snapshotProgressListener,
|
||||||
|
dataChangeEventListener,
|
||||||
|
notificationService);
|
||||||
|
}
|
||||||
|
|
||||||
|
private MySqlFieldReader resolveFieldReader() {
|
||||||
|
// todo: this null check is needed for the connection validation (try to rework)
|
||||||
|
return connectorConfig != null && connectorConfig.useCursorFetch()
|
||||||
|
? new MySqlBinaryProtocolFieldReader(connectorConfig)
|
||||||
|
: new MySqlTextProtocolFieldReader(connectorConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,472 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mysql;
|
||||||
|
|
||||||
|
import java.util.AbstractMap;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.LinkedList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.TreeMap;
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import io.debezium.annotation.Immutable;
|
||||||
|
import io.debezium.connector.mysql.GtidSet;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Represents a set of MySQL GTIDs.
|
||||||
|
*
|
||||||
|
* This is an improvement ove {@link com.github.shyiko.mysql.binlog.GtidSet} that is immutable, and
|
||||||
|
* more properly supports comparisons.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford, Randall Hauch
|
||||||
|
*/
|
||||||
|
@Immutable
|
||||||
|
public class MySqlGtidSet implements GtidSet {
|
||||||
|
|
||||||
|
private final Map<String, UUIDSet> uuidSetsByServerId = new TreeMap<>(); // sorts on keys
|
||||||
|
public static Pattern GTID_DELIMITER = Pattern.compile(":");
|
||||||
|
|
||||||
|
public MySqlGtidSet(String gtids) {
|
||||||
|
if (gtids != null) {
|
||||||
|
gtids = gtids.replace("\n", "").replace("\r", "");
|
||||||
|
new com.github.shyiko.mysql.binlog.GtidSet(gtids).getUUIDSets().forEach(uuidSet -> {
|
||||||
|
uuidSetsByServerId.put(uuidSet.getUUID(), new UUIDSet(uuidSet));
|
||||||
|
});
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
uuidSetsByServerId.values().forEach(uuidSet -> {
|
||||||
|
if (sb.length() != 0) {
|
||||||
|
sb.append(',');
|
||||||
|
}
|
||||||
|
sb.append(uuidSet.toString());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected MySqlGtidSet(Map<String, UUIDSet> uuidSetsByServerId) {
|
||||||
|
this.uuidSetsByServerId.putAll(uuidSetsByServerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isEmpty() {
|
||||||
|
return uuidSetsByServerId.isEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
public MySqlGtidSet retainAll(Predicate<String> sourceFilter) {
|
||||||
|
if (sourceFilter == null) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
Map<String, UUIDSet> newSets = this.uuidSetsByServerId.entrySet()
|
||||||
|
.stream()
|
||||||
|
.filter(entry -> sourceFilter.test(entry.getKey()))
|
||||||
|
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||||
|
return new MySqlGtidSet(newSets);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isContainedWithin(GtidSet other) {
|
||||||
|
if (other == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (this.equals(other)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
final MySqlGtidSet theOther = (MySqlGtidSet) other;
|
||||||
|
for (UUIDSet uuidSet : uuidSetsByServerId.values()) {
|
||||||
|
UUIDSet thatSet = theOther.forServerWithId(uuidSet.getUUID());
|
||||||
|
if (!uuidSet.isContainedWithin(thatSet)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GtidSet with(GtidSet other) {
|
||||||
|
final MySqlGtidSet theOther = (MySqlGtidSet) other;
|
||||||
|
if (theOther == null || theOther.uuidSetsByServerId.isEmpty()) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
Map<String, UUIDSet> newSet = new HashMap<>();
|
||||||
|
newSet.putAll(this.uuidSetsByServerId);
|
||||||
|
newSet.putAll(theOther.uuidSetsByServerId);
|
||||||
|
return new MySqlGtidSet(newSet);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MySqlGtidSet getGtidSetBeginning() {
|
||||||
|
Map<String, UUIDSet> newSet = new HashMap<>();
|
||||||
|
|
||||||
|
for (UUIDSet uuidSet : uuidSetsByServerId.values()) {
|
||||||
|
newSet.put(uuidSet.getUUID(), uuidSet.asIntervalBeginning());
|
||||||
|
}
|
||||||
|
|
||||||
|
return new MySqlGtidSet(newSet);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean contains(String gtid) {
|
||||||
|
String[] split = GTID_DELIMITER.split(gtid);
|
||||||
|
String sourceId = split[0];
|
||||||
|
UUIDSet uuidSet = forServerWithId(sourceId);
|
||||||
|
if (uuidSet == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
long transactionId = Long.parseLong(split[1]);
|
||||||
|
return uuidSet.contains(transactionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MySqlGtidSet subtract(GtidSet other) {
|
||||||
|
if (other == null) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
final MySqlGtidSet theOther = (MySqlGtidSet) other;
|
||||||
|
Map<String, UUIDSet> newSets = this.uuidSetsByServerId.entrySet()
|
||||||
|
.stream()
|
||||||
|
.filter(entry -> !entry.getValue().isContainedWithin(theOther.forServerWithId(entry.getKey())))
|
||||||
|
.map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey(), entry.getValue().subtract(theOther.forServerWithId(entry.getKey()))))
|
||||||
|
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||||
|
return new MySqlGtidSet(newSets);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get an immutable collection of the {@link UUIDSet range of GTIDs for a single server}.
|
||||||
|
*
|
||||||
|
* @return the {@link UUIDSet GTID ranges for each server}; never null
|
||||||
|
*/
|
||||||
|
public Collection<UUIDSet> getUUIDSets() {
|
||||||
|
return Collections.unmodifiableCollection(uuidSetsByServerId.values());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find the {@link UUIDSet} for the server with the specified Uuid.
|
||||||
|
*
|
||||||
|
* @param uuid the Uuid of the server
|
||||||
|
* @return the {@link UUIDSet} for the identified server, or {@code null} if there are no GTIDs from that server.
|
||||||
|
*/
|
||||||
|
public UUIDSet forServerWithId(String uuid) {
|
||||||
|
return uuidSetsByServerId.get(uuid);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return uuidSetsByServerId.keySet().hashCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object obj) {
|
||||||
|
if (obj == this) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (obj instanceof GtidSet) {
|
||||||
|
MySqlGtidSet that = (MySqlGtidSet) obj;
|
||||||
|
return this.uuidSetsByServerId.equals(that.uuidSetsByServerId);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
List<String> gtids = new ArrayList<String>();
|
||||||
|
for (UUIDSet uuidSet : uuidSetsByServerId.values()) {
|
||||||
|
gtids.add(uuidSet.toString());
|
||||||
|
}
|
||||||
|
return String.join(",", gtids);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A range of GTIDs for a single server with a specific Uuid.
|
||||||
|
*/
|
||||||
|
@Immutable
|
||||||
|
public static class UUIDSet {
|
||||||
|
|
||||||
|
private final String uuid;
|
||||||
|
private final LinkedList<Interval> intervals = new LinkedList<>();
|
||||||
|
|
||||||
|
protected UUIDSet(com.github.shyiko.mysql.binlog.GtidSet.UUIDSet uuidSet) {
|
||||||
|
this.uuid = uuidSet.getUUID();
|
||||||
|
uuidSet.getIntervals().forEach(interval -> {
|
||||||
|
intervals.add(new Interval(interval.getStart(), interval.getEnd()));
|
||||||
|
});
|
||||||
|
Collections.sort(this.intervals);
|
||||||
|
if (this.intervals.size() > 1) {
|
||||||
|
// Collapse adjacent intervals ...
|
||||||
|
for (int i = intervals.size() - 1; i != 0; --i) {
|
||||||
|
Interval before = this.intervals.get(i - 1);
|
||||||
|
Interval after = this.intervals.get(i);
|
||||||
|
if ((before.getEnd() + 1) == after.getStart()) {
|
||||||
|
this.intervals.set(i - 1, new Interval(before.getStart(), after.getEnd()));
|
||||||
|
this.intervals.remove(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected UUIDSet(String uuid, Interval interval) {
|
||||||
|
this.uuid = uuid;
|
||||||
|
this.intervals.add(interval);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected UUIDSet(String uuid, List<Interval> intervals) {
|
||||||
|
this.uuid = uuid;
|
||||||
|
this.intervals.addAll(intervals);
|
||||||
|
}
|
||||||
|
|
||||||
|
public UUIDSet asIntervalBeginning() {
|
||||||
|
Interval start = new Interval(intervals.get(0).getStart(), intervals.get(0).getStart());
|
||||||
|
return new UUIDSet(this.uuid, start);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the Uuid for the server that generated the GTIDs.
|
||||||
|
*
|
||||||
|
* @return the server's Uuid; never null
|
||||||
|
*/
|
||||||
|
public String getUUID() {
|
||||||
|
return uuid;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the intervals of transaction numbers.
|
||||||
|
*
|
||||||
|
* @return the immutable transaction intervals; never null
|
||||||
|
*/
|
||||||
|
public List<Interval> getIntervals() {
|
||||||
|
return Collections.unmodifiableList(intervals);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine if the set of transaction numbers from this server is completely within the set of transaction numbers from
|
||||||
|
* the set of transaction numbers in the supplied set.
|
||||||
|
*
|
||||||
|
* @param other the set to compare with this set
|
||||||
|
* @return {@code true} if this server's transaction numbers are a subset of the transaction numbers of the supplied set,
|
||||||
|
* or false otherwise
|
||||||
|
*/
|
||||||
|
public boolean isContainedWithin(UUIDSet other) {
|
||||||
|
if (other == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!this.getUUID().equalsIgnoreCase(other.getUUID())) {
|
||||||
|
// Not even the same server ...
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (this.intervals.isEmpty()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (other.intervals.isEmpty()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
assert this.intervals.size() > 0;
|
||||||
|
assert other.intervals.size() > 0;
|
||||||
|
|
||||||
|
// Every interval in this must be within an interval of the other ...
|
||||||
|
for (Interval thisInterval : this.intervals) {
|
||||||
|
boolean found = false;
|
||||||
|
for (Interval otherInterval : other.intervals) {
|
||||||
|
if (thisInterval.isContainedWithin(otherInterval)) {
|
||||||
|
found = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!found) {
|
||||||
|
return false; // didn't find a match
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean contains(long transactionId) {
|
||||||
|
for (Interval interval : this.intervals) {
|
||||||
|
if (interval.contains(transactionId)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return uuid.hashCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object obj) {
|
||||||
|
if (obj == this) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (obj instanceof UUIDSet) {
|
||||||
|
UUIDSet that = (UUIDSet) obj;
|
||||||
|
return this.getUUID().equalsIgnoreCase(that.getUUID()) && this.getIntervals().equals(that.getIntervals());
|
||||||
|
}
|
||||||
|
return super.equals(obj);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append(uuid).append(':');
|
||||||
|
Iterator<Interval> iter = intervals.iterator();
|
||||||
|
if (iter.hasNext()) {
|
||||||
|
sb.append(iter.next());
|
||||||
|
}
|
||||||
|
while (iter.hasNext()) {
|
||||||
|
sb.append(':');
|
||||||
|
sb.append(iter.next());
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public UUIDSet subtract(UUIDSet other) {
|
||||||
|
if (!uuid.equals(other.getUUID())) {
|
||||||
|
throw new IllegalArgumentException("UUIDSet subtraction is supported only within a single server UUID");
|
||||||
|
}
|
||||||
|
List<Interval> result = new ArrayList<>();
|
||||||
|
for (Interval interval : intervals) {
|
||||||
|
result.addAll(interval.removeAll(other.getIntervals()));
|
||||||
|
}
|
||||||
|
return new UUIDSet(uuid, result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Immutable
|
||||||
|
public static class Interval implements Comparable<Interval> {
|
||||||
|
|
||||||
|
private final long start;
|
||||||
|
private final long end;
|
||||||
|
|
||||||
|
public Interval(long start, long end) {
|
||||||
|
this.start = start;
|
||||||
|
this.end = end;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the starting transaction number in this interval.
|
||||||
|
*
|
||||||
|
* @return this interval's first transaction number
|
||||||
|
*/
|
||||||
|
public long getStart() {
|
||||||
|
return start;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the ending transaction number in this interval.
|
||||||
|
*
|
||||||
|
* @return this interval's last transaction number
|
||||||
|
*/
|
||||||
|
public long getEnd() {
|
||||||
|
return end;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine if this interval is completely within the supplied interval.
|
||||||
|
*
|
||||||
|
* @param other the interval to compare with
|
||||||
|
* @return {@code true} if the {@link #getStart() start} is greater than or equal to the supplied interval's
|
||||||
|
* {@link #getStart() start} and the {@link #getEnd() end} is less than or equal to the supplied interval's
|
||||||
|
* {@link #getEnd() end}, or {@code false} otherwise
|
||||||
|
*/
|
||||||
|
public boolean isContainedWithin(Interval other) {
|
||||||
|
if (other == this) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (other == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return this.getStart() >= other.getStart() && this.getEnd() <= other.getEnd();
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean contains(long transactionId) {
|
||||||
|
return getStart() <= transactionId && transactionId <= getEnd();
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean contains(Interval other) {
|
||||||
|
return getStart() <= other.getStart() && getEnd() >= other.getEnd();
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean nonintersecting(Interval other) {
|
||||||
|
return other.getEnd() < this.getStart() || other.getStart() > this.getEnd();
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Interval> remove(Interval other) {
|
||||||
|
if (nonintersecting(other)) {
|
||||||
|
return Collections.singletonList(this);
|
||||||
|
}
|
||||||
|
if (other.contains(this)) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
List<Interval> result = new LinkedList<>();
|
||||||
|
if (this.getStart() < other.getStart()) {
|
||||||
|
Interval part = new Interval(this.getStart(), other.getStart() - 1);
|
||||||
|
result.add(part);
|
||||||
|
}
|
||||||
|
if (other.getEnd() < this.getEnd()) {
|
||||||
|
Interval part = new Interval(other.getEnd() + 1, this.getEnd());
|
||||||
|
result.add(part);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Interval> removeAll(List<Interval> otherIntervals) {
|
||||||
|
List<Interval> thisIntervals = new LinkedList<>();
|
||||||
|
thisIntervals.add(this);
|
||||||
|
List<Interval> result = new LinkedList<>();
|
||||||
|
result.add(this);
|
||||||
|
for (Interval other : otherIntervals) {
|
||||||
|
result = new LinkedList<>();
|
||||||
|
for (Interval thisInterval : thisIntervals) {
|
||||||
|
result.addAll(thisInterval.remove(other));
|
||||||
|
}
|
||||||
|
thisIntervals = result;
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int compareTo(Interval that) {
|
||||||
|
if (that == this) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
long diff = this.start - that.start;
|
||||||
|
if (diff > Integer.MAX_VALUE) {
|
||||||
|
return Integer.MAX_VALUE;
|
||||||
|
}
|
||||||
|
if (diff < Integer.MIN_VALUE) {
|
||||||
|
return Integer.MIN_VALUE;
|
||||||
|
}
|
||||||
|
return (int) diff;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return (int) getStart();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object obj) {
|
||||||
|
if (this == obj) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (obj instanceof Interval) {
|
||||||
|
Interval that = (Interval) obj;
|
||||||
|
return this.getStart() == that.getStart() && this.getEnd() == that.getEnd();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "" + getStart() + "-" + getEnd();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,27 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.mysql.strategy.mysql;
|
||||||
|
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
|
||||||
|
import io.debezium.connector.mysql.GtidSet;
|
||||||
|
import io.debezium.connector.mysql.strategy.AbstractHistoryRecordComparator;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class MySqlHistoryRecordComparator extends AbstractHistoryRecordComparator {
|
||||||
|
|
||||||
|
public MySqlHistoryRecordComparator(Predicate<String> gtidSourceFilter) {
|
||||||
|
super(gtidSourceFilter);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected GtidSet createGtidSet(String gtidSet) {
|
||||||
|
return new MySqlGtidSet(gtidSet);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -3,7 +3,7 @@
|
|||||||
*
|
*
|
||||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
*/
|
*/
|
||||||
package io.debezium.connector.mysql;
|
package io.debezium.connector.mysql.strategy.mysql;
|
||||||
|
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -14,6 +14,9 @@
|
|||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import io.debezium.DebeziumException;
|
import io.debezium.DebeziumException;
|
||||||
|
import io.debezium.connector.mysql.MySqlOffsetContext;
|
||||||
|
import io.debezium.connector.mysql.MySqlPartition;
|
||||||
|
import io.debezium.connector.mysql.SourceInfo;
|
||||||
import io.debezium.jdbc.JdbcConnection;
|
import io.debezium.jdbc.JdbcConnection;
|
||||||
import io.debezium.pipeline.EventDispatcher;
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
import io.debezium.pipeline.notification.NotificationService;
|
import io.debezium.pipeline.notification.NotificationService;
|
||||||
@ -174,14 +177,14 @@ protected void updateHighWatermark() {
|
|||||||
getExecutedGtidSet(getContext()::setHighWatermark);
|
getExecutedGtidSet(getContext()::setHighWatermark);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void getExecutedGtidSet(Consumer<GtidSet> watermark) {
|
private void getExecutedGtidSet(Consumer<MySqlGtidSet> watermark) {
|
||||||
try {
|
try {
|
||||||
jdbcConnection.query(SHOW_MASTER_STMT, rs -> {
|
jdbcConnection.query(SHOW_MASTER_STMT, rs -> {
|
||||||
if (rs.next()) {
|
if (rs.next()) {
|
||||||
if (rs.getMetaData().getColumnCount() > 4) {
|
if (rs.getMetaData().getColumnCount() > 4) {
|
||||||
// This column exists only in MySQL 5.6.5 or later ...
|
// This column exists only in MySQL 5.6.5 or later ...
|
||||||
final String gtidSet = rs.getString(5); // GTID set, may be null, blank, or contain a GTID set
|
final String gtidSet = rs.getString(5); // GTID set, may be null, blank, or contain a GTID set
|
||||||
watermark.accept(new GtidSet(gtidSet));
|
watermark.accept(new MySqlGtidSet(gtidSet));
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
throw new UnsupportedOperationException("Need to add support for executed GTIDs for versions prior to 5.6.5");
|
throw new UnsupportedOperationException("Need to add support for executed GTIDs for versions prior to 5.6.5");
|
@ -3,28 +3,27 @@
|
|||||||
*
|
*
|
||||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
*/
|
*/
|
||||||
package io.debezium.connector.mysql;
|
package io.debezium.connector.mysql.strategy.mysql;
|
||||||
|
|
||||||
import static io.debezium.connector.mysql.GtidSet.GTID_DELIMITER;
|
import static io.debezium.connector.mysql.strategy.mysql.MySqlGtidSet.GTID_DELIMITER;
|
||||||
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import io.debezium.annotation.NotThreadSafe;
|
import io.debezium.connector.mysql.SourceInfo;
|
||||||
import io.debezium.pipeline.source.snapshot.incremental.AbstractIncrementalSnapshotContext;
|
import io.debezium.pipeline.source.snapshot.incremental.AbstractIncrementalSnapshotContext;
|
||||||
import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotContext;
|
import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotContext;
|
||||||
import io.debezium.pipeline.spi.OffsetContext;
|
import io.debezium.pipeline.spi.OffsetContext;
|
||||||
|
|
||||||
@NotThreadSafe
|
|
||||||
public class MySqlReadOnlyIncrementalSnapshotContext<T> extends AbstractIncrementalSnapshotContext<T> {
|
public class MySqlReadOnlyIncrementalSnapshotContext<T> extends AbstractIncrementalSnapshotContext<T> {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(MySqlReadOnlyIncrementalSnapshotContext.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(MySqlReadOnlyIncrementalSnapshotContext.class);
|
||||||
private GtidSet previousLowWatermark;
|
private MySqlGtidSet previousLowWatermark;
|
||||||
private GtidSet previousHighWatermark;
|
private MySqlGtidSet previousHighWatermark;
|
||||||
private GtidSet lowWatermark;
|
private MySqlGtidSet lowWatermark;
|
||||||
private GtidSet highWatermark;
|
private MySqlGtidSet highWatermark;
|
||||||
private Long signalOffset;
|
private Long signalOffset;
|
||||||
public static final String SIGNAL_OFFSET = INCREMENTAL_SNAPSHOT_KEY + "_signal_offset";
|
public static final String SIGNAL_OFFSET = INCREMENTAL_SNAPSHOT_KEY + "_signal_offset";
|
||||||
|
|
||||||
@ -53,11 +52,11 @@ public static <U> MySqlReadOnlyIncrementalSnapshotContext<U> load(Map<String, ?>
|
|||||||
return context;
|
return context;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setLowWatermark(GtidSet lowWatermark) {
|
public void setLowWatermark(MySqlGtidSet lowWatermark) {
|
||||||
this.lowWatermark = lowWatermark;
|
this.lowWatermark = lowWatermark;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setHighWatermark(GtidSet highWatermark) {
|
public void setHighWatermark(MySqlGtidSet highWatermark) {
|
||||||
this.highWatermark = highWatermark.subtract(lowWatermark);
|
this.highWatermark = highWatermark.subtract(lowWatermark);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -89,10 +88,10 @@ public boolean reachedHighWatermark(String currentGtid) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
String[] gtid = GTID_DELIMITER.split(currentGtid);
|
String[] gtid = GTID_DELIMITER.split(currentGtid);
|
||||||
GtidSet.UUIDSet uuidSet = getUuidSet(gtid[0]);
|
MySqlGtidSet.UUIDSet uuidSet = getUuidSet(gtid[0]);
|
||||||
if (uuidSet != null) {
|
if (uuidSet != null) {
|
||||||
long maxTransactionId = uuidSet.getIntervals().stream()
|
long maxTransactionId = uuidSet.getIntervals().stream()
|
||||||
.mapToLong(GtidSet.Interval::getEnd)
|
.mapToLong(MySqlGtidSet.Interval::getEnd)
|
||||||
.max()
|
.max()
|
||||||
.getAsLong();
|
.getAsLong();
|
||||||
if (maxTransactionId <= Long.parseLong(gtid[1])) {
|
if (maxTransactionId <= Long.parseLong(gtid[1])) {
|
||||||
@ -115,7 +114,7 @@ public void closeWindow() {
|
|||||||
lowWatermark = null;
|
lowWatermark = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private GtidSet.UUIDSet getUuidSet(String serverId) {
|
private MySqlGtidSet.UUIDSet getUuidSet(String serverId) {
|
||||||
return highWatermark.getUUIDSets().isEmpty() ? lowWatermark.forServerWithId(serverId) : highWatermark.forServerWithId(serverId);
|
return highWatermark.getUUIDSets().isEmpty() ? lowWatermark.forServerWithId(serverId) : highWatermark.forServerWithId(serverId);
|
||||||
}
|
}
|
||||||
|
|
@ -16,8 +16,9 @@
|
|||||||
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import io.debezium.connector.mysql.GtidSet.Interval;
|
import io.debezium.connector.mysql.strategy.mysql.MySqlGtidSet;
|
||||||
import io.debezium.connector.mysql.GtidSet.UUIDSet;
|
import io.debezium.connector.mysql.strategy.mysql.MySqlGtidSet.Interval;
|
||||||
|
import io.debezium.connector.mysql.strategy.mysql.MySqlGtidSet.UUIDSet;
|
||||||
import io.debezium.util.Collect;
|
import io.debezium.util.Collect;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -28,11 +29,11 @@ public class GtidSetTest {
|
|||||||
|
|
||||||
private static final String UUID1 = "24bc7850-2c16-11e6-a073-0242ac110002";
|
private static final String UUID1 = "24bc7850-2c16-11e6-a073-0242ac110002";
|
||||||
|
|
||||||
private GtidSet gtids;
|
private MySqlGtidSet gtids;
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldCreateSetWithSingleInterval() {
|
public void shouldCreateSetWithSingleInterval() {
|
||||||
gtids = new GtidSet(UUID1 + ":1-191");
|
gtids = new MySqlGtidSet(UUID1 + ":1-191");
|
||||||
asertIntervalCount(UUID1, 1);
|
asertIntervalCount(UUID1, 1);
|
||||||
asertIntervalExists(UUID1, 1, 191);
|
asertIntervalExists(UUID1, 1, 191);
|
||||||
asertFirstInterval(UUID1, 1, 191);
|
asertFirstInterval(UUID1, 1, 191);
|
||||||
@ -42,7 +43,7 @@ public void shouldCreateSetWithSingleInterval() {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldCollapseAdjacentIntervals() {
|
public void shouldCollapseAdjacentIntervals() {
|
||||||
gtids = new GtidSet(UUID1 + ":1-191:192-199");
|
gtids = new MySqlGtidSet(UUID1 + ":1-191:192-199");
|
||||||
asertIntervalCount(UUID1, 1);
|
asertIntervalCount(UUID1, 1);
|
||||||
asertIntervalExists(UUID1, 1, 199);
|
asertIntervalExists(UUID1, 1, 199);
|
||||||
asertFirstInterval(UUID1, 1, 199);
|
asertFirstInterval(UUID1, 1, 199);
|
||||||
@ -52,7 +53,7 @@ public void shouldCollapseAdjacentIntervals() {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldNotCollapseNonAdjacentIntervals() {
|
public void shouldNotCollapseNonAdjacentIntervals() {
|
||||||
gtids = new GtidSet(UUID1 + ":1-191:193-199");
|
gtids = new MySqlGtidSet(UUID1 + ":1-191:193-199");
|
||||||
asertIntervalCount(UUID1, 2);
|
asertIntervalCount(UUID1, 2);
|
||||||
asertFirstInterval(UUID1, 1, 191);
|
asertFirstInterval(UUID1, 1, 191);
|
||||||
asertLastInterval(UUID1, 193, 199);
|
asertLastInterval(UUID1, 193, 199);
|
||||||
@ -61,7 +62,7 @@ public void shouldNotCollapseNonAdjacentIntervals() {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldCreateWithMultipleIntervals() {
|
public void shouldCreateWithMultipleIntervals() {
|
||||||
gtids = new GtidSet(UUID1 + ":1-191:193-199:1000-1033");
|
gtids = new MySqlGtidSet(UUID1 + ":1-191:193-199:1000-1033");
|
||||||
asertIntervalCount(UUID1, 3);
|
asertIntervalCount(UUID1, 3);
|
||||||
asertFirstInterval(UUID1, 1, 191);
|
asertFirstInterval(UUID1, 1, 191);
|
||||||
asertIntervalExists(UUID1, 193, 199);
|
asertIntervalExists(UUID1, 193, 199);
|
||||||
@ -71,7 +72,7 @@ public void shouldCreateWithMultipleIntervals() {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldCreateWithMultipleIntervalsThatMayBeAdjacent() {
|
public void shouldCreateWithMultipleIntervalsThatMayBeAdjacent() {
|
||||||
gtids = new GtidSet(UUID1 + ":1-191:192-199:1000-1033:1035-1036:1038-1039");
|
gtids = new MySqlGtidSet(UUID1 + ":1-191:192-199:1000-1033:1035-1036:1038-1039");
|
||||||
asertIntervalCount(UUID1, 4);
|
asertIntervalCount(UUID1, 4);
|
||||||
asertFirstInterval(UUID1, 1, 199);
|
asertFirstInterval(UUID1, 1, 199);
|
||||||
asertIntervalExists(UUID1, 1000, 1033);
|
asertIntervalExists(UUID1, 1000, 1033);
|
||||||
@ -82,19 +83,19 @@ public void shouldCreateWithMultipleIntervalsThatMayBeAdjacent() {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldCorrectlyDetermineIfSimpleGtidSetIsContainedWithinAnother() {
|
public void shouldCorrectlyDetermineIfSimpleGtidSetIsContainedWithinAnother() {
|
||||||
gtids = new GtidSet("7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41");
|
gtids = new MySqlGtidSet("7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41");
|
||||||
assertThat(gtids.isContainedWithin(new GtidSet("7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41"))).isTrue();
|
assertThat(gtids.isContainedWithin(new MySqlGtidSet("7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41"))).isTrue();
|
||||||
assertThat(gtids.isContainedWithin(new GtidSet("7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-42"))).isTrue();
|
assertThat(gtids.isContainedWithin(new MySqlGtidSet("7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-42"))).isTrue();
|
||||||
assertThat(gtids.isContainedWithin(new GtidSet("7c1de3f2-3fd2-11e6-9cdc-42010af000bc:2-41"))).isFalse();
|
assertThat(gtids.isContainedWithin(new MySqlGtidSet("7c1de3f2-3fd2-11e6-9cdc-42010af000bc:2-41"))).isFalse();
|
||||||
assertThat(gtids.isContainedWithin(new GtidSet("7145bf69-d1ca-11e5-a588-0242ac110004:1"))).isFalse();
|
assertThat(gtids.isContainedWithin(new MySqlGtidSet("7145bf69-d1ca-11e5-a588-0242ac110004:1"))).isFalse();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldCorrectlyDetermineIfComplexGtidSetIsContainedWithinAnother() {
|
public void shouldCorrectlyDetermineIfComplexGtidSetIsContainedWithinAnother() {
|
||||||
GtidSet connector = new GtidSet("036d85a9-64e5-11e6-9b48-42010af0000c:1-2,"
|
MySqlGtidSet connector = new MySqlGtidSet("036d85a9-64e5-11e6-9b48-42010af0000c:1-2,"
|
||||||
+ "7145bf69-d1ca-11e5-a588-0242ac110004:1-3200,"
|
+ "7145bf69-d1ca-11e5-a588-0242ac110004:1-3200,"
|
||||||
+ "7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41");
|
+ "7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41");
|
||||||
GtidSet server = new GtidSet("036d85a9-64e5-11e6-9b48-42010af0000c:1-2,"
|
MySqlGtidSet server = new MySqlGtidSet("036d85a9-64e5-11e6-9b48-42010af0000c:1-2,"
|
||||||
+ "7145bf69-d1ca-11e5-a588-0242ac110004:1-3202,"
|
+ "7145bf69-d1ca-11e5-a588-0242ac110004:1-3202,"
|
||||||
+ "7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41");
|
+ "7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41");
|
||||||
assertThat(connector.isContainedWithin(server)).isTrue();
|
assertThat(connector.isContainedWithin(server)).isTrue();
|
||||||
@ -102,12 +103,12 @@ public void shouldCorrectlyDetermineIfComplexGtidSetIsContainedWithinAnother() {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldCorrectlyDetermineIfComplexGtidSetWithVariousLineSeparatorsIsContainedWithinAnother() {
|
public void shouldCorrectlyDetermineIfComplexGtidSetWithVariousLineSeparatorsIsContainedWithinAnother() {
|
||||||
GtidSet connector = new GtidSet("036d85a9-64e5-11e6-9b48-42010af0000c:1-2,"
|
GtidSet connector = new MySqlGtidSet("036d85a9-64e5-11e6-9b48-42010af0000c:1-2,"
|
||||||
+ "7145bf69-d1ca-11e5-a588-0242ac110004:1-3200,"
|
+ "7145bf69-d1ca-11e5-a588-0242ac110004:1-3200,"
|
||||||
+ "7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41");
|
+ "7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41");
|
||||||
Arrays.stream(new String[]{ "\r\n", "\n", "\r" })
|
Arrays.stream(new String[]{ "\r\n", "\n", "\r" })
|
||||||
.forEach(separator -> {
|
.forEach(separator -> {
|
||||||
GtidSet server = new GtidSet("036d85a9-64e5-11e6-9b48-42010af0000c:1-2," + separator +
|
GtidSet server = new MySqlGtidSet("036d85a9-64e5-11e6-9b48-42010af0000c:1-2," + separator +
|
||||||
"7145bf69-d1ca-11e5-a588-0242ac110004:1-3202," + separator +
|
"7145bf69-d1ca-11e5-a588-0242ac110004:1-3202," + separator +
|
||||||
"7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41");
|
"7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-41");
|
||||||
assertThat(connector.isContainedWithin(server)).isTrue();
|
assertThat(connector.isContainedWithin(server)).isTrue();
|
||||||
@ -122,12 +123,12 @@ public void shouldFilterServerUuids() {
|
|||||||
Collection<String> keepers = Collect.arrayListOf("036d85a9-64e5-11e6-9b48-42010af0000c",
|
Collection<String> keepers = Collect.arrayListOf("036d85a9-64e5-11e6-9b48-42010af0000c",
|
||||||
"7c1de3f2-3fd2-11e6-9cdc-42010af000bc",
|
"7c1de3f2-3fd2-11e6-9cdc-42010af000bc",
|
||||||
"wont-be-found");
|
"wont-be-found");
|
||||||
GtidSet original = new GtidSet(gtidStr);
|
MySqlGtidSet original = new MySqlGtidSet(gtidStr);
|
||||||
assertThat(original.forServerWithId("036d85a9-64e5-11e6-9b48-42010af0000c")).isNotNull();
|
assertThat(original.forServerWithId("036d85a9-64e5-11e6-9b48-42010af0000c")).isNotNull();
|
||||||
assertThat(original.forServerWithId("7c1de3f2-3fd2-11e6-9cdc-42010af000bc")).isNotNull();
|
assertThat(original.forServerWithId("7c1de3f2-3fd2-11e6-9cdc-42010af000bc")).isNotNull();
|
||||||
assertThat(original.forServerWithId("7145bf69-d1ca-11e5-a588-0242ac110004")).isNotNull();
|
assertThat(original.forServerWithId("7145bf69-d1ca-11e5-a588-0242ac110004")).isNotNull();
|
||||||
|
|
||||||
GtidSet filtered = original.retainAll(keepers::contains);
|
MySqlGtidSet filtered = original.retainAll(keepers::contains);
|
||||||
List<String> actualUuids = filtered.getUUIDSets().stream().map(UUIDSet::getUUID).collect(Collectors.toList());
|
List<String> actualUuids = filtered.getUUIDSets().stream().map(UUIDSet::getUUID).collect(Collectors.toList());
|
||||||
assertThat(keepers.containsAll(actualUuids)).isTrue();
|
assertThat(keepers.containsAll(actualUuids)).isTrue();
|
||||||
assertThat(filtered.forServerWithId("7145bf69-d1ca-11e5-a588-0242ac110004")).isNull();
|
assertThat(filtered.forServerWithId("7145bf69-d1ca-11e5-a588-0242ac110004")).isNull();
|
||||||
@ -144,11 +145,11 @@ public void subtract() {
|
|||||||
String diff = "036d85a9-64e5-11e6-9b48-42010af0000c:21,"
|
String diff = "036d85a9-64e5-11e6-9b48-42010af0000c:21,"
|
||||||
+ "7145bf69-d1ca-11e5-a588-0242ac110004:4500,"
|
+ "7145bf69-d1ca-11e5-a588-0242ac110004:4500,"
|
||||||
+ "7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-4:9-11:19-24:66-70:80-100";
|
+ "7c1de3f2-3fd2-11e6-9cdc-42010af000bc:1-4:9-11:19-24:66-70:80-100";
|
||||||
GtidSet gtidSet1 = new GtidSet(gtidStr1);
|
MySqlGtidSet gtidSet1 = new MySqlGtidSet(gtidStr1);
|
||||||
GtidSet gtidSet2 = new GtidSet(gtidStr2);
|
MySqlGtidSet gtidSet2 = new MySqlGtidSet(gtidStr2);
|
||||||
|
|
||||||
GtidSet gtidSetDiff = gtidSet2.subtract(gtidSet1);
|
MySqlGtidSet gtidSetDiff = gtidSet2.subtract(gtidSet1);
|
||||||
GtidSet expectedDiff = new GtidSet(diff);
|
MySqlGtidSet expectedDiff = new MySqlGtidSet(diff);
|
||||||
assertThat(gtidSetDiff).isEqualTo(expectedDiff);
|
assertThat(gtidSetDiff).isEqualTo(expectedDiff);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -68,7 +68,8 @@ private MySqlDatabaseSchema getSchema(Configuration config) {
|
|||||||
BigIntUnsignedMode.LONG,
|
BigIntUnsignedMode.LONG,
|
||||||
BinaryHandlingMode.BYTES,
|
BinaryHandlingMode.BYTES,
|
||||||
MySqlValueConverters::adjustTemporal,
|
MySqlValueConverters::adjustTemporal,
|
||||||
MySqlValueConverters::defaultParsingErrorHandler);
|
MySqlValueConverters::defaultParsingErrorHandler,
|
||||||
|
connectorConfig.getConnectorAdapter());
|
||||||
return new MySqlDatabaseSchema(
|
return new MySqlDatabaseSchema(
|
||||||
connectorConfig,
|
connectorConfig,
|
||||||
mySqlValueConverters,
|
mySqlValueConverters,
|
||||||
|
@ -119,7 +119,7 @@ public void testSkipInvalidJsonValues() {
|
|||||||
TemporalPrecisionMode.CONNECT, JdbcValueConverters.BigIntUnsignedMode.LONG, BinaryHandlingMode.BYTES,
|
TemporalPrecisionMode.CONNECT, JdbcValueConverters.BigIntUnsignedMode.LONG, BinaryHandlingMode.BYTES,
|
||||||
x -> x, (message, exception) -> {
|
x -> x, (message, exception) -> {
|
||||||
errorCount.incrementAndGet();
|
errorCount.incrementAndGet();
|
||||||
});
|
}, null);
|
||||||
|
|
||||||
DdlParser parser = new MySqlAntlrDdlParser();
|
DdlParser parser = new MySqlAntlrDdlParser();
|
||||||
Tables tables = new Tables();
|
Tables tables = new Tables();
|
||||||
@ -148,7 +148,7 @@ public void testErrorOnInvalidJsonValues() {
|
|||||||
TemporalPrecisionMode.CONNECT, JdbcValueConverters.BigIntUnsignedMode.LONG, BinaryHandlingMode.BYTES,
|
TemporalPrecisionMode.CONNECT, JdbcValueConverters.BigIntUnsignedMode.LONG, BinaryHandlingMode.BYTES,
|
||||||
x -> x, (message, exception) -> {
|
x -> x, (message, exception) -> {
|
||||||
throw new DebeziumException(message, exception);
|
throw new DebeziumException(message, exception);
|
||||||
});
|
}, null);
|
||||||
|
|
||||||
DdlParser parser = new MySqlAntlrDdlParser();
|
DdlParser parser = new MySqlAntlrDdlParser();
|
||||||
Tables tables = new Tables();
|
Tables tables = new Tables();
|
||||||
@ -171,7 +171,7 @@ public void testFallbackDecimalValueScale() {
|
|||||||
TemporalPrecisionMode.CONNECT, JdbcValueConverters.BigIntUnsignedMode.LONG, BinaryHandlingMode.BYTES,
|
TemporalPrecisionMode.CONNECT, JdbcValueConverters.BigIntUnsignedMode.LONG, BinaryHandlingMode.BYTES,
|
||||||
x -> x, (message, exception) -> {
|
x -> x, (message, exception) -> {
|
||||||
throw new DebeziumException(message, exception);
|
throw new DebeziumException(message, exception);
|
||||||
});
|
}, null);
|
||||||
|
|
||||||
DdlParser parser = new MySqlAntlrDdlParser();
|
DdlParser parser = new MySqlAntlrDdlParser();
|
||||||
Tables tables = new Tables();
|
Tables tables = new Tables();
|
||||||
@ -194,7 +194,7 @@ public void testZonedDateTimeWithMicrosecondPrecision() {
|
|||||||
TemporalPrecisionMode.ADAPTIVE_TIME_MICROSECONDS, JdbcValueConverters.BigIntUnsignedMode.LONG, BinaryHandlingMode.BYTES,
|
TemporalPrecisionMode.ADAPTIVE_TIME_MICROSECONDS, JdbcValueConverters.BigIntUnsignedMode.LONG, BinaryHandlingMode.BYTES,
|
||||||
x -> x, (message, exception) -> {
|
x -> x, (message, exception) -> {
|
||||||
throw new DebeziumException(message, exception);
|
throw new DebeziumException(message, exception);
|
||||||
});
|
}, null);
|
||||||
|
|
||||||
DdlParser parser = new MySqlAntlrDdlParser();
|
DdlParser parser = new MySqlAntlrDdlParser();
|
||||||
Tables tables = new Tables();
|
Tables tables = new Tables();
|
||||||
|
@ -26,6 +26,7 @@
|
|||||||
import io.debezium.config.CommonConnectorConfig;
|
import io.debezium.config.CommonConnectorConfig;
|
||||||
import io.debezium.config.Configuration;
|
import io.debezium.config.Configuration;
|
||||||
import io.debezium.connector.AbstractSourceInfoStructMaker;
|
import io.debezium.connector.AbstractSourceInfoStructMaker;
|
||||||
|
import io.debezium.connector.mysql.strategy.mysql.MySqlHistoryRecordComparator;
|
||||||
import io.debezium.data.VerifyRecord;
|
import io.debezium.data.VerifyRecord;
|
||||||
import io.debezium.doc.FixFor;
|
import io.debezium.doc.FixFor;
|
||||||
import io.debezium.document.Document;
|
import io.debezium.document.Document;
|
||||||
|
@ -220,6 +220,9 @@ public Configuration.Builder defaultJdbcConfigBuilder() {
|
|||||||
builder.with(FileSchemaHistory.FILE_PATH, dbHistoryPath);
|
builder.with(FileSchemaHistory.FILE_PATH, dbHistoryPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
String connectorAdapter = System.getProperty("connector.adapter", "mysql");
|
||||||
|
builder.with(MySqlConnectorConfig.CONNECTOR_ADAPTER, connectorAdapter);
|
||||||
|
|
||||||
return builder;
|
return builder;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -31,9 +31,9 @@
|
|||||||
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
import io.debezium.connector.mysql.MySqlConnectorConfig;
|
||||||
import io.debezium.connector.mysql.MySqlOffsetContext;
|
import io.debezium.connector.mysql.MySqlOffsetContext;
|
||||||
import io.debezium.connector.mysql.MySqlPartition;
|
import io.debezium.connector.mysql.MySqlPartition;
|
||||||
import io.debezium.connector.mysql.MySqlReadOnlyIncrementalSnapshotContext;
|
|
||||||
import io.debezium.connector.mysql.SourceInfo;
|
import io.debezium.connector.mysql.SourceInfo;
|
||||||
import io.debezium.connector.mysql.antlr.MySqlAntlrDdlParser;
|
import io.debezium.connector.mysql.antlr.MySqlAntlrDdlParser;
|
||||||
|
import io.debezium.connector.mysql.strategy.mysql.MySqlReadOnlyIncrementalSnapshotContext;
|
||||||
import io.debezium.doc.FixFor;
|
import io.debezium.doc.FixFor;
|
||||||
import io.debezium.junit.logging.LogInterceptor;
|
import io.debezium.junit.logging.LogInterceptor;
|
||||||
import io.debezium.kafka.KafkaCluster;
|
import io.debezium.kafka.KafkaCluster;
|
||||||
|
Loading…
Reference in New Issue
Block a user