alexey.rassokhin/tet123
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

DBZ-7645 Revert removal of LogMiner continuous mining

Browse Source
This commit is contained in:
Chris Cranford 2024-03-14 17:28:28 -04:00 committed by Jiri Pechanec
parent 95a8145015
commit e26d3efedc
5 changed files with 57 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
debezium-connector-oracle/src/main/java/io/debezium/connector/oracle/OracleConnectorConfig.java
View File

@ -621,6 +621,17 @@ public class OracleConnectorConfig extends HistorizedRelationalDatabaseConnector
.withValidation(Field::isNonNegativeInteger) .withValidation(Field::isNonNegativeInteger)
.withDescription("The number of attempts to retry database errors during snapshots before failing."); .withDescription("The number of attempts to retry database errors during snapshots before failing.");
@Deprecated
public static final Field LOG_MINING_CONTINUOUS_MINE = Field.create("log.mining.continuous.mine")
.withDisplayName("Should log mining session configured with CONTINUOUS_MINE setting?")
.withType(Type.BOOLEAN)
.withWidth(Width.SHORT)
.withImportance(Importance.LOW)
.withDefault(false)
.withValidation(Field::isBoolean)
.withDescription("(Deprecated) if true, CONTINUOUS_MINE option will be added to the log mining session. " +
"This will manage log files switches seamlessly.");
private static final ConfigDefinition CONFIG_DEFINITION = HistorizedRelationalDatabaseConnectorConfig.CONFIG_DEFINITION.edit() private static final ConfigDefinition CONFIG_DEFINITION = HistorizedRelationalDatabaseConnectorConfig.CONFIG_DEFINITION.edit()
.name("Oracle") .name("Oracle")
.excluding( .excluding(
@ -692,7 +703,8 @@ public class OracleConnectorConfig extends HistorizedRelationalDatabaseConnector
OLR_SOURCE, OLR_SOURCE,
OLR_HOST, OLR_HOST,
OLR_PORT, OLR_PORT,
SNAPSHOT_DATABASE_ERRORS_MAX_RETRIES) SNAPSHOT_DATABASE_ERRORS_MAX_RETRIES,
LOG_MINING_CONTINUOUS_MINE)
.events(SOURCE_INFO_STRUCT_MAKER) .events(SOURCE_INFO_STRUCT_MAKER)
.create(); .create();
@ -760,6 +772,7 @@ public static ConfigDef configDef() {
private final String logMiningInifispanGlobalConfiguration; private final String logMiningInifispanGlobalConfiguration;
private final Set<String> logMiningSchemaChangesUsernameExcludes; private final Set<String> logMiningSchemaChangesUsernameExcludes;
private final Boolean logMiningIncludeRedoSql; private final Boolean logMiningIncludeRedoSql;
private final boolean logMiningContinuousMining;
private final String openLogReplicatorSource; private final String openLogReplicatorSource;
private final String openLogReplicatorHostname; private final String openLogReplicatorHostname;
@ -828,6 +841,7 @@ public OracleConnectorConfig(Configuration config) {
this.logMiningInifispanGlobalConfiguration = config.getString(LOG_MINING_BUFFER_INFINISPAN_CACHE_GLOBAL); this.logMiningInifispanGlobalConfiguration = config.getString(LOG_MINING_BUFFER_INFINISPAN_CACHE_GLOBAL);
this.logMiningSchemaChangesUsernameExcludes = Strings.setOf(config.getString(LOG_MINING_SCHEMA_CHANGES_USERNAME_EXCLUDE_LIST), String::new); this.logMiningSchemaChangesUsernameExcludes = Strings.setOf(config.getString(LOG_MINING_SCHEMA_CHANGES_USERNAME_EXCLUDE_LIST), String::new);
this.logMiningIncludeRedoSql = config.getBoolean(LOG_MINING_INCLUDE_REDO_SQL); this.logMiningIncludeRedoSql = config.getBoolean(LOG_MINING_INCLUDE_REDO_SQL);
this.logMiningContinuousMining = config.getBoolean(LOG_MINING_CONTINUOUS_MINE);
// OpenLogReplicator // OpenLogReplicator
this.openLogReplicatorSource = config.getString(OLR_SOURCE); this.openLogReplicatorSource = config.getString(OLR_SOURCE);
@ -1875,6 +1889,16 @@ public boolean isLogMiningIncludeRedoSql() {
return logMiningIncludeRedoSql; return logMiningIncludeRedoSql;
} }
/**
* Returns whether the LogMiner adapter should use continuous mining or not.
*
* @return true continuous mining should be used
*/
@Deprecated
public boolean isLogMiningContinuousMining() {
return logMiningContinuousMining;
}
/** /**
* Returns the logical source to stream changes from when connecting to OpenLogReplicator. * Returns the logical source to stream changes from when connecting to OpenLogReplicator.
* *

25
debezium-connector-oracle/src/main/java/io/debezium/connector/oracle/logminer/LogMinerStreamingChangeEventSource.java
View File

@ -82,6 +82,7 @@ public class LogMinerStreamingChangeEventSource implements StreamingChangeEventS
private final int logFileQueryMaxRetries; private final int logFileQueryMaxRetries;
private final Duration initialDelay; private final Duration initialDelay;
private final Duration maxDelay; private final Duration maxDelay;
private final boolean continuousMining;
private Scn startScn; // startScn is the **exclusive** lower bound for mining private Scn startScn; // startScn is the **exclusive** lower bound for mining
private Scn endScn; private Scn endScn;
@ -115,6 +116,8 @@ public LogMinerStreamingChangeEventSource(OracleConnectorConfig connectorConfig,
this.maxDelay = connectorConfig.getLogMiningMaxDelay(); this.maxDelay = connectorConfig.getLogMiningMaxDelay();
this.currentBatchSize = connectorConfig.getLogMiningBatchSizeDefault(); this.currentBatchSize = connectorConfig.getLogMiningBatchSizeDefault();
this.currentSleepTime = connectorConfig.getLogMiningSleepTimeDefault().toMillis(); this.currentSleepTime = connectorConfig.getLogMiningSleepTimeDefault().toMillis();
this.continuousMining = connectorConfig.isLogMiningContinuousMining();
this.snapshotterService = snapshotterService; this.snapshotterService = snapshotterService;
this.streamingMetrics.setBatchSize(this.currentBatchSize); this.streamingMetrics.setBatchSize(this.currentBatchSize);
@ -166,7 +169,7 @@ public void execute(ChangeEventSourceContext context, OraclePartition partition,
} }
try (LogWriterFlushStrategy flushStrategy = resolveFlushStrategy()) { try (LogWriterFlushStrategy flushStrategy = resolveFlushStrategy()) {
if (startScn.compareTo(firstScn.subtract(Scn.ONE)) < 0) { if (!continuousMining && startScn.compareTo(firstScn.subtract(Scn.ONE)) < 0) {
// startScn is the exclusive lower bound, so must be >= (firstScn - 1) // startScn is the exclusive lower bound, so must be >= (firstScn - 1)
throw new DebeziumException( throw new DebeziumException(
"Online REDO LOG files or archive log files do not contain the offset scn " + startScn + ". Please perform a new snapshot."); "Online REDO LOG files or archive log files do not contain the offset scn " + startScn + ". Please perform a new snapshot.");
@ -181,7 +184,7 @@ public void execute(ChangeEventSourceContext context, OraclePartition partition,
return; return;
} }
initializeRedoLogsForMining(jdbcConnection, startScn); initializeRedoLogsForMining(jdbcConnection, false, startScn);
int retryAttempts = 1; int retryAttempts = 1;
Stopwatch sw = Stopwatch.accumulating().start(); Stopwatch sw = Stopwatch.accumulating().start();
@ -237,7 +240,7 @@ public void execute(ChangeEventSourceContext context, OraclePartition partition,
if (connectorConfig.isLogMiningRestartConnection()) { if (connectorConfig.isLogMiningRestartConnection()) {
prepareConnection(true); prepareConnection(true);
} }
initializeRedoLogsForMining(jdbcConnection, startScn); initializeRedoLogsForMining(jdbcConnection, true, startScn);
// log switch or restart required, re-create a new stop watch // log switch or restart required, re-create a new stop watch
sw = Stopwatch.accumulating().start(); sw = Stopwatch.accumulating().start();
@ -393,14 +396,16 @@ private LogMinerEventProcessor createProcessor(ChangeEventSourceContext context,
return bufferType.createProcessor(context, connectorConfig, jdbcConnection, dispatcher, partition, offsetContext, schema, streamingMetrics); return bufferType.createProcessor(context, connectorConfig, jdbcConnection, dispatcher, partition, offsetContext, schema, streamingMetrics);
} }
private void initializeRedoLogsForMining(OracleConnection connection, Scn startScn) throws SQLException { private void initializeRedoLogsForMining(OracleConnection connection, boolean postEndMiningSession, Scn startScn) throws SQLException {
if (OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO.equals(strategy)) { if ((!postEndMiningSession || !continuousMining)
&& OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO.equals(strategy)) {
buildDataDictionary(connection); buildDataDictionary(connection);
} }
if (!continuousMining) {
currentLogFiles = setLogFilesForMining(connection, startScn, archiveLogRetention, archiveLogOnlyMode, currentLogFiles = setLogFilesForMining(connection, startScn, archiveLogRetention, archiveLogOnlyMode,
archiveDestinationName, logFileQueryMaxRetries, initialDelay, maxDelay); archiveDestinationName, logFileQueryMaxRetries, initialDelay, maxDelay);
currentRedoLogSequences = getCurrentLogFileSequences(currentLogFiles); currentRedoLogSequences = getCurrentLogFileSequences(currentLogFiles);
}
updateRedoLogMetrics(); updateRedoLogMetrics();
} }
@ -600,12 +605,12 @@ private OffsetDateTime getDatabaseSystemTime(OracleConnection connection) throws
* @throws SQLException if mining session failed to start * @throws SQLException if mining session failed to start
*/ */
public boolean startMiningSession(OracleConnection connection, Scn startScn, Scn endScn, int attempts) throws SQLException { public boolean startMiningSession(OracleConnection connection, Scn startScn, Scn endScn, int attempts) throws SQLException {
LOGGER.debug("Starting mining session startScn={}, endScn={}, strategy={}", startScn, endScn, strategy); LOGGER.debug("Starting mining session startScn={}, endScn={}, strategy={}, continuous={}", startScn, endScn, strategy, continuousMining);
try { try {
Instant start = Instant.now(); Instant start = Instant.now();
// NOTE: we treat startSCN as the _exclusive_ lower bound for mining, // NOTE: we treat startSCN as the _exclusive_ lower bound for mining,
// whereas START_LOGMNR takes an _inclusive_ lower bound, hence the increment. // whereas START_LOGMNR takes an _inclusive_ lower bound, hence the increment.
connection.executeWithoutCommitting(SqlUtils.startLogMinerStatement(startScn.add(Scn.ONE), endScn, strategy)); connection.executeWithoutCommitting(SqlUtils.startLogMinerStatement(startScn.add(Scn.ONE), endScn, strategy, continuousMining));
streamingMetrics.setLastMiningSessionStartDuration(Duration.between(start, Instant.now())); streamingMetrics.setLastMiningSessionStartDuration(Duration.between(start, Instant.now()));
return true; return true;
} }
@ -636,8 +641,8 @@ public boolean startMiningSession(OracleConnection connection, Scn startScn, Scn
*/ */
public void endMiningSession(OracleConnection connection, OracleOffsetContext offsetContext) throws SQLException { public void endMiningSession(OracleConnection connection, OracleOffsetContext offsetContext) throws SQLException {
try { try {
LOGGER.trace("Ending log mining startScn={}, endScn={}, offsetContext.getScn={}, strategy={}", LOGGER.trace("Ending log mining startScn={}, endScn={}, offsetContext.getScn={}, strategy={}, continuous={}",
startScn, endScn, offsetContext.getScn(), strategy); startScn, endScn, offsetContext.getScn(), strategy, continuousMining);
connection.executeWithoutCommitting("BEGIN SYS.DBMS_LOGMNR.END_LOGMNR(); END;"); connection.executeWithoutCommitting("BEGIN SYS.DBMS_LOGMNR.END_LOGMNR(); END;");
} }
catch (SQLException e) { catch (SQLException e) {

6
debezium-connector-oracle/src/main/java/io/debezium/connector/oracle/logminer/SqlUtils.java
View File

@ -197,9 +197,10 @@ private static String localArchiveLogDestinationsOnlyQuery(String archiveDestina
* @param startScn mine from * @param startScn mine from
* @param endScn mine till * @param endScn mine till
* @param strategy Log Mining strategy * @param strategy Log Mining strategy
* @param continuousMining whether to use continuous mining
* @return statement todo: handle corruption. STATUS (Double) — value of 0 indicates it is executable * @return statement todo: handle corruption. STATUS (Double) — value of 0 indicates it is executable
*/ */
static String startLogMinerStatement(Scn startScn, Scn endScn, OracleConnectorConfig.LogMiningStrategy strategy) { static String startLogMinerStatement(Scn startScn, Scn endScn, OracleConnectorConfig.LogMiningStrategy strategy, boolean continuousMining) {
String miningStrategy; String miningStrategy;
if (strategy.equals(OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO)) { if (strategy.equals(OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO)) {
miningStrategy = "DBMS_LOGMNR.DICT_FROM_REDO_LOGS + DBMS_LOGMNR.DDL_DICT_TRACKING "; miningStrategy = "DBMS_LOGMNR.DICT_FROM_REDO_LOGS + DBMS_LOGMNR.DDL_DICT_TRACKING ";
@ -207,6 +208,9 @@ static String startLogMinerStatement(Scn startScn, Scn endScn, OracleConnectorCo
else { else {
miningStrategy = "DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG "; miningStrategy = "DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG ";
} }
if (continuousMining) {
miningStrategy += " + DBMS_LOGMNR.CONTINUOUS_MINE ";
}
return "BEGIN sys.dbms_logmnr.start_logmnr(" + return "BEGIN sys.dbms_logmnr.start_logmnr(" +
"startScn => '" + startScn + "', " + "startScn => '" + startScn + "', " +
"endScn => '" + endScn + "', " + "endScn => '" + endScn + "', " +

12
debezium-connector-oracle/src/test/java/io/debezium/connector/oracle/logminer/LogMinerUtilsTest.java
View File

@ -34,26 +34,26 @@ public class LogMinerUtilsTest {
@Test @Test
public void testStartLogMinerStatement() { public void testStartLogMinerStatement() {
String statement = SqlUtils.startLogMinerStatement(SCN, OTHER_SCN, OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO); String statement = SqlUtils.startLogMinerStatement(SCN, OTHER_SCN, OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO, false);
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isTrue(); assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isTrue();
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isTrue(); assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isTrue();
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isFalse(); assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isFalse();
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isFalse(); assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isFalse();
statement = SqlUtils.startLogMinerStatement(SCN, OTHER_SCN, OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG); statement = SqlUtils.startLogMinerStatement(SCN, OTHER_SCN, OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG, false);
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isFalse(); assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isFalse();
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isFalse(); assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isFalse();
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isTrue(); assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isTrue();
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isFalse(); assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isFalse();
statement = SqlUtils.startLogMinerStatement(SCN, OTHER_SCN, OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO); statement = SqlUtils.startLogMinerStatement(SCN, OTHER_SCN, OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO, true);
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isTrue(); assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isTrue();
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isTrue(); assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isTrue();
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isFalse(); assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isFalse();
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isFalse(); assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isTrue();
statement = SqlUtils.startLogMinerStatement(SCN, OTHER_SCN, OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG); statement = SqlUtils.startLogMinerStatement(SCN, OTHER_SCN, OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG, true);
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isFalse(); assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isFalse();
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isFalse(); assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isFalse();
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isTrue(); assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isTrue();
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isFalse(); assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isTrue();
} }
// todo delete after replacement == -1 in the code // todo delete after replacement == -1 in the code

6
debezium-connector-oracle/src/test/java/io/debezium/connector/oracle/logminer/SqlUtilsTest.java
View File

@ -40,12 +40,12 @@ public void testStatements() {
expected = "SELECT 'KEY', LOG_GROUP_TYPE FROM ALL_LOG_GROUPS WHERE OWNER = 's' AND TABLE_NAME = 't'"; expected = "SELECT 'KEY', LOG_GROUP_TYPE FROM ALL_LOG_GROUPS WHERE OWNER = 's' AND TABLE_NAME = 't'";
assertThat(result).isEqualTo(expected); assertThat(result).isEqualTo(expected);
result = SqlUtils.startLogMinerStatement(Scn.valueOf(10L), Scn.valueOf(20L), OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG); result = SqlUtils.startLogMinerStatement(Scn.valueOf(10L), Scn.valueOf(20L), OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG, true);
expected = "BEGIN sys.dbms_logmnr.start_logmnr(startScn => '10', endScn => '20', " + expected = "BEGIN sys.dbms_logmnr.start_logmnr(startScn => '10', endScn => '20', " +
"OPTIONS => DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG + DBMS_LOGMNR.NO_ROWID_IN_STMT);END;"; "OPTIONS => DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG + DBMS_LOGMNR.CONTINUOUS_MINE + DBMS_LOGMNR.NO_ROWID_IN_STMT);END;";
assertThat(result).isEqualTo(expected); assertThat(result).isEqualTo(expected);
result = SqlUtils.startLogMinerStatement(Scn.valueOf(10L), Scn.valueOf(20L), OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO); result = SqlUtils.startLogMinerStatement(Scn.valueOf(10L), Scn.valueOf(20L), OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO, false);
expected = "BEGIN sys.dbms_logmnr.start_logmnr(startScn => '10', endScn => '20', " + expected = "BEGIN sys.dbms_logmnr.start_logmnr(startScn => '10', endScn => '20', " +
"OPTIONS => DBMS_LOGMNR.DICT_FROM_REDO_LOGS + DBMS_LOGMNR.DDL_DICT_TRACKING + DBMS_LOGMNR.NO_ROWID_IN_STMT);END;"; "OPTIONS => DBMS_LOGMNR.DICT_FROM_REDO_LOGS + DBMS_LOGMNR.DDL_DICT_TRACKING + DBMS_LOGMNR.NO_ROWID_IN_STMT);END;";
assertThat(result).isEqualTo(expected); assertThat(result).isEqualTo(expected);