DBZ-137 log miner oracle contribution
This commit is contained in:
parent
7c2cc444bb
commit
d344ca48f8
@ -74,6 +74,12 @@ Then the Oracle connector can be built like so:
|
||||
|
||||
$ mvn clean install -pl debezium-connector-oracle -am -Poracle -Dinstantclient.dir=/path/to/instant-client-dir
|
||||
|
||||
#### Oracle Log Miner
|
||||
|
||||
If the connector is to be built and tested using the Oracle Log Miner implementation, it can be built like so:
|
||||
|
||||
$ mvn clean install -pl debezium-connector-oracle -am -Poracle,logminer -Dinstantclient.dir=/home/to/instant-client-dir
|
||||
|
||||
#### For Oracle 11g
|
||||
|
||||
To run Debezium Oracle connector with Oracle 11g, add these additional parameters. If running with Oracle 12c+, leave these parameters to default.
|
||||
|
@ -41,6 +41,11 @@
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-runtime</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.jsqlparser</groupId>
|
||||
<artifactId>jsqlparser</artifactId>
|
||||
<version>2.1</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Testing -->
|
||||
<dependency>
|
||||
@ -85,6 +90,12 @@
|
||||
<artifactId>awaitility</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Required by VerifyRecord -->
|
||||
<dependency>
|
||||
<groupId>io.confluent</groupId>
|
||||
@ -93,6 +104,10 @@
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<properties>
|
||||
<adapter.name>xstream</adapter.name>
|
||||
</properties>
|
||||
|
||||
<build>
|
||||
<resources>
|
||||
<!-- Apply the properties set in the POM to the resource files -->
|
||||
@ -139,6 +154,9 @@
|
||||
<enableAssertions>true</enableAssertions>
|
||||
<forkCount>1</forkCount>
|
||||
<argLine>-Djava.library.path=${instantclient.dir} -Ddebezium.embedded.shutdown.pause.before.interrupt.ms=20000</argLine>
|
||||
<systemPropertyVariables>
|
||||
<database.connection.adapter>${adapter.name}</database.connection.adapter>
|
||||
</systemPropertyVariables>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
@ -186,5 +204,11 @@
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>logminer</id>
|
||||
<properties>
|
||||
<adapter.name>logminer</adapter.name>
|
||||
</properties>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle;
|
||||
|
||||
import io.debezium.data.Envelope.Operation;
|
||||
import io.debezium.pipeline.spi.OffsetContext;
|
||||
import io.debezium.relational.RelationalChangeRecordEmitter;
|
||||
import io.debezium.relational.Table;
|
||||
import io.debezium.util.Clock;
|
||||
|
||||
/**
|
||||
* Base class to emit change data based on a single entry event.
|
||||
*/
|
||||
public abstract class BaseChangeRecordEmitter<T> extends RelationalChangeRecordEmitter {
|
||||
|
||||
protected final Table table;
|
||||
|
||||
protected BaseChangeRecordEmitter(OffsetContext offset, Table table, Clock clock) {
|
||||
super(offset, clock);
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
abstract protected Operation getOperation();
|
||||
|
||||
abstract protected String getColumnName(T columnValue);
|
||||
|
||||
abstract protected Object getColumnData(T columnValue);
|
||||
|
||||
protected Object[] getColumnValues(T[] columnValues) {
|
||||
Object[] values = new Object[table.columns().size()];
|
||||
|
||||
for (T columnValue : columnValues) {
|
||||
int index = table.columnWithName(getColumnName(columnValue)).position() - 1;
|
||||
values[index] = getColumnData(columnValue);
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
}
|
@ -18,25 +18,31 @@
|
||||
import io.debezium.schema.SchemaChangeEvent;
|
||||
import io.debezium.schema.SchemaChangeEvent.SchemaChangeEventType;
|
||||
|
||||
import oracle.streams.DDLLCR;
|
||||
|
||||
/**
|
||||
* {@link SchemaChangeEventEmitter} implementation based on Oracle.
|
||||
*
|
||||
* @author Gunnar Morling
|
||||
*/
|
||||
public class OracleSchemaChangeEventEmitter implements SchemaChangeEventEmitter {
|
||||
public class BaseOracleSchemaChangeEventEmitter implements SchemaChangeEventEmitter {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(OracleSchemaChangeEventEmitter.class);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(BaseOracleSchemaChangeEventEmitter.class);
|
||||
|
||||
private final OracleOffsetContext offsetContext;
|
||||
private final TableId tableId;
|
||||
private final DDLLCR ddlLcr;
|
||||
private String sourceDatabaseName;
|
||||
private String objectOwner;
|
||||
private String ddlText;
|
||||
private String commandType;
|
||||
|
||||
public OracleSchemaChangeEventEmitter(OracleOffsetContext offsetContext, TableId tableId, DDLLCR ddlLcr) {
|
||||
public BaseOracleSchemaChangeEventEmitter(OracleOffsetContext offsetContext, TableId tableId,
|
||||
String sourceDatabaseName, String objectOwner, String ddlText,
|
||||
String commandType) {
|
||||
this.offsetContext = offsetContext;
|
||||
this.tableId = tableId;
|
||||
this.ddlLcr = ddlLcr;
|
||||
this.sourceDatabaseName = sourceDatabaseName;
|
||||
this.objectOwner = objectOwner;
|
||||
this.ddlText = ddlText;
|
||||
this.commandType = commandType;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -49,13 +55,13 @@ public void emitSchemaChangeEvent(Receiver receiver) throws InterruptedException
|
||||
Tables tables = new Tables();
|
||||
|
||||
OracleDdlParser parser = new OracleDdlParser();
|
||||
parser.setCurrentDatabase(ddlLcr.getSourceDatabaseName());
|
||||
parser.setCurrentSchema(ddlLcr.getObjectOwner());
|
||||
parser.parse(ddlLcr.getDDLText(), tables);
|
||||
parser.setCurrentDatabase(sourceDatabaseName);
|
||||
parser.setCurrentSchema(objectOwner);
|
||||
parser.parse(ddlText, tables);
|
||||
|
||||
Set<TableId> changedTableIds = tables.drainChanges();
|
||||
if (changedTableIds.isEmpty()) {
|
||||
throw new IllegalArgumentException("Couldn't parse DDL statement " + ddlLcr.getDDLText());
|
||||
throw new IllegalArgumentException("Couldn't parse DDL statement " + ddlText);
|
||||
}
|
||||
|
||||
Table table = tables.forTable(tableId);
|
||||
@ -64,16 +70,16 @@ public void emitSchemaChangeEvent(Receiver receiver) throws InterruptedException
|
||||
offsetContext.getPartition(),
|
||||
offsetContext.getOffset(),
|
||||
offsetContext.getSourceInfo(),
|
||||
ddlLcr.getSourceDatabaseName(),
|
||||
ddlLcr.getObjectOwner(),
|
||||
ddlLcr.getDDLText(),
|
||||
sourceDatabaseName,
|
||||
objectOwner,
|
||||
ddlText,
|
||||
table,
|
||||
eventType,
|
||||
false));
|
||||
}
|
||||
|
||||
private SchemaChangeEventType getSchemaChangeEventType() {
|
||||
switch (ddlLcr.getCommandType()) {
|
||||
switch (commandType) {
|
||||
case "CREATE TABLE":
|
||||
return SchemaChangeEventType.CREATE;
|
||||
case "ALTER TABLE":
|
||||
@ -83,7 +89,7 @@ private SchemaChangeEventType getSchemaChangeEventType() {
|
||||
LOGGER.warn("DROP TABLE not yet implemented");
|
||||
break;
|
||||
default:
|
||||
LOGGER.debug("Ignoring DDL event of type {}", ddlLcr.getCommandType());
|
||||
LOGGER.debug("Ignoring DDL event of type {}", commandType);
|
||||
}
|
||||
|
||||
return null;
|
@ -5,6 +5,9 @@
|
||||
*/
|
||||
package io.debezium.connector.oracle;
|
||||
|
||||
import io.debezium.config.Configuration;
|
||||
import io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSource;
|
||||
import io.debezium.connector.oracle.xstream.XstreamStreamingChangeEventSource;
|
||||
import io.debezium.pipeline.ErrorHandler;
|
||||
import io.debezium.pipeline.EventDispatcher;
|
||||
import io.debezium.pipeline.source.spi.ChangeEventSourceFactory;
|
||||
@ -23,15 +26,20 @@ public class OracleChangeEventSourceFactory implements ChangeEventSourceFactory
|
||||
private final EventDispatcher<TableId> dispatcher;
|
||||
private final Clock clock;
|
||||
private final OracleDatabaseSchema schema;
|
||||
private final Configuration config;
|
||||
private final OracleTaskContext taskContext;
|
||||
|
||||
public OracleChangeEventSourceFactory(OracleConnectorConfig configuration, OracleConnection jdbcConnection,
|
||||
ErrorHandler errorHandler, EventDispatcher<TableId> dispatcher, Clock clock, OracleDatabaseSchema schema) {
|
||||
ErrorHandler errorHandler, EventDispatcher<TableId> dispatcher, Clock clock, OracleDatabaseSchema schema,
|
||||
Configuration config, OracleTaskContext taskContext) {
|
||||
this.configuration = configuration;
|
||||
this.jdbcConnection = jdbcConnection;
|
||||
this.errorHandler = errorHandler;
|
||||
this.dispatcher = dispatcher;
|
||||
this.clock = clock;
|
||||
this.schema = schema;
|
||||
this.config = config;
|
||||
this.taskContext = taskContext;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -42,7 +50,9 @@ public SnapshotChangeEventSource getSnapshotChangeEventSource(OffsetContext offs
|
||||
|
||||
@Override
|
||||
public StreamingChangeEventSource getStreamingChangeEventSource(OffsetContext offsetContext) {
|
||||
return new OracleStreamingChangeEventSource(
|
||||
OracleConnectorConfig.ConnectorAdapter adapter = configuration.getAdapter();
|
||||
if (adapter == OracleConnectorConfig.ConnectorAdapter.XSTREAM) {
|
||||
return new XstreamStreamingChangeEventSource(
|
||||
configuration,
|
||||
(OracleOffsetContext) offsetContext,
|
||||
jdbcConnection,
|
||||
@ -51,4 +61,14 @@ public StreamingChangeEventSource getStreamingChangeEventSource(OffsetContext of
|
||||
clock,
|
||||
schema);
|
||||
}
|
||||
return new LogMinerStreamingChangeEventSource(
|
||||
configuration,
|
||||
(OracleOffsetContext) offsetContext,
|
||||
jdbcConnection,
|
||||
dispatcher,
|
||||
errorHandler,
|
||||
clock,
|
||||
schema,
|
||||
taskContext);
|
||||
}
|
||||
}
|
||||
|
@ -5,12 +5,22 @@
|
||||
*/
|
||||
package io.debezium.connector.oracle;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Types;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
@ -19,6 +29,7 @@
|
||||
import io.debezium.config.Configuration;
|
||||
import io.debezium.jdbc.JdbcConnection;
|
||||
import io.debezium.relational.Column;
|
||||
import io.debezium.relational.ColumnEditor;
|
||||
import io.debezium.relational.TableEditor;
|
||||
import io.debezium.relational.TableId;
|
||||
import io.debezium.relational.Tables;
|
||||
@ -36,8 +47,8 @@ public class OracleConnection extends JdbcConnection {
|
||||
*/
|
||||
private static final int ORACLE_UNSET_SCALE = -127;
|
||||
|
||||
public OracleConnection(Configuration config, ConnectionFactory connectionFactory) {
|
||||
super(config, connectionFactory);
|
||||
public OracleConnection(Configuration config, ConnectionFactory connectionFactory, Supplier<ClassLoader> classLoaderSupplier) {
|
||||
super(config, connectionFactory, classLoaderSupplier);
|
||||
}
|
||||
|
||||
public void setSessionToPdb(String pdbName) {
|
||||
@ -96,6 +107,89 @@ public Set<TableId> readTableNames(String databaseCatalog, String schemaNamePatt
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
protected Set<TableId> getAllTableIds(String catalogName, String schemaNamePattern, boolean isView) throws SQLException {
|
||||
|
||||
String query = "select table_name, owner from all_tables where table_name NOT LIKE 'MDRT_%' AND table_name not LIKE 'MDXT_%' " +
|
||||
" and owner like '%" + schemaNamePattern.toUpperCase() + "%'";
|
||||
if (isView) {
|
||||
query = "select view_name, owner from all_views where owner like '%" + schemaNamePattern.toUpperCase() + "%'";
|
||||
}
|
||||
Set<TableId> tableIds = new HashSet<>();
|
||||
|
||||
try (PreparedStatement statement = connection().prepareStatement(query);
|
||||
ResultSet result = statement.executeQuery();) {
|
||||
while (result.next()) {
|
||||
String tableName = result.getString(1);
|
||||
final String schemaName = result.getString(2);
|
||||
TableId tableId = new TableId(catalogName, schemaName, tableName);
|
||||
tableIds.add(tableId);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
LOGGER.trace("TableIds are: {}", tableIds);
|
||||
}
|
||||
return tableIds;
|
||||
}
|
||||
|
||||
@Override // todo replace metadata with a query
|
||||
protected Optional<ColumnEditor> readTableColumn(ResultSet columnMetadata, TableId tableId, ColumnNameFilter columnFilter) throws SQLException {
|
||||
return super.readTableColumn(columnMetadata, tableId, columnFilter);
|
||||
}
|
||||
|
||||
// todo replace metadata with something like this
|
||||
private ResultSet getTableColumnsInfo(String schemaNamePattern, String tableName) throws SQLException {
|
||||
String columnQuery = "select column_name, data_type, data_length, data_precision, data_scale, default_length, density, char_length from " +
|
||||
"all_tab_columns where owner like '" + schemaNamePattern + "' and table_name='" + tableName + "'";
|
||||
|
||||
PreparedStatement statement = connection().prepareStatement(columnQuery);
|
||||
return statement.executeQuery();
|
||||
}
|
||||
|
||||
// this is much faster, we will use it until full replacement of the metadata usage TODO
|
||||
public void readSchemaForCapturedTables(Tables tables, String databaseCatalog, String schemaNamePattern,
|
||||
ColumnNameFilter columnFilter, boolean removeTablesNotFoundInJdbc, Set<TableId> capturedTables)
|
||||
throws SQLException {
|
||||
|
||||
Set<TableId> tableIdsBefore = new HashSet<>(tables.tableIds());
|
||||
|
||||
DatabaseMetaData metadata = connection().getMetaData();
|
||||
Map<TableId, List<Column>> columnsByTable = new HashMap<>();
|
||||
|
||||
for (TableId tableId : capturedTables) {
|
||||
try (ResultSet columnMetadata = metadata.getColumns(databaseCatalog, schemaNamePattern, tableId.table(), null)) {
|
||||
while (columnMetadata.next()) {
|
||||
// add all whitelisted columns
|
||||
readTableColumn(columnMetadata, tableId, columnFilter).ifPresent(column -> {
|
||||
columnsByTable.computeIfAbsent(tableId, t -> new ArrayList<>())
|
||||
.add(column.create());
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Read the metadata for the primary keys ...
|
||||
for (Map.Entry<TableId, List<Column>> tableEntry : columnsByTable.entrySet()) {
|
||||
// First get the primary key information, which must be done for *each* table ...
|
||||
List<String> pkColumnNames = readPrimaryKeyNames(metadata, tableEntry.getKey());
|
||||
|
||||
// Then define the table ...
|
||||
List<Column> columns = tableEntry.getValue();
|
||||
Collections.sort(columns);
|
||||
tables.overwriteTable(tableEntry.getKey(), columns, pkColumnNames, null);
|
||||
}
|
||||
|
||||
if (removeTablesNotFoundInJdbc) {
|
||||
// Remove any definitions for tables that were not found in the database metadata ...
|
||||
tableIdsBefore.removeAll(columnsByTable.keySet());
|
||||
tableIdsBefore.forEach(tables::removeTable);
|
||||
}
|
||||
|
||||
for (TableId tableId : capturedTables) {
|
||||
overrideOracleSpecificColumnTypes(tables, tableId, tableId);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void readSchema(Tables tables, String databaseCatalog, String schemaNamePattern, TableFilter tableFilter,
|
||||
ColumnNameFilter columnFilter, boolean removeTablesNotFoundInJdbc)
|
||||
@ -111,6 +205,14 @@ public void readSchema(Tables tables, String databaseCatalog, String schemaNameP
|
||||
TableId tableIdWithCatalog = new TableId(databaseCatalog, tableId.schema(), tableId.table());
|
||||
|
||||
if (tableFilter.isIncluded(tableIdWithCatalog)) {
|
||||
overrideOracleSpecificColumnTypes(tables, tableId, tableIdWithCatalog);
|
||||
}
|
||||
|
||||
tables.removeTable(tableId);
|
||||
}
|
||||
}
|
||||
|
||||
private void overrideOracleSpecificColumnTypes(Tables tables, TableId tableId, TableId tableIdWithCatalog) {
|
||||
TableEditor editor = tables.editTable(tableId);
|
||||
editor.tableId(tableIdWithCatalog);
|
||||
|
||||
@ -140,7 +242,22 @@ else if (column.jdbcType() == OracleTypes.NUMBER) {
|
||||
tables.overwriteTable(editor.create());
|
||||
}
|
||||
|
||||
tables.removeTable(tableId);
|
||||
public OracleConnection executeLegacy(String... sqlStatements) throws SQLException {
|
||||
return executeLegacy(statement -> {
|
||||
for (String sqlStatement : sqlStatements) {
|
||||
if (sqlStatement != null) {
|
||||
statement.execute(sqlStatement);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public OracleConnection executeLegacy(Operations operations) throws SQLException {
|
||||
Connection conn = connection();
|
||||
try (Statement statement = conn.createStatement()) {
|
||||
operations.apply(statement);
|
||||
commit();
|
||||
}
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
@ -22,7 +22,9 @@ public Connection connect(JdbcConfiguration config) throws SQLException {
|
||||
String user = config.getUser();
|
||||
String password = config.getPassword();
|
||||
|
||||
return DriverManager.getConnection(
|
||||
"jdbc:oracle:oci:@" + hostName + ":" + port + "/" + database, user, password);
|
||||
String driverType = config.getString(OracleConnectorConfig.DRIVER_TYPE);
|
||||
final String url = "jdbc:oracle:" + driverType + ":@" + hostName + ":" + port + "/" + database;
|
||||
|
||||
return DriverManager.getConnection(url, user, password);
|
||||
}
|
||||
}
|
||||
|
@ -5,6 +5,8 @@
|
||||
*/
|
||||
package io.debezium.connector.oracle;
|
||||
|
||||
import java.util.function.Predicate;
|
||||
|
||||
import org.apache.kafka.common.config.ConfigDef;
|
||||
import org.apache.kafka.common.config.ConfigDef.Importance;
|
||||
import org.apache.kafka.common.config.ConfigDef.Type;
|
||||
@ -17,12 +19,17 @@
|
||||
import io.debezium.config.Field.ValidationOutput;
|
||||
import io.debezium.connector.AbstractSourceInfo;
|
||||
import io.debezium.connector.SourceInfoStructMaker;
|
||||
import io.debezium.connector.oracle.xstream.LcrPosition;
|
||||
import io.debezium.connector.oracle.xstream.OracleVersion;
|
||||
import io.debezium.document.Document;
|
||||
import io.debezium.function.Predicates;
|
||||
import io.debezium.heartbeat.Heartbeat;
|
||||
import io.debezium.jdbc.JdbcConfiguration;
|
||||
import io.debezium.relational.ColumnId;
|
||||
import io.debezium.relational.HistorizedRelationalDatabaseConnectorConfig;
|
||||
import io.debezium.relational.RelationalDatabaseConnectorConfig;
|
||||
import io.debezium.relational.TableId;
|
||||
import io.debezium.relational.Tables;
|
||||
import io.debezium.relational.Tables.TableFilter;
|
||||
import io.debezium.relational.history.HistoryRecordComparator;
|
||||
import io.debezium.relational.history.KafkaDatabaseHistory;
|
||||
@ -90,6 +97,13 @@ public class OracleConnectorConfig extends HistorizedRelationalDatabaseConnector
|
||||
.withDescription("Name of the pluggable database when working with a multi-tenant set-up. "
|
||||
+ "The CDB name must be given via " + DATABASE_NAME.name() + " in this case.");
|
||||
|
||||
public static final Field SCHEMA_NAME = Field.create(DATABASE_CONFIG_PREFIX + "schema")
|
||||
.withDisplayName("Schema name")
|
||||
.withType(Type.STRING)
|
||||
.withWidth(Width.MEDIUM)
|
||||
.withImportance(Importance.HIGH)
|
||||
.withDescription("Name of the connection user to the database ");
|
||||
|
||||
public static final Field XSTREAM_SERVER_NAME = Field.create(DATABASE_CONFIG_PREFIX + "out.server.name")
|
||||
.withDisplayName("XStream out server name")
|
||||
.withType(Type.STRING)
|
||||
@ -125,6 +139,55 @@ public class OracleConnectorConfig extends HistorizedRelationalDatabaseConnector
|
||||
public static final Field SERVER_NAME = RelationalDatabaseConnectorConfig.SERVER_NAME
|
||||
.withValidation(CommonConnectorConfig::validateServerNameIsDifferentFromHistoryTopicName);
|
||||
|
||||
public static final Field CONNECTOR_ADAPTER = Field.create(DATABASE_CONFIG_PREFIX + "connection.adapter")
|
||||
.withDisplayName("Connector adapter")
|
||||
.withEnum(ConnectorAdapter.class, ConnectorAdapter.XSTREAM)
|
||||
.withWidth(Width.MEDIUM)
|
||||
.withImportance(Importance.HIGH)
|
||||
.withDescription("There are two adapters: XStream and LogMiner.");
|
||||
|
||||
public static final Field SNAPSHOT_SKIP_LOCKS = Field.create("snapshot.skip.locks")
|
||||
.withDisplayName("Should schema be locked as we build the schema snapshot?")
|
||||
.withType(Type.BOOLEAN)
|
||||
.withWidth(Width.SHORT)
|
||||
.withImportance(Importance.LOW)
|
||||
.withDefault(false)
|
||||
.withValidation(Field::isBoolean)
|
||||
.withDescription(
|
||||
"If true, locks all tables to be captured as we build the schema snapshot. This will prevent from any concurrent schema changes being applied to them.");
|
||||
|
||||
public static final Field LOG_MINING_STRATEGY = Field.create("log.mining.strategy")
|
||||
.withDisplayName("Log Mining Strategy")
|
||||
.withEnum(LogMiningStrategy.class, LogMiningStrategy.CATALOG_IN_REDO)
|
||||
.withWidth(Width.MEDIUM)
|
||||
.withImportance(Importance.HIGH)
|
||||
.withDescription("There are strategies: Online catalog with faster mining but no captured DDL. Another - with data dictionary loaded into REDO LOG files");
|
||||
|
||||
// this option could be true up to Oracle 18c version. Starting from Oracle 19c this option cannot be true todo should we do it?
|
||||
public static final Field CONTINUOUS_MINE = Field.create("log.mining.continuous.mine")
|
||||
.withDisplayName("Should log mining session configured with CONTINUOUS_MINE setting?")
|
||||
.withType(Type.BOOLEAN)
|
||||
.withWidth(Width.SHORT)
|
||||
.withImportance(Importance.LOW)
|
||||
.withDefault(false)
|
||||
.withValidation(Field::isBoolean)
|
||||
.withDescription("If true, CONTINUOUS_MINE option will be added to the log mining session. This will manage log files switches seamlessly.");
|
||||
|
||||
public static final Field SNAPSHOT_ENHANCEMENT_TOKEN = Field.create("snapshot.enhance.predicate.scn")
|
||||
.withDisplayName("A string to replace on snapshot predicate enhancement")
|
||||
.withType(Type.STRING)
|
||||
.withWidth(Width.MEDIUM)
|
||||
.withImportance(Importance.HIGH)
|
||||
.withDescription("A token to replace on snapshot predicate template");
|
||||
|
||||
public static final Field DRIVER_TYPE = Field.create("database.driver.type")
|
||||
.withDisplayName("oci for xStream or thin for LogMiner")
|
||||
.withType(Type.STRING)
|
||||
.withWidth(Width.SHORT)
|
||||
.withImportance(Importance.HIGH)
|
||||
.withDefault("oci")
|
||||
.withDescription("A token to use in connection factories");
|
||||
|
||||
/**
|
||||
* The set of {@link Field}s defined as part of this configuration.
|
||||
*/
|
||||
@ -149,11 +212,18 @@ public class OracleConnectorConfig extends HistorizedRelationalDatabaseConnector
|
||||
CommonConnectorConfig.MAX_BATCH_SIZE,
|
||||
CommonConnectorConfig.MAX_QUEUE_SIZE,
|
||||
CommonConnectorConfig.SNAPSHOT_DELAY_MS,
|
||||
CommonConnectorConfig.SNAPSHOT_FETCH_SIZE,
|
||||
CommonConnectorConfig.PROVIDE_TRANSACTION_METADATA,
|
||||
Heartbeat.HEARTBEAT_INTERVAL,
|
||||
Heartbeat.HEARTBEAT_TOPICS_PREFIX,
|
||||
TABLENAME_CASE_INSENSITIVE,
|
||||
ORACLE_VERSION,
|
||||
SNAPSHOT_SKIP_LOCKS,
|
||||
SCHEMA_NAME,
|
||||
CONNECTOR_ADAPTER,
|
||||
LOG_MINING_STRATEGY,
|
||||
SNAPSHOT_ENHANCEMENT_TOKEN,
|
||||
DRIVER_TYPE,
|
||||
CommonConnectorConfig.EVENT_PROCESSING_FAILURE_HANDLING_MODE);
|
||||
|
||||
private final String databaseName;
|
||||
@ -163,23 +233,46 @@ public class OracleConnectorConfig extends HistorizedRelationalDatabaseConnector
|
||||
|
||||
private final boolean tablenameCaseInsensitive;
|
||||
private final OracleVersion oracleVersion;
|
||||
private final String schemaName;
|
||||
private final Tables.ColumnNameFilter columnFilter;
|
||||
|
||||
public OracleConnectorConfig(Configuration config) {
|
||||
super(OracleConnector.class, config, config.getString(SERVER_NAME), new SystemTablesPredicate(), x -> x.schema() + "." + x.table(), true);
|
||||
|
||||
this.databaseName = config.getString(DATABASE_NAME);
|
||||
this.pdbName = config.getString(PDB_NAME);
|
||||
this.databaseName = setUpperCase(config.getString(DATABASE_NAME));
|
||||
this.pdbName = setUpperCase(config.getString(PDB_NAME));
|
||||
this.xoutServerName = config.getString(XSTREAM_SERVER_NAME);
|
||||
this.snapshotMode = SnapshotMode.parse(config.getString(SNAPSHOT_MODE));
|
||||
this.tablenameCaseInsensitive = config.getBoolean(TABLENAME_CASE_INSENSITIVE);
|
||||
this.oracleVersion = OracleVersion.parse(config.getString(ORACLE_VERSION));
|
||||
this.schemaName = setUpperCase(config.getString(SCHEMA_NAME));
|
||||
String blacklistedColumns = setUpperCase(config.getString(RelationalDatabaseConnectorConfig.COLUMN_BLACKLIST));
|
||||
this.columnFilter = getColumnNameFilter(blacklistedColumns);
|
||||
}
|
||||
|
||||
private String setUpperCase(String property) {
|
||||
property = property == null ? null : property.toUpperCase();
|
||||
return property;
|
||||
}
|
||||
|
||||
protected Tables.ColumnNameFilter getColumnNameFilter(String excludedColumnPatterns) {
|
||||
return new Tables.ColumnNameFilter() {
|
||||
|
||||
Predicate<ColumnId> delegate = Predicates.excludes(excludedColumnPatterns, ColumnId::toString);
|
||||
|
||||
@Override
|
||||
public boolean matches(String catalogName, String schemaName, String tableName, String columnName) {
|
||||
// ignore database name and schema name, we are supposed to capture from one database and one schema
|
||||
return delegate.test(new ColumnId(new TableId(null, null, tableName), columnName));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public static ConfigDef configDef() {
|
||||
ConfigDef config = new ConfigDef();
|
||||
|
||||
Field.group(config, "Oracle", HOSTNAME, PORT, USER, PASSWORD, SERVER_NAME, DATABASE_NAME, PDB_NAME,
|
||||
XSTREAM_SERVER_NAME, SNAPSHOT_MODE);
|
||||
XSTREAM_SERVER_NAME, SNAPSHOT_MODE, CONNECTOR_ADAPTER, LOG_MINING_STRATEGY);
|
||||
Field.group(config, "History Storage", KafkaDatabaseHistory.BOOTSTRAP_SERVERS,
|
||||
KafkaDatabaseHistory.TOPIC, KafkaDatabaseHistory.RECOVERY_POLL_ATTEMPTS,
|
||||
KafkaDatabaseHistory.RECOVERY_POLL_INTERVAL_MS, HistorizedRelationalDatabaseConnectorConfig.DATABASE_HISTORY);
|
||||
@ -188,12 +281,14 @@ public static ConfigDef configDef() {
|
||||
RelationalDatabaseConnectorConfig.TABLE_BLACKLIST,
|
||||
RelationalDatabaseConnectorConfig.TABLE_EXCLUDE_LIST,
|
||||
RelationalDatabaseConnectorConfig.MSG_KEY_COLUMNS,
|
||||
RelationalDatabaseConnectorConfig.COLUMN_BLACKLIST,
|
||||
RelationalDatabaseConnectorConfig.TABLE_IGNORE_BUILTIN,
|
||||
CommonConnectorConfig.PROVIDE_TRANSACTION_METADATA,
|
||||
Heartbeat.HEARTBEAT_INTERVAL, Heartbeat.HEARTBEAT_TOPICS_PREFIX,
|
||||
CommonConnectorConfig.EVENT_PROCESSING_FAILURE_HANDLING_MODE);
|
||||
Field.group(config, "Connector", CommonConnectorConfig.POLL_INTERVAL_MS, CommonConnectorConfig.MAX_BATCH_SIZE,
|
||||
CommonConnectorConfig.MAX_QUEUE_SIZE, CommonConnectorConfig.SNAPSHOT_DELAY_MS);
|
||||
CommonConnectorConfig.MAX_QUEUE_SIZE, CommonConnectorConfig.SNAPSHOT_DELAY_MS, CommonConnectorConfig.SNAPSHOT_FETCH_SIZE,
|
||||
SNAPSHOT_SKIP_LOCKS, SNAPSHOT_ENHANCEMENT_TOKEN, DRIVER_TYPE);
|
||||
|
||||
return config;
|
||||
}
|
||||
@ -222,20 +317,37 @@ public OracleVersion getOracleVersion() {
|
||||
return oracleVersion;
|
||||
}
|
||||
|
||||
public String getSchemaName() {
|
||||
return schemaName;
|
||||
}
|
||||
|
||||
public Tables.ColumnNameFilter getColumnFilter() {
|
||||
return columnFilter;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected HistoryRecordComparator getHistoryRecordComparator() {
|
||||
return new HistoryRecordComparator() {
|
||||
@Override
|
||||
protected boolean isPositionAtOrBefore(Document recorded, Document desired) {
|
||||
Long recordedScn;
|
||||
Long desiredScn;
|
||||
if (getAdapter() == OracleConnectorConfig.ConnectorAdapter.XSTREAM) {
|
||||
final LcrPosition recordedPosition = LcrPosition.valueOf(recorded.getString(SourceInfo.LCR_POSITION_KEY));
|
||||
final LcrPosition desiredPosition = LcrPosition.valueOf(desired.getString(SourceInfo.LCR_POSITION_KEY));
|
||||
final Long recordedScn = recordedPosition != null ? recordedPosition.getScn() : recorded.getLong(SourceInfo.SCN_KEY);
|
||||
final Long desiredScn = desiredPosition != null ? desiredPosition.getScn() : desired.getLong(SourceInfo.SCN_KEY);
|
||||
|
||||
recordedScn = recordedPosition != null ? recordedPosition.getScn() : recorded.getLong(SourceInfo.SCN_KEY);
|
||||
desiredScn = desiredPosition != null ? desiredPosition.getScn() : desired.getLong(SourceInfo.SCN_KEY);
|
||||
return (recordedPosition != null && desiredPosition != null)
|
||||
? recordedPosition.compareTo(desiredPosition) < 1
|
||||
: recordedScn.compareTo(desiredScn) < 1;
|
||||
}
|
||||
else {
|
||||
recordedScn = recorded.getLong(SourceInfo.SCN_KEY);
|
||||
desiredScn = desired.getLong(SourceInfo.SCN_KEY);
|
||||
return recordedScn.compareTo(desiredScn) < 1;
|
||||
}
|
||||
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@ -375,6 +487,117 @@ public static SnapshotMode parse(String value, String defaultValue) {
|
||||
}
|
||||
}
|
||||
|
||||
public enum ConnectorAdapter implements EnumeratedValue {
|
||||
|
||||
/**
|
||||
* This is based on XStream API.
|
||||
*/
|
||||
XSTREAM("XStream"),
|
||||
|
||||
/**
|
||||
* This is based on LogMiner utility.
|
||||
*/
|
||||
LOG_MINER("LogMiner");
|
||||
|
||||
private final String value;
|
||||
|
||||
ConnectorAdapter(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if the supplied value is one of the predefined options.
|
||||
*
|
||||
* @param value the configuration property value; may not be null
|
||||
* @return the matching option, or null if no match is found
|
||||
*/
|
||||
public static ConnectorAdapter parse(String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
value = value.trim();
|
||||
for (ConnectorAdapter adapter : ConnectorAdapter.values()) {
|
||||
if (adapter.getValue().equalsIgnoreCase(value)) {
|
||||
return adapter;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static ConnectorAdapter parse(String value, String defaultValue) {
|
||||
ConnectorAdapter mode = parse(value);
|
||||
|
||||
if (mode == null && defaultValue != null) {
|
||||
mode = parse(defaultValue);
|
||||
}
|
||||
|
||||
return mode;
|
||||
}
|
||||
}
|
||||
|
||||
public enum LogMiningStrategy implements EnumeratedValue {
|
||||
|
||||
/**
|
||||
* This strategy uses Log Miner with data dictionary in online catalog.
|
||||
* This option will not capture DDL , but acts fast on REDO LOG switch events
|
||||
* This option does not use CONTINUOUS_MINE option
|
||||
*/
|
||||
ONLINE_CATALOG("online_catalog"),
|
||||
|
||||
/**
|
||||
* This strategy uses Log Miner with data dictionary in REDO LOG files.
|
||||
* This option will capture DDL, but will develop some lag on REDO LOG switch event and will eventually catch up
|
||||
* This option does not use CONTINUOUS_MINE option
|
||||
* This is default value
|
||||
*/
|
||||
CATALOG_IN_REDO("redo_log_catalog");
|
||||
|
||||
private final String value;
|
||||
|
||||
LogMiningStrategy(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if the supplied value is one of the predefined options.
|
||||
*
|
||||
* @param value the configuration property value; may not be null
|
||||
* @return the matching option, or null if no match is found
|
||||
*/
|
||||
public static LogMiningStrategy parse(String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
value = value.trim();
|
||||
for (LogMiningStrategy adapter : LogMiningStrategy.values()) {
|
||||
if (adapter.getValue().equalsIgnoreCase(value)) {
|
||||
return adapter;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static LogMiningStrategy parse(String value, String defaultValue) {
|
||||
LogMiningStrategy mode = parse(value);
|
||||
|
||||
if (mode == null && defaultValue != null) {
|
||||
mode = parse(defaultValue);
|
||||
}
|
||||
|
||||
return mode;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A {@link TableFilter} that excludes all Oracle system tables.
|
||||
*
|
||||
@ -415,6 +638,38 @@ public String getContextName() {
|
||||
return Module.contextName();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if lock to be obtained before building the schema
|
||||
*/
|
||||
public boolean skipSnapshotLock() {
|
||||
return getConfig().getBoolean(SNAPSHOT_SKIP_LOCKS);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return connection adapter
|
||||
*/
|
||||
public ConnectorAdapter getAdapter() {
|
||||
return ConnectorAdapter.parse(getConfig().getString(CONNECTOR_ADAPTER));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Log Mining strategy
|
||||
*/
|
||||
public LogMiningStrategy getLogMiningStrategy() {
|
||||
return LogMiningStrategy.parse(getConfig().getString(LOG_MINING_STRATEGY));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return String token to replace
|
||||
*/
|
||||
public String getTokenToReplaceInSnapshotPredicate() {
|
||||
return getConfig().getString(SNAPSHOT_ENHANCEMENT_TOKEN);
|
||||
}
|
||||
|
||||
public boolean isContinuousMining() {
|
||||
return getConfig().getBoolean(CONTINUOUS_MINE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the time.precision.mode configuration.
|
||||
*
|
||||
|
@ -51,11 +51,15 @@ public ChangeEventSourceCoordinator start(Configuration config) {
|
||||
SchemaNameAdjuster schemaNameAdjuster = SchemaNameAdjuster.create(LOGGER);
|
||||
|
||||
Configuration jdbcConfig = config.subset("database.", true);
|
||||
jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());
|
||||
jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory(), () -> getClass().getClassLoader());
|
||||
this.schema = new OracleDatabaseSchema(connectorConfig, schemaNameAdjuster, topicSelector, jdbcConnection);
|
||||
this.schema.initializeStorage();
|
||||
|
||||
OffsetContext previousOffset = getPreviousOffset(new OracleOffsetContext.Loader(connectorConfig));
|
||||
String adapterString = config.getString("connection.adapter");
|
||||
adapterString = adapterString == null ? config.getString(OracleConnectorConfig.CONNECTOR_ADAPTER) : adapterString;
|
||||
OracleConnectorConfig.ConnectorAdapter adapter = OracleConnectorConfig.ConnectorAdapter.parse(adapterString);
|
||||
OffsetContext previousOffset = getPreviousOffset(new OracleOffsetContext.Loader(connectorConfig, adapter));
|
||||
|
||||
if (previousOffset != null) {
|
||||
schema.recover(previousOffset);
|
||||
}
|
||||
@ -91,7 +95,7 @@ public ChangeEventSourceCoordinator start(Configuration config) {
|
||||
errorHandler,
|
||||
OracleConnector.class,
|
||||
connectorConfig,
|
||||
new OracleChangeEventSourceFactory(connectorConfig, jdbcConnection, errorHandler, dispatcher, clock, schema),
|
||||
new OracleChangeEventSourceFactory(connectorConfig, jdbcConnection, errorHandler, dispatcher, clock, schema, jdbcConfig, taskContext),
|
||||
new DefaultChangeEventSourceMetricsFactory(),
|
||||
dispatcher,
|
||||
schema);
|
||||
|
@ -13,6 +13,7 @@
|
||||
import io.debezium.relational.Table;
|
||||
import io.debezium.relational.TableId;
|
||||
import io.debezium.relational.TableSchemaBuilder;
|
||||
import io.debezium.relational.Tables;
|
||||
import io.debezium.relational.ddl.DdlParser;
|
||||
import io.debezium.relational.history.TableChanges;
|
||||
import io.debezium.schema.SchemaChangeEvent;
|
||||
@ -42,6 +43,10 @@ public OracleDatabaseSchema(OracleConnectorConfig connectorConfig, SchemaNameAdj
|
||||
connectorConfig.getKeyMapper());
|
||||
}
|
||||
|
||||
public Tables getTables() {
|
||||
return tables();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected DdlParser getDdlParser() {
|
||||
return new OracleDdlParser();
|
||||
|
@ -14,6 +14,7 @@
|
||||
import org.apache.kafka.connect.data.Struct;
|
||||
|
||||
import io.debezium.connector.SnapshotRecord;
|
||||
import io.debezium.connector.oracle.xstream.LcrPosition;
|
||||
import io.debezium.pipeline.spi.OffsetContext;
|
||||
import io.debezium.pipeline.txmetadata.TransactionContext;
|
||||
import io.debezium.relational.TableId;
|
||||
@ -35,6 +36,12 @@ public class OracleOffsetContext implements OffsetContext {
|
||||
*/
|
||||
private boolean snapshotCompleted;
|
||||
|
||||
public OracleOffsetContext(OracleConnectorConfig connectorConfig, long scn, Long commitScn, LcrPosition lcrPosition,
|
||||
boolean snapshot, boolean snapshotCompleted, TransactionContext transactionContext) {
|
||||
this(connectorConfig, scn, lcrPosition, snapshot, snapshotCompleted, transactionContext);
|
||||
sourceInfo.setCommitScn(commitScn);
|
||||
}
|
||||
|
||||
private OracleOffsetContext(OracleConnectorConfig connectorConfig, long scn, LcrPosition lcrPosition,
|
||||
boolean snapshot, boolean snapshotCompleted, TransactionContext transactionContext) {
|
||||
partition = Collections.singletonMap(SERVER_PARTITION_KEY, connectorConfig.getLogicalName());
|
||||
@ -126,6 +133,7 @@ public static Builder create() {
|
||||
}
|
||||
else {
|
||||
offset.put(SourceInfo.SCN_KEY, sourceInfo.getScn());
|
||||
offset.put(SourceInfo.COMMIT_SCN_KEY, sourceInfo.getCommitScn());
|
||||
}
|
||||
return transactionContext.store(offset);
|
||||
}
|
||||
@ -145,10 +153,18 @@ public void setScn(long scn) {
|
||||
sourceInfo.setScn(scn);
|
||||
}
|
||||
|
||||
public void setCommitScn(Long commitScn) {
|
||||
sourceInfo.setCommitScn(commitScn);
|
||||
}
|
||||
|
||||
public long getScn() {
|
||||
return sourceInfo.getScn();
|
||||
}
|
||||
|
||||
public Long getCommitScn() {
|
||||
return sourceInfo.getCommitScn();
|
||||
}
|
||||
|
||||
public void setLcrPosition(LcrPosition lcrPosition) {
|
||||
sourceInfo.setLcrPosition(lcrPosition);
|
||||
}
|
||||
@ -223,9 +239,12 @@ public TransactionContext getTransactionContext() {
|
||||
public static class Loader implements OffsetContext.Loader {
|
||||
|
||||
private final OracleConnectorConfig connectorConfig;
|
||||
private final OracleConnectorConfig.ConnectorAdapter adapter;
|
||||
|
||||
public Loader(OracleConnectorConfig connectorConfig) {
|
||||
// todo resolve adapter from the config rather than passing it
|
||||
public Loader(OracleConnectorConfig connectorConfig, OracleConnectorConfig.ConnectorAdapter adapter) {
|
||||
this.connectorConfig = connectorConfig;
|
||||
this.adapter = adapter;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -235,12 +254,20 @@ public Loader(OracleConnectorConfig connectorConfig) {
|
||||
|
||||
@Override
|
||||
public OffsetContext load(Map<String, ?> offset) {
|
||||
LcrPosition lcrPosition = LcrPosition.valueOf((String) offset.get(SourceInfo.LCR_POSITION_KEY));
|
||||
Long scn = lcrPosition != null ? lcrPosition.getScn() : (Long) offset.get(SourceInfo.SCN_KEY);
|
||||
boolean snapshot = Boolean.TRUE.equals(offset.get(SourceInfo.SNAPSHOT_KEY));
|
||||
boolean snapshotCompleted = Boolean.TRUE.equals(offset.get(SNAPSHOT_COMPLETED_KEY));
|
||||
|
||||
Long scn;
|
||||
if (adapter == OracleConnectorConfig.ConnectorAdapter.LOG_MINER) {
|
||||
scn = (Long) offset.get(SourceInfo.SCN_KEY);
|
||||
Long commitScn = (Long) offset.get(SourceInfo.COMMIT_SCN_KEY);
|
||||
return new OracleOffsetContext(connectorConfig, scn, commitScn, null, snapshot, snapshotCompleted, TransactionContext.load(offset));
|
||||
}
|
||||
else {
|
||||
LcrPosition lcrPosition = LcrPosition.valueOf((String) offset.get(SourceInfo.LCR_POSITION_KEY));
|
||||
scn = lcrPosition != null ? lcrPosition.getScn() : (Long) offset.get(SourceInfo.SCN_KEY);
|
||||
return new OracleOffsetContext(connectorConfig, scn, lcrPosition, snapshot, snapshotCompleted, TransactionContext.load(offset));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -10,6 +10,9 @@
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Savepoint;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
@ -17,6 +20,7 @@
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.connector.oracle.logminer.LogMinerHelper;
|
||||
import io.debezium.pipeline.EventDispatcher;
|
||||
import io.debezium.pipeline.source.spi.SnapshotProgressListener;
|
||||
import io.debezium.pipeline.source.spi.StreamingChangeEventSource;
|
||||
@ -54,6 +58,7 @@ public OracleSnapshotChangeEventSource(OracleConnectorConfig connectorConfig, Or
|
||||
protected SnapshottingTask getSnapshottingTask(OffsetContext previousOffset) {
|
||||
boolean snapshotSchema = true;
|
||||
boolean snapshotData = true;
|
||||
boolean skipSnapsotLock = false;
|
||||
|
||||
// found a previous offset and the earlier snapshot has completed
|
||||
if (previousOffset != null && !previousOffset.isSnapshotRunning()) {
|
||||
@ -62,9 +67,10 @@ protected SnapshottingTask getSnapshottingTask(OffsetContext previousOffset) {
|
||||
}
|
||||
else {
|
||||
snapshotData = connectorConfig.getSnapshotMode().includeData();
|
||||
skipSnapsotLock = connectorConfig.skipSnapshotLock();
|
||||
}
|
||||
|
||||
return new SnapshottingTask(snapshotSchema, snapshotData);
|
||||
return new SnapshottingTask(snapshotSchema, snapshotData, skipSnapsotLock);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -79,7 +85,9 @@ protected SnapshotContext prepare(ChangeEventSourceContext context) throws Excep
|
||||
|
||||
@Override
|
||||
protected Set<TableId> getAllTableIds(RelationalSnapshotContext ctx) throws Exception {
|
||||
return jdbcConnection.readTableNames(ctx.catalogName, null, null, new String[]{ "TABLE" });
|
||||
return jdbcConnection.getAllTableIds(ctx.catalogName, connectorConfig.getSchemaName(), false);
|
||||
// this very slow approach(commented out), it took 30 minutes on an instance with 600 tables
|
||||
// return jdbcConnection.readTableNames(ctx.catalogName, null, null, new String[] {"TABLE"} );
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -126,6 +134,10 @@ protected void determineSnapshotOffset(RelationalSnapshotContext ctx) throws Exc
|
||||
}
|
||||
|
||||
private long getCurrentScn(SnapshotContext ctx) throws SQLException {
|
||||
if (connectorConfig.getAdapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
|
||||
return LogMinerHelper.getCurrentScn(jdbcConnection.connection());
|
||||
}
|
||||
|
||||
try (Statement statement = jdbcConnection.connection().createStatement();
|
||||
ResultSet rs = statement.executeQuery("select CURRENT_SCN from V$DATABASE")) {
|
||||
|
||||
@ -195,6 +207,19 @@ protected void readTableStructure(ChangeEventSourceContext sourceContext, Relati
|
||||
throw new InterruptedException("Interrupted while reading structure of schema " + schema);
|
||||
}
|
||||
|
||||
// todo: DBZ-137 the new readSchemaForCapturedTables seems to cause failures.
|
||||
// For now, reverted to the default readSchema implementation as the intended goal
|
||||
// with the new implementation was to be faster, not change behavior.
|
||||
// if (connectorConfig.getAdapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
|
||||
// jdbcConnection.readSchemaForCapturedTables(
|
||||
// snapshotContext.tables,
|
||||
// snapshotContext.catalogName,
|
||||
// schema,
|
||||
// connectorConfig.getColumnFilter(),
|
||||
// false,
|
||||
// snapshotContext.capturedTables);
|
||||
// }
|
||||
// else {
|
||||
jdbcConnection.readSchema(
|
||||
snapshotContext.tables,
|
||||
snapshotContext.catalogName,
|
||||
@ -202,9 +227,22 @@ protected void readTableStructure(ChangeEventSourceContext sourceContext, Relati
|
||||
connectorConfig.getTableFilters().dataCollectionFilter(),
|
||||
null,
|
||||
false);
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String enhanceOverriddenSelect(RelationalSnapshotContext snapshotContext, String overriddenSelect, TableId tableId) {
|
||||
String columnString = buildSelectColumns(connectorConfig.getConfig().getString(connectorConfig.COLUMN_BLACKLIST), snapshotContext.tables.forTable(tableId));
|
||||
overriddenSelect = overriddenSelect.replaceFirst("\\*", columnString);
|
||||
long snapshotOffset = (Long) snapshotContext.offset.getOffset().get("scn");
|
||||
String token = connectorConfig.getTokenToReplaceInSnapshotPredicate();
|
||||
if (token != null) {
|
||||
return overriddenSelect.replaceAll(token, " AS OF SCN " + snapshotOffset);
|
||||
}
|
||||
return overriddenSelect;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SchemaChangeEvent getCreateTableEvent(RelationalSnapshotContext snapshotContext, Table table) throws SQLException {
|
||||
try (Statement statement = jdbcConnection.connection().createStatement();
|
||||
@ -231,11 +269,44 @@ protected SchemaChangeEvent getCreateTableEvent(RelationalSnapshotContext snapsh
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Optional<String> getSnapshotSelect(SnapshotContext snapshotContext, TableId tableId) {
|
||||
protected Optional<String> getSnapshotSelect(RelationalSnapshotContext snapshotContext, TableId tableId) {
|
||||
String columnString = buildSelectColumns(connectorConfig.getConfig().getString(connectorConfig.COLUMN_BLACKLIST), snapshotContext.tables.forTable(tableId));
|
||||
|
||||
long snapshotOffset = (Long) snapshotContext.offset.getOffset().get("scn");
|
||||
return Optional.of("SELECT * FROM " + quote(tableId) + " AS OF SCN " + snapshotOffset);
|
||||
}
|
||||
|
||||
/**
|
||||
* This is to build "whitelisted" column list
|
||||
* @param blackListColumnStr comma separated columns blacklist
|
||||
* @param table the table
|
||||
* @return column list for select
|
||||
*/
|
||||
public static String buildSelectColumns(String blackListColumnStr, Table table) {
|
||||
String columnsToSelect = "*";
|
||||
if (blackListColumnStr != null && blackListColumnStr.trim().length() > 0
|
||||
&& blackListColumnStr.toUpperCase().contains(table.id().table())) {
|
||||
String allTableColumns = table.retrieveColumnNames().stream()
|
||||
.map(columnName -> {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
if (!columnName.contains(table.id().table())) {
|
||||
sb.append(table.id().table()).append(".").append(columnName);
|
||||
}
|
||||
else {
|
||||
sb.append(columnName);
|
||||
}
|
||||
return sb.toString();
|
||||
}).collect(Collectors.joining(","));
|
||||
// todo this is an unnecessary code, fix unit test, then remove it
|
||||
String catalog = table.id().catalog();
|
||||
List<String> blackList = new ArrayList<>(Arrays.asList(blackListColumnStr.trim().toUpperCase().replaceAll(catalog + ".", "").split(",")));
|
||||
List<String> allColumns = new ArrayList<>(Arrays.asList(allTableColumns.toUpperCase().split(",")));
|
||||
allColumns.removeAll(blackList);
|
||||
columnsToSelect = String.join(",", allColumns);
|
||||
}
|
||||
return columnsToSelect;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void complete(SnapshotContext snapshotContext) {
|
||||
if (connectorConfig.getPdbName() != null) {
|
||||
|
@ -23,6 +23,7 @@ public OracleSourceInfoStructMaker(String connector, String version, CommonConne
|
||||
.field(SourceInfo.TABLE_NAME_KEY, Schema.STRING_SCHEMA)
|
||||
.field(SourceInfo.TXID_KEY, Schema.OPTIONAL_STRING_SCHEMA)
|
||||
.field(SourceInfo.SCN_KEY, Schema.OPTIONAL_INT64_SCHEMA)
|
||||
.field(SourceInfo.COMMIT_SCN_KEY, Schema.OPTIONAL_INT64_SCHEMA)
|
||||
.field(SourceInfo.LCR_POSITION_KEY, Schema.OPTIONAL_STRING_SCHEMA)
|
||||
.build();
|
||||
}
|
||||
@ -43,6 +44,9 @@ public Struct struct(SourceInfo sourceInfo) {
|
||||
if (sourceInfo.getLcrPosition() != null) {
|
||||
ret.put(SourceInfo.LCR_POSITION_KEY, sourceInfo.getLcrPosition().toString());
|
||||
}
|
||||
if (sourceInfo.getCommitScn() != null) {
|
||||
ret.put(SourceInfo.COMMIT_SCN_KEY, sourceInfo.getCommitScn());
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
@ -18,6 +18,7 @@
|
||||
import org.apache.kafka.connect.data.Field;
|
||||
import org.apache.kafka.connect.data.SchemaBuilder;
|
||||
|
||||
import io.debezium.config.CommonConnectorConfig.BinaryHandlingMode;
|
||||
import io.debezium.data.SpecialValueDecimal;
|
||||
import io.debezium.data.VariableScaleDecimal;
|
||||
import io.debezium.jdbc.JdbcValueConverters;
|
||||
@ -25,6 +26,7 @@
|
||||
import io.debezium.jdbc.TemporalPrecisionMode;
|
||||
import io.debezium.relational.Column;
|
||||
import io.debezium.relational.ValueConverter;
|
||||
import io.debezium.time.Date;
|
||||
import io.debezium.time.MicroDuration;
|
||||
import io.debezium.time.ZonedTimestamp;
|
||||
import io.debezium.util.NumberConversions;
|
||||
@ -33,7 +35,9 @@
|
||||
import oracle.jdbc.OracleTypes;
|
||||
import oracle.sql.BINARY_DOUBLE;
|
||||
import oracle.sql.BINARY_FLOAT;
|
||||
import oracle.sql.BLOB;
|
||||
import oracle.sql.CHAR;
|
||||
import oracle.sql.CLOB;
|
||||
import oracle.sql.DATE;
|
||||
import oracle.sql.INTERVALDS;
|
||||
import oracle.sql.INTERVALYM;
|
||||
@ -78,8 +82,13 @@ public SchemaBuilder schemaBuilder(Column column) {
|
||||
case OracleTypes.INTERVALYM:
|
||||
case OracleTypes.INTERVALDS:
|
||||
return MicroDuration.builder();
|
||||
default:
|
||||
return super.schemaBuilder(column);
|
||||
case Types.STRUCT:
|
||||
return SchemaBuilder.string();
|
||||
default: {
|
||||
SchemaBuilder builder = super.schemaBuilder(column);
|
||||
logger.info("JdbcValueConverters returned '{}' for column '{}'", builder.getClass().getName(), column.name());
|
||||
return builder;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -92,12 +101,17 @@ private SchemaBuilder getNumericSchema(Column column) {
|
||||
if (scale <= 0) {
|
||||
int width = column.length() - scale;
|
||||
|
||||
if (scale == 0 && column.length() == 1) {
|
||||
return SchemaBuilder.bool();
|
||||
}
|
||||
if (width < 3) {
|
||||
return SchemaBuilder.int8();
|
||||
}
|
||||
else if (width < 5) {
|
||||
return SchemaBuilder.int16();
|
||||
}
|
||||
// todo: DBZ-137 this was changed and caused issues with datatype tests, reverted for now.
|
||||
// else if (width < 10 || (width == 10 && scale == 0)) {
|
||||
else if (width < 10) {
|
||||
return SchemaBuilder.int32();
|
||||
}
|
||||
@ -128,7 +142,11 @@ public ValueConverter converter(Column column, Field fieldDefn) {
|
||||
case Types.VARCHAR:
|
||||
case Types.NCHAR:
|
||||
case Types.NVARCHAR:
|
||||
case Types.STRUCT:
|
||||
case Types.CLOB:
|
||||
return data -> convertString(column, fieldDefn, data);
|
||||
case Types.BLOB:
|
||||
return data -> convertBinary(column, fieldDefn, data, binaryMode);
|
||||
case OracleTypes.BINARY_FLOAT:
|
||||
return data -> convertFloat(column, fieldDefn, data);
|
||||
case OracleTypes.BINARY_DOUBLE:
|
||||
@ -136,6 +154,8 @@ public ValueConverter converter(Column column, Field fieldDefn) {
|
||||
case Types.NUMERIC:
|
||||
return getNumericConverter(column, fieldDefn);
|
||||
case Types.FLOAT:
|
||||
// todo: DBZ-137 is there a reason why floats need to be converted to doubles rather than var-scales?
|
||||
// return data -> convertDouble(column, fieldDefn, data);
|
||||
return data -> convertVariableScale(column, fieldDefn, data);
|
||||
case OracleTypes.TIMESTAMPTZ:
|
||||
case OracleTypes.TIMESTAMPLTZ:
|
||||
@ -149,11 +169,23 @@ public ValueConverter converter(Column column, Field fieldDefn) {
|
||||
return super.converter(column, fieldDefn);
|
||||
}
|
||||
|
||||
private Object getFloatConverter(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof BigDecimal) {
|
||||
return ((BigDecimal) data).floatValue();
|
||||
}
|
||||
return convertVariableScale(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
private ValueConverter getNumericConverter(Column column, Field fieldDefn) {
|
||||
if (column.scale().isPresent()) {
|
||||
Integer scale = column.scale().get();
|
||||
|
||||
if (scale <= 0) {
|
||||
// Boolean represtented as Number(1,0)
|
||||
if (scale == 0 && column.length() == 1) {
|
||||
return data -> convertBoolean(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
int width = column.length() - scale;
|
||||
|
||||
if (width < 3) {
|
||||
@ -162,6 +194,8 @@ private ValueConverter getNumericConverter(Column column, Field fieldDefn) {
|
||||
else if (width < 5) {
|
||||
return data -> convertNumericAsSmallInt(column, fieldDefn, data);
|
||||
}
|
||||
// todo: DBZ-137 this was changed and caused issues with datatype tests, reverted for now.
|
||||
// else if (width < 10 || (width == 10 && scale == 0)) {
|
||||
else if (width < 10) {
|
||||
return data -> convertNumericAsInteger(column, fieldDefn, data);
|
||||
}
|
||||
@ -183,10 +217,21 @@ protected Object convertString(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof CHAR) {
|
||||
return ((CHAR) data).stringValue();
|
||||
}
|
||||
if (data instanceof CLOB) {
|
||||
return ((CLOB) data).toString();
|
||||
}
|
||||
|
||||
return super.convertString(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertBinary(Column column, Field fieldDefn, Object data, BinaryHandlingMode mode) {
|
||||
if (data instanceof BLOB) {
|
||||
return ((BLOB) data).getBytes();
|
||||
}
|
||||
return super.convertBinary(column, fieldDefn, data, mode);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertInteger(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof NUMBER) {
|
||||
@ -312,6 +357,24 @@ protected Object convertNumericAsBigInteger(Column column, Field fieldDefn, Obje
|
||||
return super.convertBigInt(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a value object for an expected JDBC type of {@link Types#BOOLEAN}.
|
||||
*
|
||||
* @param column the column definition describing the {@code data} value; never null
|
||||
* @param fieldDefn the field definition; never null
|
||||
* @param data the data object to be converted into a {@link Date Kafka Connect date} type; never null
|
||||
* @return the converted value, or null if the conversion could not be made and the column allows nulls
|
||||
* @throws IllegalArgumentException if the value could not be converted but the column does not allow nulls
|
||||
*/
|
||||
@Override
|
||||
protected Object convertBoolean(Column column, Field fieldDefn, Object data) {
|
||||
|
||||
if (data instanceof BigDecimal) {
|
||||
return ((BigDecimal) data).byteValue() == 0 ? Boolean.FALSE : Boolean.TRUE;
|
||||
}
|
||||
return super.convertBoolean(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertTinyInt(Column column, Field fieldDefn, Object data) {
|
||||
return convertValue(column, fieldDefn, data, BYTE_FALSE, (r) -> {
|
||||
@ -379,6 +442,9 @@ else if (data instanceof TIMESTAMPLTZ) {
|
||||
|
||||
@Override
|
||||
protected Object convertTimestampToEpochMicros(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof Long) {
|
||||
return data;
|
||||
}
|
||||
return super.convertTimestampToEpochMicros(column, fieldDefn, fromOracleTimeClasses(column, data));
|
||||
}
|
||||
|
||||
|
@ -9,6 +9,7 @@
|
||||
|
||||
import io.debezium.annotation.NotThreadSafe;
|
||||
import io.debezium.connector.common.BaseSourceInfo;
|
||||
import io.debezium.connector.oracle.xstream.LcrPosition;
|
||||
import io.debezium.relational.TableId;
|
||||
|
||||
@NotThreadSafe
|
||||
@ -16,10 +17,12 @@ public class SourceInfo extends BaseSourceInfo {
|
||||
|
||||
public static final String TXID_KEY = "txId";
|
||||
public static final String SCN_KEY = "scn";
|
||||
public static final String COMMIT_SCN_KEY = "commit_scn";
|
||||
public static final String LCR_POSITION_KEY = "lcr_position";
|
||||
public static final String SNAPSHOT_KEY = "snapshot";
|
||||
|
||||
private long scn;
|
||||
private Long commitScn;
|
||||
private LcrPosition lcrPosition;
|
||||
private String transactionId;
|
||||
private Instant sourceTime;
|
||||
@ -33,10 +36,18 @@ public long getScn() {
|
||||
return scn;
|
||||
}
|
||||
|
||||
public Long getCommitScn() {
|
||||
return commitScn;
|
||||
}
|
||||
|
||||
public void setScn(long scn) {
|
||||
this.scn = scn;
|
||||
}
|
||||
|
||||
public void setCommitScn(Long commitScn) {
|
||||
this.commitScn = commitScn;
|
||||
}
|
||||
|
||||
public LcrPosition getLcrPosition() {
|
||||
return lcrPosition;
|
||||
}
|
||||
|
@ -0,0 +1,95 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.antlr;
|
||||
|
||||
import org.antlr.v4.runtime.CharStream;
|
||||
import org.antlr.v4.runtime.CommonTokenStream;
|
||||
import org.antlr.v4.runtime.tree.ParseTree;
|
||||
|
||||
import io.debezium.antlr.AntlrDdlParser;
|
||||
import io.debezium.antlr.AntlrDdlParserListener;
|
||||
import io.debezium.antlr.DataTypeResolver;
|
||||
import io.debezium.connector.oracle.antlr.listener.OracleDmlParserListener;
|
||||
import io.debezium.connector.oracle.logminer.OracleChangeRecordValueConverter;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlLexer;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||
import io.debezium.relational.SystemVariables;
|
||||
import io.debezium.relational.Tables;
|
||||
|
||||
/**
|
||||
* This is the main Oracle Antlr DML parser
|
||||
*/
|
||||
public class OracleDmlParser extends AntlrDdlParser<PlSqlLexer, PlSqlParser> {
|
||||
|
||||
private LogMinerDmlEntry dmlEntry;
|
||||
protected String catalogName;
|
||||
protected String schemaName;
|
||||
private OracleChangeRecordValueConverter converter;
|
||||
|
||||
public OracleDmlParser(boolean throwErrorsFromTreeWalk, final String catalogName, final String schemaName, OracleChangeRecordValueConverter converter) {
|
||||
super(throwErrorsFromTreeWalk);
|
||||
this.catalogName = catalogName;
|
||||
this.schemaName = schemaName;
|
||||
this.converter = converter;
|
||||
}
|
||||
|
||||
public LogMinerDmlEntry getDmlEntry() {
|
||||
return dmlEntry;
|
||||
}
|
||||
|
||||
public void setDmlEntry(LogMinerDmlEntry dml) {
|
||||
this.dmlEntry = dml;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void parse(String dmlContent, Tables databaseTables) {
|
||||
if (!dmlContent.endsWith(";")) {
|
||||
dmlContent = dmlContent + ";";
|
||||
}
|
||||
// DML content is case sensitive
|
||||
super.parse(dmlContent, databaseTables);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ParseTree parseTree(PlSqlParser parser) {
|
||||
return parser.unit_statement();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected AntlrDdlParserListener createParseTreeWalkerListener() {
|
||||
return new OracleDmlParserListener(catalogName, schemaName, this);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PlSqlLexer createNewLexerInstance(CharStream charStreams) {
|
||||
return new PlSqlLexer(charStreams);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PlSqlParser createNewParserInstance(CommonTokenStream commonTokenStream) {
|
||||
return new PlSqlParser(commonTokenStream);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean isGrammarInUpperCase() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected DataTypeResolver initializeDataTypeResolver() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SystemVariables createNewSystemVariablesInstance() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public OracleChangeRecordValueConverter getConverters() {
|
||||
return converter;
|
||||
}
|
||||
}
|
@ -6,6 +6,8 @@
|
||||
package io.debezium.connector.oracle.antlr.listener;
|
||||
|
||||
import static io.debezium.antlr.AntlrDdlParser.getText;
|
||||
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getColumnName;
|
||||
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getTableName;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.antlr.listener;
|
||||
|
||||
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getTableName;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||
import io.debezium.connector.oracle.logminer.OracleChangeRecordValueConverter;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueImpl;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParserBaseListener;
|
||||
import io.debezium.relational.Column;
|
||||
import io.debezium.relational.Table;
|
||||
import io.debezium.text.ParsingException;
|
||||
|
||||
/**
|
||||
* This class contains common methods for DML parser listeners
|
||||
*/
|
||||
abstract class BaseDmlParserListener<T> extends PlSqlParserBaseListener {
|
||||
|
||||
protected String catalogName;
|
||||
protected String schemaName;
|
||||
protected Table table;
|
||||
final OracleChangeRecordValueConverter converter;
|
||||
String alias;
|
||||
|
||||
protected OracleDmlParser parser;
|
||||
|
||||
Map<T, LogMinerColumnValueWrapper> newColumnValues = new LinkedHashMap<>();
|
||||
Map<T, LogMinerColumnValueWrapper> oldColumnValues = new LinkedHashMap<>();
|
||||
|
||||
BaseDmlParserListener(String catalogName, String schemaName, OracleDmlParser parser) {
|
||||
this.parser = parser;
|
||||
this.catalogName = catalogName;
|
||||
this.schemaName = schemaName;
|
||||
this.converter = parser.getConverters();
|
||||
}
|
||||
|
||||
// Defines the key of the Map of LogMinerColumnValueWrapper. It could be String or Integer
|
||||
abstract protected T getKey(Column column, int index);
|
||||
|
||||
/**
|
||||
* This method prepares all column value placeholders, based on the table metadata
|
||||
* @param ctx DML table expression context
|
||||
*/
|
||||
void init(PlSqlParser.Dml_table_expression_clauseContext ctx) {
|
||||
String tableName = getTableName(ctx.tableview_name());
|
||||
table = parser.databaseTables().forTable(catalogName, schemaName, tableName);
|
||||
if (table == null) {
|
||||
throw new ParsingException(null, "Trying to parse a table, which does not exist.");
|
||||
}
|
||||
for (int i = 0; i < table.columns().size(); i++) {
|
||||
Column column = table.columns().get(i);
|
||||
int type = column.jdbcType();
|
||||
T key = getKey(column, i);
|
||||
String name = ParserUtils.stripeQuotes(column.name().toUpperCase());
|
||||
newColumnValues.put(key, new LogMinerColumnValueWrapper(new LogMinerColumnValueImpl(name, type)));
|
||||
oldColumnValues.put(key, new LogMinerColumnValueWrapper(new LogMinerColumnValueImpl(name, type)));
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.antlr.listener;
|
||||
|
||||
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||
import io.debezium.relational.Column;
|
||||
|
||||
/**
|
||||
* This class parses recursively logical expression tree for DELETE and UPDATE statements
|
||||
*/
|
||||
abstract class BaseDmlStringParserListener extends BaseDmlParserListener<String> {
|
||||
|
||||
boolean isUpdate;
|
||||
|
||||
BaseDmlStringParserListener(String catalogName, String schemaName, OracleDmlParser parser) {
|
||||
super(catalogName, schemaName, parser);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void enterTable_alias(PlSqlParser.Table_aliasContext ctx) {
|
||||
alias = ctx.getText().toUpperCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Logical expressions are trees and (column name, value) pairs are nested in this tree.
|
||||
* This methods extracts those pairs and store them in List<LogMinerColumnValue> oldValues
|
||||
* This method is used by VALUES parsers of update and delete statements.
|
||||
* @param logicalExpression expression tree
|
||||
*/
|
||||
void parseRecursively(PlSqlParser.Logical_expressionContext logicalExpression) {
|
||||
|
||||
int count = logicalExpression.logical_expression().size();
|
||||
if (count == 0) {
|
||||
|
||||
String nullValue = logicalExpression.getStop().getText();
|
||||
|
||||
String expression = logicalExpression.getText();
|
||||
String columnName = "";
|
||||
String value = "";
|
||||
if (expression.contains("=")) {
|
||||
columnName = expression.substring(0, expression.indexOf("=")).toUpperCase();
|
||||
value = expression.substring(expression.indexOf("=") + 1);
|
||||
}
|
||||
if ("null".equalsIgnoreCase(nullValue)) {
|
||||
columnName = expression.substring(0, expression.toUpperCase().indexOf("ISNULL")).toUpperCase();
|
||||
value = nullValue;
|
||||
}
|
||||
|
||||
columnName = ParserUtils.stripeAlias(columnName, alias);
|
||||
columnName = ParserUtils.stripeQuotes(columnName);
|
||||
|
||||
Column column = table.columnWithName(columnName);
|
||||
Object stripedValue = ParserUtils.removeApostrophes(value);
|
||||
|
||||
LogMinerColumnValueWrapper logMinerColumnValueWrapper = oldColumnValues.get(columnName);
|
||||
if (logMinerColumnValueWrapper != null) { // todo this used to happen for ROWID pseudo column. Test if this is not a problem after NO_ROWID_IN_STMT option
|
||||
Object valueObject = ParserUtils.convertValueToSchemaType(column, stripedValue, converter);
|
||||
logMinerColumnValueWrapper.setProcessed(true);
|
||||
logMinerColumnValueWrapper.getColumnValue().setColumnData(valueObject);
|
||||
}
|
||||
|
||||
}
|
||||
for (int i = 0; i < count; i++) {
|
||||
parseRecursively(logicalExpression.logical_expression(i));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
@ -57,11 +57,19 @@ public void enterPrimary_key_clause(PlSqlParser.Primary_key_clauseContext ctx) {
|
||||
|
||||
// todo use dataTypeResolver instead
|
||||
private void resolveColumnDataType(PlSqlParser.Column_definitionContext ctx) {
|
||||
PlSqlParser.Precision_partContext precisionPart = ctx.datatype().precision_part();
|
||||
|
||||
columnEditor.name(getColumnName(ctx.column_name()));
|
||||
|
||||
if (ctx.datatype().native_datatype_element() != null) {
|
||||
PlSqlParser.Precision_partContext precisionPart = null;
|
||||
if (ctx.datatype() != null) {
|
||||
precisionPart = ctx.datatype().precision_part();
|
||||
}
|
||||
|
||||
if (ctx.datatype() == null) {
|
||||
if (ctx.type_name() != null && "\"MDSYS\".\"SDO_GEOMETRY\"".equalsIgnoreCase(ctx.type_name().getText())) {
|
||||
columnEditor.jdbcType(Types.STRUCT).type("MDSYS.SDO_GEOMETRY");
|
||||
}
|
||||
}
|
||||
else if (ctx.datatype().native_datatype_element() != null) {
|
||||
if (ctx.datatype().native_datatype_element().INT() != null
|
||||
|| ctx.datatype().native_datatype_element().INTEGER() != null
|
||||
|| ctx.datatype().native_datatype_element().SMALLINT() != null
|
||||
|
@ -5,6 +5,9 @@
|
||||
*/
|
||||
package io.debezium.connector.oracle.antlr.listener;
|
||||
|
||||
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getColumnName;
|
||||
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getTableName;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@ -49,19 +52,21 @@ public void enterCreate_table(PlSqlParser.Create_tableContext ctx) {
|
||||
public void exitCreate_table(PlSqlParser.Create_tableContext ctx) {
|
||||
Table table = getTable();
|
||||
assert table != null;
|
||||
|
||||
parser.runIfNotNull(() -> {
|
||||
listeners.remove(columnDefinitionParserListener);
|
||||
columnDefinitionParserListener = null;
|
||||
parser.databaseTables().overwriteTable(table);
|
||||
// parser.signalCreateTable(tableEditor.tableId(), ctx); todo ?
|
||||
}, tableEditor, table);
|
||||
|
||||
super.exitCreate_table(ctx);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void enterColumn_definition(PlSqlParser.Column_definitionContext ctx) {
|
||||
parser.runIfNotNull(() -> {
|
||||
String columnName = getColumnName(ctx.column_name());
|
||||
String columnName = ParserUtils.stripeQuotes(getColumnName(ctx.column_name()));
|
||||
ColumnEditor columnEditor = Column.editor().name(columnName);
|
||||
if (columnDefinitionParserListener == null) {
|
||||
columnDefinitionParserListener = new ColumnDefinitionParserListener(tableEditor, columnEditor, parser.dataTypeResolver());
|
||||
@ -87,7 +92,7 @@ public void exitColumn_definition(PlSqlParser.Column_definitionContext ctx) {
|
||||
public void exitOut_of_line_constraint(PlSqlParser.Out_of_line_constraintContext ctx) {
|
||||
if (ctx.PRIMARY() != null) {
|
||||
List<String> pkColumnNames = ctx.column_name().stream()
|
||||
.map(this::getColumnName)
|
||||
.map(ParserUtils::getColumnName)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
tableEditor.setPrimaryKeyNames(pkColumnNames);
|
||||
|
@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.antlr.listener;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntryImpl;
|
||||
import io.debezium.data.Envelope;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||
import io.debezium.relational.Column;
|
||||
|
||||
/**
|
||||
* This class parses delete statements.
|
||||
* LogMiner instruments all the values in WHERE cause regardless of original statement.
|
||||
* In other words if the original statement is
|
||||
* delete from debezium where col1 = 2 and if there are 2 records to delete,
|
||||
* LogMiner will contain following two statements:
|
||||
*
|
||||
* delete from "DEBEZIUM" where "ID" = 6 and "COL1" = 2 and "COL2" = 'text' and "COL3" = 'text' and "COL4" IS NULL and "COL5" IS NULL and "COL6" IS NULL and "COL7" IS NULL and "COL8" IS NULL
|
||||
* delete from "DEBEZIUM" where "ID" = 7 and "COL1" = 2 and "COL2" = 'text' and "COL3" = 'text' and "COL4" IS NULL and "COL5" IS NULL and "COL6" IS NULL and "COL7" IS NULL and "COL8" IS NULL
|
||||
*
|
||||
*/
|
||||
public class DeleteParserListener extends BaseDmlStringParserListener {
|
||||
|
||||
DeleteParserListener(final String catalogName, final String schemaName, final OracleDmlParser parser) {
|
||||
super(catalogName, schemaName, parser);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getKey(Column column, int index) {
|
||||
return column.name();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void enterDelete_statement(PlSqlParser.Delete_statementContext ctx) {
|
||||
init(ctx.general_table_ref().dml_table_expression_clause());
|
||||
newColumnValues.clear();
|
||||
PlSqlParser.Table_aliasContext tableAlias = ctx.general_table_ref().table_alias();
|
||||
alias = tableAlias == null ? "" : tableAlias.getText().toUpperCase();
|
||||
PlSqlParser.Where_clauseContext where = ctx.where_clause();
|
||||
if (where != null) {
|
||||
parseRecursively(ctx.where_clause().expression().logical_expression());
|
||||
}
|
||||
else {
|
||||
oldColumnValues.clear();
|
||||
}
|
||||
super.enterDelete_statement(ctx);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exitDelete_statement(PlSqlParser.Delete_statementContext ctx) {
|
||||
List<LogMinerColumnValue> actualOldValues = oldColumnValues.values()
|
||||
.stream().map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||
LogMinerDmlEntry newRecord = new LogMinerDmlEntryImpl(Envelope.Operation.DELETE, Collections.emptyList(), actualOldValues);
|
||||
parser.setDmlEntry(newRecord);
|
||||
super.exitDelete_statement(ctx);
|
||||
}
|
||||
}
|
@ -5,14 +5,18 @@
|
||||
*/
|
||||
package io.debezium.connector.oracle.antlr.listener;
|
||||
|
||||
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getTableName;
|
||||
|
||||
import io.debezium.connector.oracle.antlr.OracleDdlParser;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParserBaseListener;
|
||||
import io.debezium.relational.TableId;
|
||||
|
||||
/**
|
||||
* This class is parsing Oracle drop table statements.
|
||||
*/
|
||||
public class DropTableParserListener extends BaseParserListener {
|
||||
public class DropTableParserListener extends PlSqlParserBaseListener {
|
||||
|
||||
private String catalogName;
|
||||
private String schemaName;
|
||||
private OracleDdlParser parser;
|
||||
|
@ -0,0 +1,87 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.antlr.listener;
|
||||
|
||||
import static io.debezium.antlr.AntlrDdlParser.getText;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntryImpl;
|
||||
import io.debezium.data.Envelope;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||
import io.debezium.relational.Column;
|
||||
import io.debezium.text.ParsingException;
|
||||
|
||||
/**
|
||||
* This class parses Oracle INSERT statements.
|
||||
* if the original tested query was: insert into DEBEZIUM (id,col3) values (2, 'some text')
|
||||
*
|
||||
* LogMiner will supply:
|
||||
*
|
||||
* insert into "DEBEZIUM"("ID","COL1","COL2","COL3","COL4","COL5","COL6","COL7","COL8","COL9","COL10")
|
||||
* values (2,NULL,'debezium','some text',NULL,NULL,NULL,NULL,NULL,EMPTY_BLOB(),EMPTY_CLOB())
|
||||
* update "DEBEZIUM" set "COL9" = NULL, "COL10" = NULL where "ID" = 2 and "COL1" IS NULL and "COL2" = 'debezium'
|
||||
* and "COL3" = 'some text' and "COL4" IS NULL and "COL5" IS NULL and "COL6" IS NULL
|
||||
* and "COL7" IS NULL and "COL8" IS NULL
|
||||
*
|
||||
*/
|
||||
public class InsertParserListener extends BaseDmlParserListener<Integer> {
|
||||
|
||||
InsertParserListener(String catalogName, String schemaName, OracleDmlParser parser) {
|
||||
super(catalogName, schemaName, parser);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Integer getKey(Column column, int index) {
|
||||
return index;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void enterInsert_statement(PlSqlParser.Insert_statementContext ctx) {
|
||||
init(ctx.single_table_insert().insert_into_clause().general_table_ref().dml_table_expression_clause());
|
||||
oldColumnValues.clear();
|
||||
super.enterInsert_statement(ctx);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void enterValues_clause(PlSqlParser.Values_clauseContext ctx) {
|
||||
if (table == null) {
|
||||
throw new ParsingException(null, "Trying to parse a statement for a table which does not exist. " +
|
||||
"Statement: " + getText(ctx));
|
||||
}
|
||||
|
||||
List<PlSqlParser.ExpressionContext> values = ctx.expressions().expression();
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
PlSqlParser.ExpressionContext value = values.get(i);
|
||||
LogMinerColumnValueWrapper columnObject = newColumnValues.get(i);
|
||||
|
||||
String columnName = columnObject.getColumnValue().getColumnName();
|
||||
Column column = table.columnWithName(columnName);
|
||||
|
||||
String valueText = value.logical_expression().getText();
|
||||
valueText = ParserUtils.removeApostrophes(valueText);
|
||||
Object valueObject = ParserUtils.convertValueToSchemaType(column, valueText, converter);
|
||||
|
||||
columnObject.getColumnValue().setColumnData(valueObject);
|
||||
}
|
||||
super.enterValues_clause(ctx);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exitSingle_table_insert(PlSqlParser.Single_table_insertContext ctx) {
|
||||
List<LogMinerColumnValue> actualNewValues = newColumnValues.values()
|
||||
.stream().map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||
LogMinerDmlEntry newRecord = new LogMinerDmlEntryImpl(Envelope.Operation.CREATE, actualNewValues, Collections.emptyList());
|
||||
parser.setDmlEntry(newRecord);
|
||||
super.exitSingle_table_insert(ctx);
|
||||
}
|
||||
}
|
@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.antlr.listener;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
import org.antlr.v4.runtime.ParserRuleContext;
|
||||
import org.antlr.v4.runtime.tree.ParseTreeListener;
|
||||
|
||||
import io.debezium.antlr.AntlrDdlParserListener;
|
||||
import io.debezium.antlr.ProxyParseTreeListenerUtil;
|
||||
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParserBaseListener;
|
||||
import io.debezium.text.ParsingException;
|
||||
|
||||
/**
|
||||
* This class is Oracle main DML parser listener class.
|
||||
* It instantiates supported listeners, walks listeners through every parsing rule and collects parsing exceptions.
|
||||
*
|
||||
*/
|
||||
public class OracleDmlParserListener extends PlSqlParserBaseListener implements AntlrDdlParserListener {
|
||||
|
||||
private final List<ParseTreeListener> listeners = new CopyOnWriteArrayList<>();
|
||||
private final Collection<ParsingException> errors = new ArrayList<>();
|
||||
|
||||
public OracleDmlParserListener(final String catalogName, final String schemaName,
|
||||
final OracleDmlParser parser) {
|
||||
listeners.add(new InsertParserListener(catalogName, schemaName, parser));
|
||||
listeners.add(new UpdateParserListener(catalogName, schemaName, parser));
|
||||
listeners.add(new DeleteParserListener(catalogName, schemaName, parser));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<ParsingException> getErrors() {
|
||||
return errors;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void enterEveryRule(ParserRuleContext ctx) {
|
||||
ProxyParseTreeListenerUtil.delegateEnterRule(ctx, listeners, errors);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exitEveryRule(ParserRuleContext ctx) {
|
||||
ProxyParseTreeListenerUtil.delegateExitRule(ctx, listeners, errors);
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,149 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.antlr.listener;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.kafka.connect.data.Field;
|
||||
import org.apache.kafka.connect.data.Schema;
|
||||
import org.apache.kafka.connect.data.SchemaBuilder;
|
||||
|
||||
import io.debezium.connector.oracle.logminer.OracleChangeRecordValueConverter;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||
import io.debezium.relational.Column;
|
||||
import io.debezium.relational.Table;
|
||||
import io.debezium.relational.ValueConverter;
|
||||
|
||||
/**
|
||||
* This class contains a few methods for parser listeners
|
||||
*/
|
||||
public class ParserUtils {
|
||||
|
||||
private ParserUtils() {
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns pure column name without quotes
|
||||
* @param ctx column name context
|
||||
* @return column name
|
||||
*/
|
||||
public static String getColumnName(final PlSqlParser.Column_nameContext ctx) {
|
||||
return stripeQuotes(ctx.identifier().id_expression().getText());
|
||||
}
|
||||
|
||||
/**
|
||||
* stripes double quotes that surrounds a string
|
||||
* @param text text
|
||||
* @return text without surrounding double quotes
|
||||
*/
|
||||
public static String stripeQuotes(String text) {
|
||||
if (text != null && text.indexOf("\"") == 0 && text.lastIndexOf("\"") == text.length() - 1) {
|
||||
return text.substring(1, text.length() - 1);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
/**
|
||||
* this method stripes table alias and dot give string
|
||||
* @param text string with possible alias
|
||||
* @param alias table alias
|
||||
* @return striped string
|
||||
*/
|
||||
public static String stripeAlias(String text, String alias) {
|
||||
int index = text.indexOf(alias + ".");
|
||||
if (index >= 0) {
|
||||
return text.substring(alias.length() + 1);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize new column values with old column values.
|
||||
* It does not override new values which were processed already in where clause parsing
|
||||
* @param newColumnValues values to set or insert
|
||||
* @param oldColumnValues values in WHERE clause
|
||||
* @param table Debezium Table object
|
||||
*/
|
||||
public static void cloneOldToNewColumnValues(Map<String, LogMinerColumnValueWrapper> newColumnValues, Map<String, LogMinerColumnValueWrapper> oldColumnValues,
|
||||
Table table) {
|
||||
for (Column column : table.columns()) {
|
||||
final LogMinerColumnValueWrapper newColumnValue = newColumnValues.get(column.name());
|
||||
if (!newColumnValue.isProcessed()) {
|
||||
final LogMinerColumnValueWrapper oldColumnValue = oldColumnValues.get(column.name());
|
||||
newColumnValue.setProcessed(true);
|
||||
newColumnValue.getColumnValue().setColumnData(oldColumnValue.getColumnValue().getColumnData());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This converts the given value to the appropriate object. The conversion is based on the column definition
|
||||
*
|
||||
* @param column column Object
|
||||
* @param value value object
|
||||
* @param converters given converter
|
||||
* @return object as the result of this conversion. It could be null if converter cannot build the schema
|
||||
* or if converter or value are null
|
||||
*/
|
||||
public static Object convertValueToSchemaType(Column column, Object value, OracleChangeRecordValueConverter converters) {
|
||||
if (converters != null && value != null) {
|
||||
final SchemaBuilder schemaBuilder = converters.schemaBuilder(column);
|
||||
if (schemaBuilder == null) {
|
||||
return null;
|
||||
}
|
||||
final Schema schema = schemaBuilder.build();
|
||||
final Field field = new Field(column.name(), 1, schema);
|
||||
final ValueConverter valueConverter = converters.converter(column, field);
|
||||
|
||||
return valueConverter.convert(value);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* In some cases values of the parsed expression are enclosed in apostrophes.
|
||||
* Even null values are surrounded by single apostrophes. This method removes them.
|
||||
*
|
||||
* @param text supplied value which might be enclosed by apostrophes.
|
||||
* @return clean String or null in case if test = "null" or = "NULL"
|
||||
*/
|
||||
public static String removeApostrophes(String text) {
|
||||
if (text != null && text.indexOf("'") == 0 && text.lastIndexOf("'") == text.length() - 1) {
|
||||
return text.substring(1, text.length() - 1);
|
||||
}
|
||||
if ("null".equalsIgnoreCase(text)) {
|
||||
return null;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
/**
|
||||
* this is to handle cases when a record contains escape character(s)
|
||||
* @param text before parsing we replaced it with double escape, now revert it back
|
||||
* @return string with double slashes
|
||||
*/
|
||||
public static String replaceDoubleBackSlashes(String text) {
|
||||
if (text != null && text.contains("\\\\")) {
|
||||
return text.replaceAll("\\\\\\\\", "\\\\");
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Obtains the table name
|
||||
* @param tableview_name table view context
|
||||
* @return table name
|
||||
*/
|
||||
static String getTableName(final PlSqlParser.Tableview_nameContext tableview_name) {
|
||||
if (tableview_name.id_expression() != null) {
|
||||
return stripeQuotes(tableview_name.id_expression().getText());
|
||||
}
|
||||
else {
|
||||
return stripeQuotes(tableview_name.identifier().id_expression().getText());
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.antlr.listener;
|
||||
|
||||
import static io.debezium.antlr.AntlrDdlParser.getText;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntryImpl;
|
||||
import io.debezium.data.Envelope;
|
||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||
import io.debezium.relational.Column;
|
||||
import io.debezium.text.ParsingException;
|
||||
|
||||
/**
|
||||
* This class parses UPDATE statements.
|
||||
* For the original query:
|
||||
* update debezium set test = '7' where test1 = '6' (let's assume we have 3 records with such value)
|
||||
*
|
||||
* logMiner with supply:
|
||||
*
|
||||
* update "debezium" set "TEST" = '7' where "DUMMY" = '1' and "TEST" = '2' and "TEST1" = '6' and "TEST2" = '1'
|
||||
* update "debezium" set "TEST" = '7' where "DUMMY" = '2' and "TEST" = '2' and "TEST1" = '6' and "TEST2" = '1'
|
||||
* update "debezium" set "TEST" = '7' where "DUMMY" = '3' and "TEST" = '2' and "TEST1" = '6' and "TEST2" = '1'
|
||||
*
|
||||
*/
|
||||
public class UpdateParserListener extends BaseDmlStringParserListener {
|
||||
|
||||
UpdateParserListener(String catalogName, String schemaName, OracleDmlParser parser) {
|
||||
super(catalogName, schemaName, parser);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getKey(Column column, int index) {
|
||||
return column.name();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void enterUpdate_statement(PlSqlParser.Update_statementContext ctx) {
|
||||
init(ctx.general_table_ref().dml_table_expression_clause());
|
||||
isUpdate = true;
|
||||
super.enterUpdate_statement(ctx);
|
||||
}
|
||||
|
||||
/**
|
||||
* this method could be invoked by delete, insert or update statements, but we should act on update only
|
||||
* @param ctx where clause context
|
||||
*/
|
||||
@Override
|
||||
public void enterWhere_clause(PlSqlParser.Where_clauseContext ctx) {
|
||||
if (isUpdate) {
|
||||
parseRecursively(ctx.expression().logical_expression());
|
||||
ParserUtils.cloneOldToNewColumnValues(newColumnValues, oldColumnValues, table);
|
||||
}
|
||||
isUpdate = false;
|
||||
super.enterWhere_clause(ctx);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void enterColumn_based_update_set_clause(PlSqlParser.Column_based_update_set_clauseContext ctx) {
|
||||
if (table == null) {
|
||||
throw new ParsingException(null, "Trying to parse a statement for a table which does not exist. " +
|
||||
"Statement: " + getText(ctx));
|
||||
}
|
||||
String columnName = ctx.column_name().getText().toUpperCase();
|
||||
String stripedName = ParserUtils.stripeAlias(columnName, alias);
|
||||
stripedName = ParserUtils.stripeQuotes(stripedName);
|
||||
String value = ctx.getText().substring(columnName.length() + 1);
|
||||
String nullValue = ctx.expression().getStop().getText();
|
||||
if ("null".equalsIgnoreCase(nullValue)) {
|
||||
value = nullValue;
|
||||
}
|
||||
Object stripedValue = ParserUtils.removeApostrophes(value);
|
||||
|
||||
Column column = table.columnWithName(stripedName);
|
||||
Object valueObject = ParserUtils.convertValueToSchemaType(column, stripedValue, converter);
|
||||
|
||||
LogMinerColumnValueWrapper logMinerColumnValueWrapper = newColumnValues.get(stripedName);
|
||||
logMinerColumnValueWrapper.setProcessed(true);
|
||||
logMinerColumnValueWrapper.getColumnValue().setColumnData(valueObject);
|
||||
|
||||
super.enterColumn_based_update_set_clause(ctx);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exitUpdate_statement(PlSqlParser.Update_statementContext ctx) {
|
||||
List<LogMinerColumnValue> actualNewValues = newColumnValues.values().stream()
|
||||
.filter(LogMinerColumnValueWrapper::isProcessed).map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||
List<LogMinerColumnValue> actualOldValues = oldColumnValues.values().stream()
|
||||
.filter(LogMinerColumnValueWrapper::isProcessed).map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||
LogMinerDmlEntry newRecord = new LogMinerDmlEntryImpl(Envelope.Operation.UPDATE, actualNewValues, actualOldValues);
|
||||
parser.setDmlEntry(newRecord);
|
||||
super.exitUpdate_statement(ctx);
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,287 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.jsqlparser;
|
||||
|
||||
import java.io.StringReader;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.connector.oracle.antlr.listener.ParserUtils;
|
||||
import io.debezium.connector.oracle.logminer.OracleChangeRecordValueConverter;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueImpl;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntryImpl;
|
||||
import io.debezium.data.Envelope;
|
||||
import io.debezium.relational.Column;
|
||||
import io.debezium.relational.Table;
|
||||
import io.debezium.relational.TableId;
|
||||
import io.debezium.relational.Tables;
|
||||
import io.debezium.text.ParsingException;
|
||||
|
||||
import net.sf.jsqlparser.JSQLParserException;
|
||||
import net.sf.jsqlparser.expression.Alias;
|
||||
import net.sf.jsqlparser.expression.Expression;
|
||||
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
|
||||
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
|
||||
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
|
||||
import net.sf.jsqlparser.expression.operators.relational.IsNullExpression;
|
||||
import net.sf.jsqlparser.expression.operators.relational.ItemsList;
|
||||
import net.sf.jsqlparser.expression.operators.relational.ItemsListVisitorAdapter;
|
||||
import net.sf.jsqlparser.parser.CCJSqlParserManager;
|
||||
import net.sf.jsqlparser.statement.Statement;
|
||||
import net.sf.jsqlparser.statement.delete.Delete;
|
||||
import net.sf.jsqlparser.statement.insert.Insert;
|
||||
import net.sf.jsqlparser.statement.update.Update;
|
||||
|
||||
/**
|
||||
* This class does parsing of simple DML: insert, update, delete.
|
||||
* Log Miner supplies very simple syntax , that this parser should be sufficient to parse those.
|
||||
* It does no support joins, merge, sub-selects and other complicated cases, which should be OK for Log Miner case
|
||||
*/
|
||||
public class SimpleDmlParser {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SimpleDmlParser.class);
|
||||
protected String catalogName;
|
||||
protected String schemaName;
|
||||
protected Table table;
|
||||
private final OracleChangeRecordValueConverter converter;
|
||||
private String aliasName;
|
||||
private Map<String, LogMinerColumnValueWrapper> newColumnValues = new LinkedHashMap<>();
|
||||
private Map<String, LogMinerColumnValueWrapper> oldColumnValues = new LinkedHashMap<>();
|
||||
private CCJSqlParserManager pm;
|
||||
|
||||
/**
|
||||
* Constructor
|
||||
* @param catalogName database name
|
||||
* @param schemaName user name
|
||||
* @param converter value converter
|
||||
*/
|
||||
public SimpleDmlParser(String catalogName, String schemaName, OracleChangeRecordValueConverter converter) {
|
||||
this.catalogName = catalogName;
|
||||
this.schemaName = schemaName;
|
||||
this.converter = converter;
|
||||
pm = new CCJSqlParserManager();
|
||||
}
|
||||
|
||||
/**
|
||||
* This parses a DML
|
||||
* @param dmlContent DML
|
||||
* @param tables debezium Tables
|
||||
* @return parsed value holder class
|
||||
*/
|
||||
public LogMinerDmlEntry parse(String dmlContent, Tables tables, String txId) {
|
||||
try {
|
||||
|
||||
// If a table contains Spatial data type, DML input generates two entries in REDO LOG.
|
||||
// First with actual statement and second with NULL. It is not relevant at this point
|
||||
if (dmlContent == null) {
|
||||
LOGGER.debug("Cannot parse NULL , transaction: {}", txId);
|
||||
return null;
|
||||
}
|
||||
// todo investigate: happens on CTAS
|
||||
if (dmlContent.endsWith(";null;")) {
|
||||
dmlContent = dmlContent.substring(0, dmlContent.lastIndexOf(";null;"));
|
||||
}
|
||||
if (!dmlContent.endsWith(";")) {
|
||||
dmlContent = dmlContent + ";";
|
||||
}
|
||||
// this is to handle cases when a record contains escape character(s). This parser throws.
|
||||
dmlContent = dmlContent.replaceAll("\\\\", "\\\\\\\\");
|
||||
dmlContent = dmlContent.replaceAll("= Unsupported Type", "= null"); // todo address spatial data types
|
||||
|
||||
newColumnValues.clear();
|
||||
oldColumnValues.clear();
|
||||
|
||||
Statement st = pm.parse(new StringReader(dmlContent));
|
||||
if (st instanceof Update) {
|
||||
parseUpdate(tables, (Update) st);
|
||||
List<LogMinerColumnValue> actualNewValues = newColumnValues.values().stream()
|
||||
.filter(LogMinerColumnValueWrapper::isProcessed).map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||
List<LogMinerColumnValue> actualOldValues = oldColumnValues.values().stream()
|
||||
.filter(LogMinerColumnValueWrapper::isProcessed).map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||
return new LogMinerDmlEntryImpl(Envelope.Operation.UPDATE, actualNewValues, actualOldValues);
|
||||
|
||||
}
|
||||
else if (st instanceof Insert) {
|
||||
parseInsert(tables, (Insert) st);
|
||||
List<LogMinerColumnValue> actualNewValues = newColumnValues.values()
|
||||
.stream().map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||
return new LogMinerDmlEntryImpl(Envelope.Operation.CREATE, actualNewValues, Collections.emptyList());
|
||||
|
||||
}
|
||||
else if (st instanceof Delete) {
|
||||
parseDelete(tables, (Delete) st);
|
||||
List<LogMinerColumnValue> actualOldValues = oldColumnValues.values()
|
||||
.stream().map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||
return new LogMinerDmlEntryImpl(Envelope.Operation.DELETE, Collections.emptyList(), actualOldValues);
|
||||
|
||||
}
|
||||
else {
|
||||
LOGGER.error("Operation {} is not supported yet", st);
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
catch (Throwable e) {
|
||||
LOGGER.error("Cannot parse statement : {}, transaction: {}, due to the {}", dmlContent, txId, e);
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void initColumns(Tables tables, String tableName) {
|
||||
table = tables.forTable(catalogName, schemaName, tableName);
|
||||
if (table == null) {
|
||||
TableId id = new TableId(catalogName, schemaName, tableName);
|
||||
throw new ParsingException(null, "Trying to parse a table '" + id + "', which does not exist.");
|
||||
}
|
||||
for (int i = 0; i < table.columns().size(); i++) {
|
||||
Column column = table.columns().get(i);
|
||||
int type = column.jdbcType();
|
||||
String key = column.name();
|
||||
String name = ParserUtils.stripeQuotes(column.name().toUpperCase());
|
||||
newColumnValues.put(key, new LogMinerColumnValueWrapper(new LogMinerColumnValueImpl(name, type)));
|
||||
oldColumnValues.put(key, new LogMinerColumnValueWrapper(new LogMinerColumnValueImpl(name, type)));
|
||||
}
|
||||
}
|
||||
|
||||
// this parses simple statement with only one table
|
||||
private void parseUpdate(Tables tables, Update st) throws JSQLParserException {
|
||||
int tableCount = st.getTables().size();
|
||||
if (tableCount > 1 || tableCount == 0) {
|
||||
throw new JSQLParserException("DML includes " + tableCount + " tables");
|
||||
}
|
||||
net.sf.jsqlparser.schema.Table parseTable = st.getTables().get(0);
|
||||
initColumns(tables, ParserUtils.stripeQuotes(parseTable.getName()));
|
||||
|
||||
List<net.sf.jsqlparser.schema.Column> columns = st.getColumns();
|
||||
Alias alias = parseTable.getAlias();
|
||||
aliasName = alias == null ? "" : alias.getName().trim();
|
||||
|
||||
List<Expression> expressions = st.getExpressions(); // new values
|
||||
setNewValues(expressions, columns);
|
||||
Expression where = st.getWhere(); // old values
|
||||
if (where != null) {
|
||||
parseWhereClause(where);
|
||||
ParserUtils.cloneOldToNewColumnValues(newColumnValues, oldColumnValues, table);
|
||||
}
|
||||
else {
|
||||
oldColumnValues.clear();
|
||||
}
|
||||
}
|
||||
|
||||
private void parseInsert(Tables tables, Insert st) {
|
||||
initColumns(tables, ParserUtils.stripeQuotes(st.getTable().getName()));
|
||||
Alias alias = st.getTable().getAlias();
|
||||
aliasName = alias == null ? "" : alias.getName().trim();
|
||||
|
||||
List<net.sf.jsqlparser.schema.Column> columns = st.getColumns();
|
||||
ItemsList values = st.getItemsList();
|
||||
values.accept(new ItemsListVisitorAdapter() {
|
||||
@Override
|
||||
public void visit(ExpressionList expressionList) {
|
||||
super.visit(expressionList);
|
||||
List<Expression> expressions = expressionList.getExpressions();
|
||||
setNewValues(expressions, columns);
|
||||
}
|
||||
});
|
||||
oldColumnValues.clear();
|
||||
}
|
||||
|
||||
private void parseDelete(Tables tables, Delete st) {
|
||||
initColumns(tables, ParserUtils.stripeQuotes(st.getTable().getName()));
|
||||
Alias alias = st.getTable().getAlias();
|
||||
aliasName = alias == null ? "" : alias.getName().trim();
|
||||
|
||||
newColumnValues.clear();
|
||||
|
||||
Expression where = st.getWhere();
|
||||
if (where != null) {
|
||||
parseWhereClause(where);
|
||||
}
|
||||
else {
|
||||
oldColumnValues.clear();
|
||||
}
|
||||
}
|
||||
|
||||
private void setNewValues(List<Expression> expressions, List<net.sf.jsqlparser.schema.Column> columns) {
|
||||
if (expressions.size() != columns.size()) {
|
||||
throw new RuntimeException("DML has " + expressions.size() + " column values, but Table object has " + columns.size() + " columns");
|
||||
}
|
||||
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
String columnName = ParserUtils.stripeQuotes(columns.get(i).getColumnName().toUpperCase());
|
||||
String value = ParserUtils.stripeQuotes(expressions.get(i).toString());
|
||||
Object stripedValue = ParserUtils.removeApostrophes(value);
|
||||
Column column = table.columnWithName(columnName);
|
||||
if (column == null) {
|
||||
LOGGER.trace("blacklisted column: {}", columnName);
|
||||
continue;
|
||||
}
|
||||
Object valueObject = ParserUtils.convertValueToSchemaType(column, stripedValue, converter);
|
||||
|
||||
LogMinerColumnValueWrapper logMinerColumnValueWrapper = newColumnValues.get(columnName);
|
||||
if (logMinerColumnValueWrapper != null) {
|
||||
logMinerColumnValueWrapper.setProcessed(true);
|
||||
logMinerColumnValueWrapper.getColumnValue().setColumnData(valueObject);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void parseWhereClause(Expression logicalExpression) {
|
||||
|
||||
logicalExpression.accept(new ExpressionVisitorAdapter() {
|
||||
@Override
|
||||
public void visit(EqualsTo expr) {
|
||||
super.visit(expr);
|
||||
String columnName = expr.getLeftExpression().toString();
|
||||
columnName = ParserUtils.stripeAlias(columnName, aliasName);
|
||||
String value = expr.getRightExpression().toString();
|
||||
columnName = ParserUtils.stripeQuotes(columnName);
|
||||
|
||||
Column column = table.columnWithName(columnName);
|
||||
if (column == null) {
|
||||
LOGGER.trace("blacklisted column in where clause: {}", columnName);
|
||||
return;
|
||||
}
|
||||
value = ParserUtils.removeApostrophes(value);
|
||||
|
||||
LogMinerColumnValueWrapper logMinerColumnValueWrapper = oldColumnValues.get(columnName.toUpperCase());
|
||||
if (logMinerColumnValueWrapper != null) {
|
||||
Object valueObject = ParserUtils.convertValueToSchemaType(column, value, converter);
|
||||
logMinerColumnValueWrapper.setProcessed(true);
|
||||
logMinerColumnValueWrapper.getColumnValue().setColumnData(valueObject);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void visit(IsNullExpression expr) {
|
||||
super.visit(expr);
|
||||
String columnName = expr.getLeftExpression().toString();
|
||||
columnName = ParserUtils.stripeAlias(columnName, aliasName);
|
||||
columnName = ParserUtils.stripeQuotes(columnName);
|
||||
Column column = table.columnWithName(columnName);
|
||||
if (column == null) {
|
||||
LOGGER.trace("blacklisted column in where clause: {}", columnName);
|
||||
return;
|
||||
}
|
||||
LogMinerColumnValueWrapper logMinerColumnValueWrapper = oldColumnValues.get(columnName.toUpperCase());
|
||||
if (logMinerColumnValueWrapper != null) {
|
||||
logMinerColumnValueWrapper.setProcessed(true);
|
||||
logMinerColumnValueWrapper.getColumnValue().setColumnData(null);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import io.debezium.connector.oracle.BaseChangeRecordEmitter;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||
import io.debezium.data.Envelope.Operation;
|
||||
import io.debezium.pipeline.spi.OffsetContext;
|
||||
import io.debezium.relational.Table;
|
||||
import io.debezium.util.Clock;
|
||||
|
||||
/**
|
||||
* Emits change record based on a single {@link LogMinerDmlEntry} event.
|
||||
*/
|
||||
public class LogMinerChangeRecordEmitter extends BaseChangeRecordEmitter<LogMinerColumnValue> {
|
||||
|
||||
private LogMinerDmlEntry dmlEntry;
|
||||
protected final Table table;
|
||||
|
||||
public LogMinerChangeRecordEmitter(OffsetContext offset, LogMinerDmlEntry dmlEntry, Table table, Clock clock) {
|
||||
super(offset, table, clock);
|
||||
this.dmlEntry = dmlEntry;
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Operation getOperation() {
|
||||
return dmlEntry.getCommandType();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object[] getOldColumnValues() {
|
||||
List<LogMinerColumnValue> valueList = dmlEntry.getOldValues();
|
||||
LogMinerColumnValue[] result = Arrays.copyOf(valueList.toArray(), valueList.size(), LogMinerColumnValue[].class);
|
||||
return getColumnValues(result);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object[] getNewColumnValues() {
|
||||
List<LogMinerColumnValue> valueList = dmlEntry.getNewValues();
|
||||
LogMinerColumnValue[] result = Arrays.copyOf(valueList.toArray(), valueList.size(), LogMinerColumnValue[].class);
|
||||
return getColumnValues(result);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getColumnName(LogMinerColumnValue columnValue) {
|
||||
return columnValue.getColumnName();
|
||||
}
|
||||
|
||||
protected Object getColumnData(LogMinerColumnValue columnValue) {
|
||||
return columnValue.getColumnData();
|
||||
}
|
||||
}
|
@ -0,0 +1,431 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.CallableStatement;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.connector.oracle.OracleConnection;
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||
import io.debezium.jdbc.JdbcConnection;
|
||||
|
||||
/**
|
||||
* This class contains methods to configure and manage Log Miner utility
|
||||
*/
|
||||
public class LogMinerHelper {
|
||||
|
||||
private final static String UNKNOWN = "unknown";
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogMinerHelper.class);
|
||||
|
||||
private enum DATATYPE {
|
||||
LONG,
|
||||
TIMESTAMP,
|
||||
STRING
|
||||
}
|
||||
|
||||
/**
|
||||
* This builds data dictionary objects in redo log files.
|
||||
* During this build, Oracle does an additional REDO LOG switch.
|
||||
* This call may take time, which leads to delay in delivering incremental changes.
|
||||
* With this option the lag between source database and dispatching event fluctuates.
|
||||
*
|
||||
* @param connection connection to the database as log miner user (connection to the container)
|
||||
* @throws SQLException any exception
|
||||
*/
|
||||
static void buildDataDictionary(Connection connection) throws SQLException {
|
||||
executeCallableStatement(connection, SqlUtils.BUILD_DICTIONARY);
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns current SCN from the database
|
||||
*
|
||||
* @param connection container level database connection
|
||||
* @return current SCN
|
||||
* @throws SQLException if anything unexpected happens
|
||||
*/
|
||||
public static long getCurrentScn(Connection connection) throws SQLException {
|
||||
try (Statement statement = connection.createStatement();
|
||||
ResultSet rs = statement.executeQuery(SqlUtils.CURRENT_SCN)) {
|
||||
|
||||
if (!rs.next()) {
|
||||
throw new IllegalStateException("Couldn't get SCN");
|
||||
}
|
||||
|
||||
long currentScn = rs.getLong(1);
|
||||
rs.close();
|
||||
return currentScn;
|
||||
}
|
||||
}
|
||||
|
||||
static void createAuditTable(Connection connection) throws SQLException {
|
||||
String tableExists = (String) getSingleResult(connection, SqlUtils.AUDIT_TABLE_EXISTS, DATATYPE.STRING);
|
||||
if (tableExists == null) {
|
||||
executeCallableStatement(connection, SqlUtils.CREATE_AUDIT_TABLE);
|
||||
}
|
||||
|
||||
String recordExists = (String) getSingleResult(connection, SqlUtils.AUDIT_TABLE_RECORD_EXISTS, DATATYPE.STRING);
|
||||
if (recordExists == null) {
|
||||
executeCallableStatement(connection, SqlUtils.INSERT_AUDIT_TABLE);
|
||||
connection.commit();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns next SCN for mining and also updates MBean metrics
|
||||
* We use a configurable limit, because the larger mining range, the slower query from Log Miner content view.
|
||||
* In addition capturing unlimited number of changes can blow up Java heap.
|
||||
* Gradual querying helps to catch up faster after long delays in mining.
|
||||
*
|
||||
* @param connection container level database connection
|
||||
* @param metrics MBean accessible metrics
|
||||
* @param startScn start SCN
|
||||
* @return next SCN to mine to
|
||||
* @throws SQLException if anything unexpected happens
|
||||
*/
|
||||
static long getEndScn(Connection connection, long startScn, LogMinerMetrics metrics) throws SQLException {
|
||||
long currentScn = getCurrentScn(connection);
|
||||
metrics.setCurrentScn(currentScn);
|
||||
int miningDiapason = metrics.getBatchSize();
|
||||
|
||||
// it is critical to flush LogWriter buffer
|
||||
executeCallableStatement(connection, SqlUtils.UPDATE_AUDIT_TABLE + currentScn);
|
||||
if (!connection.getAutoCommit()) {
|
||||
connection.commit();
|
||||
}
|
||||
|
||||
// adjust sleeping time to optimize DB impact and catchup faster when behind
|
||||
boolean isNextScnCloseToDbCurrent = currentScn < (startScn + miningDiapason);
|
||||
metrics.changeSleepingTime(isNextScnCloseToDbCurrent);
|
||||
|
||||
return isNextScnCloseToDbCurrent ? currentScn : startScn + miningDiapason;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate time difference between database and connector timers. It could be negative if DB time is ahead.
|
||||
* @param connection connection
|
||||
* @return difference in milliseconds
|
||||
*/
|
||||
static long getTimeDifference(Connection connection) throws SQLException {
|
||||
Timestamp dbCurrentMillis = (Timestamp) getSingleResult(connection, SqlUtils.CURRENT_TIMESTAMP, DATATYPE.TIMESTAMP);
|
||||
if (dbCurrentMillis == null) {
|
||||
return 0;
|
||||
}
|
||||
Instant fromDb = dbCurrentMillis.toInstant();
|
||||
Instant now = Instant.now();
|
||||
return Duration.between(fromDb, now).toMillis();
|
||||
}
|
||||
|
||||
/**
|
||||
* This method builds mining view to query changes from.
|
||||
* This view is built for online redo log files.
|
||||
* It starts log mining session.
|
||||
* It uses data dictionary objects, incorporated in previous steps.
|
||||
* It tracks DDL changes and mines committed data only.
|
||||
*
|
||||
* @param connection container level database connection
|
||||
* @param startScn the SCN to mine from
|
||||
* @param endScn the SCN to mine to
|
||||
* @param strategy this is about dictionary location
|
||||
* @param isContinuousMining works < 19 version only
|
||||
* @throws SQLException if anything unexpected happens
|
||||
*/
|
||||
static void startOnlineMining(Connection connection, Long startScn, Long endScn,
|
||||
OracleConnectorConfig.LogMiningStrategy strategy, boolean isContinuousMining)
|
||||
throws SQLException {
|
||||
String statement = SqlUtils.getStartLogMinerStatement(startScn, endScn, strategy, isContinuousMining);
|
||||
executeCallableStatement(connection, statement);
|
||||
// todo dbms_logmnr.STRING_LITERALS_IN_STMT?
|
||||
// todo If the log file is corrupted/bad, logmnr will not be able to access it, we have to switch to another one?
|
||||
}
|
||||
|
||||
/**
|
||||
* This method query the database to get CURRENT online redo log file(s). Multiple is applicable for RAC systems.
|
||||
* @param connection connection to reuse
|
||||
* @param metrics MBean accessible metrics
|
||||
* @return full redo log file name(s), including path
|
||||
* @throws SQLException if anything unexpected happens
|
||||
*/
|
||||
static Set<String> getCurrentRedoLogFiles(Connection connection, LogMinerMetrics metrics) throws SQLException {
|
||||
String checkQuery = SqlUtils.CURRENT_REDO_LOG_NAME;
|
||||
|
||||
Set<String> fileNames = new HashSet<>();
|
||||
PreparedStatement st = connection.prepareStatement(checkQuery);
|
||||
ResultSet result = st.executeQuery();
|
||||
while (result.next()) {
|
||||
fileNames.add(result.getString(1));
|
||||
LOGGER.trace(" Current Redo log fileName: {} ", fileNames);
|
||||
}
|
||||
st.close();
|
||||
result.close();
|
||||
|
||||
updateRedoLogMetrics(connection, metrics, fileNames);
|
||||
return fileNames;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method fetches the oldest SCN from online redo log files
|
||||
*
|
||||
* @param connection container level database connection
|
||||
* @return oldest SCN from online redo log
|
||||
* @throws SQLException if anything unexpected happens
|
||||
*/
|
||||
static long getFirstOnlineLogScn(Connection connection) throws SQLException {
|
||||
LOGGER.trace("getting first scn of all online logs");
|
||||
Statement s = connection.createStatement();
|
||||
ResultSet res = s.executeQuery(SqlUtils.OLDEST_FIRST_CHANGE);
|
||||
res.next();
|
||||
long firstScnOfOnlineLog = res.getLong(1);
|
||||
res.close();
|
||||
return firstScnOfOnlineLog;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets NLS parameters for mining session.
|
||||
*
|
||||
* @param connection session level database connection
|
||||
* @throws SQLException if anything unexpected happens
|
||||
*/
|
||||
static void setNlsSessionParameters(JdbcConnection connection) throws SQLException {
|
||||
connection.executeWithoutCommitting(SqlUtils.NLS_SESSION_PARAMETERS);
|
||||
}
|
||||
|
||||
/**
|
||||
* This is to update MBean metrics associated with REDO LOG groups
|
||||
* @param connection connection
|
||||
* @param fileNames name of current REDO LOG files
|
||||
* @param metrics current metrics
|
||||
*/
|
||||
private static void updateRedoLogMetrics(Connection connection, LogMinerMetrics metrics, Set<String> fileNames) {
|
||||
try {
|
||||
// update metrics
|
||||
Map<String, String> logStatuses = getRedoLogStatus(connection);
|
||||
metrics.setRedoLogStatus(logStatuses);
|
||||
|
||||
int counter = getSwitchCount(connection);
|
||||
metrics.setSwitchCount(counter);
|
||||
metrics.setCurrentLogFileName(fileNames);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
LOGGER.error("Cannot update metrics");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This fetches online redo log statuses
|
||||
* @param connection privileged connection
|
||||
* @return REDO LOG statuses Map, where key is the REDO name and value is the status
|
||||
* @throws SQLException if anything unexpected happens
|
||||
*/
|
||||
private static Map<String, String> getRedoLogStatus(Connection connection) throws SQLException {
|
||||
return getMap(connection, SqlUtils.REDO_LOGS_STATUS, UNKNOWN);
|
||||
}
|
||||
|
||||
/**
|
||||
* This fetches REDO LOG switch count for the last day
|
||||
* @param connection privileged connection
|
||||
* @return counter
|
||||
*/
|
||||
private static int getSwitchCount(Connection connection) {
|
||||
try {
|
||||
Map<String, String> total = getMap(connection, SqlUtils.SWITCH_HISTORY_TOTAL_COUNT, UNKNOWN);
|
||||
if (total != null && total.get("total") != null) {
|
||||
return Integer.parseInt(total.get("total"));
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
LOGGER.error("Cannot get switch counter due to the {}", e);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method checks if supplemental logging was set on the database level. This is critical check, cannot work if not.
|
||||
* @param connection oracle connection on logminer level
|
||||
* @param pdbName pdb name
|
||||
* @throws SQLException if anything unexpected happens
|
||||
*/
|
||||
static void checkSupplementalLogging(OracleConnection connection, String pdbName) throws SQLException {
|
||||
try {
|
||||
if (pdbName != null) {
|
||||
connection.setSessionToPdb(pdbName);
|
||||
}
|
||||
|
||||
final String key = "KEY";
|
||||
String validateGlobalLogging = "SELECT '" + key + "', " + " SUPPLEMENTAL_LOG_DATA_ALL from V$DATABASE";
|
||||
Map<String, String> globalLogging = getMap(connection.connection(false), validateGlobalLogging, UNKNOWN);
|
||||
if ("no".equalsIgnoreCase(globalLogging.get(key))) {
|
||||
throw new RuntimeException("Supplemental logging was not set. Use command: ALTER DATABASE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if (pdbName != null) {
|
||||
connection.resetSessionToCdb();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This call completes log miner session.
|
||||
* Complete gracefully.
|
||||
*
|
||||
* @param connection container level database connection
|
||||
*/
|
||||
static void endMining(Connection connection) {
|
||||
String stopMining = SqlUtils.END_LOGMNR;
|
||||
try {
|
||||
executeCallableStatement(connection, stopMining);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
if (e.getMessage().toUpperCase().contains("ORA-01307")) {
|
||||
LOGGER.info("Log Miner session was already closed");
|
||||
}
|
||||
else {
|
||||
LOGGER.error("Cannot close Log Miner session gracefully: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method substitutes CONTINUOUS_MINE functionality for online files only
|
||||
* @param connection connection
|
||||
* @param lastProcessedScn current offset
|
||||
* @throws SQLException if anything unexpected happens
|
||||
*/
|
||||
static void setRedoLogFilesForMining(Connection connection, Long lastProcessedScn) throws SQLException {
|
||||
|
||||
Map<String, Long> logFilesForMining = getLogFilesForOffsetScn(connection, lastProcessedScn);
|
||||
if (logFilesForMining.isEmpty()) {
|
||||
throw new IllegalStateException("The online log files do not contain offset SCN: " + lastProcessedScn + ", re-snapshot is required.");
|
||||
}
|
||||
|
||||
// The following check seems to be problematic and removal of it does not appear to cause
|
||||
// any detrimental impact to the connector's ability to stream changes. This was left as
|
||||
// is but commented in case we find a reason for it to exist.
|
||||
// int redoLogGroupSize = getRedoLogGroupSize(connection);
|
||||
// if (logFilesForMining.size() == redoLogGroupSize) {
|
||||
// throw new IllegalStateException("All online log files needed for mining the offset SCN: " + lastProcessedScn + ", re-snapshot is required.");
|
||||
// }
|
||||
|
||||
List<String> logFilesNamesForMining = logFilesForMining.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList());
|
||||
|
||||
for (String file : logFilesNamesForMining) {
|
||||
String addLogFileStatement = SqlUtils.getAddLogFileStatement("DBMS_LOGMNR.ADDFILE", file);
|
||||
executeCallableStatement(connection, addLogFileStatement);
|
||||
LOGGER.trace("add log file to the mining session = {}", file);
|
||||
}
|
||||
|
||||
LOGGER.debug("Last mined SCN: {}, Log file list to mine: {}\n", lastProcessedScn, logFilesForMining);
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns SCN as a watermark to abandon long lasting transactions.
|
||||
*
|
||||
* @param connection connection
|
||||
* @param offsetScn current offset
|
||||
* @return Optional last SCN in a redo log
|
||||
* @throws SQLException if anything unexpected happens
|
||||
*/
|
||||
static Optional<Long> getLastScnFromTheOldestOnlineRedo(Connection connection, Long offsetScn) throws SQLException {
|
||||
|
||||
Map<String, String> allOnlineRedoLogFiles = getMap(connection, SqlUtils.ALL_ONLINE_LOGS, "-1");
|
||||
Map<String, Long> logFilesToMine = getLogFilesForOffsetScn(connection, offsetScn);
|
||||
|
||||
LOGGER.debug("Redo log size = {}, needed for mining files size = {}", allOnlineRedoLogFiles.size(), logFilesToMine.size());
|
||||
if (allOnlineRedoLogFiles.size() - logFilesToMine.size() <= 1) {
|
||||
List<Long> lastScnsInRedoLogToMine = logFilesToMine.entrySet().stream().map(Map.Entry::getValue).collect(Collectors.toList());
|
||||
return lastScnsInRedoLogToMine.stream().min(Long::compareTo);
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
static void logWarn(TransactionalBufferMetrics metrics, String format, Object... args) {
|
||||
LOGGER.warn(format, args);
|
||||
metrics.incrementWarningCounter();
|
||||
}
|
||||
|
||||
static void logError(TransactionalBufferMetrics metrics, String format, Object... args) {
|
||||
LOGGER.error(format, args);
|
||||
metrics.incrementErrorCounter();
|
||||
}
|
||||
|
||||
/**
|
||||
* get size of online REDO groups
|
||||
* @param connection connection
|
||||
* @return size
|
||||
*/
|
||||
private static int getRedoLogGroupSize(Connection connection) throws SQLException {
|
||||
return getMap(connection, SqlUtils.ALL_ONLINE_LOGS, "-1").size();
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns all online log files, starting from one which contains offset SCN and ending with one containing largest SCN
|
||||
* 18446744073709551615 on Ora 19c is the max value of the nextScn in the current redo todo replace all Long with BigDecimal for SCN
|
||||
*/
|
||||
private static Map<String, Long> getLogFilesForOffsetScn(Connection connection, Long offsetScn) throws SQLException {
|
||||
Map<String, String> redoLogFiles = getMap(connection, SqlUtils.ALL_ONLINE_LOGS, "-1");
|
||||
return redoLogFiles.entrySet().stream()
|
||||
.filter(entry -> new BigDecimal(entry.getValue()).longValue() > offsetScn || new BigDecimal(entry.getValue()).longValue() == -1).collect(Collectors
|
||||
.toMap(Map.Entry::getKey, e -> new BigDecimal(e.getValue()).longValue() == -1 ? Long.MAX_VALUE : new BigDecimal(e.getValue()).longValue()));
|
||||
}
|
||||
|
||||
private static void executeCallableStatement(Connection connection, String statement) throws SQLException {
|
||||
Objects.requireNonNull(statement);
|
||||
CallableStatement s;
|
||||
s = connection.prepareCall(statement);
|
||||
s.execute();
|
||||
s.close();
|
||||
}
|
||||
|
||||
private static Map<String, String> getMap(Connection connection, String query, String nullReplacement) throws SQLException {
|
||||
Map<String, String> result = new LinkedHashMap<>();
|
||||
try (
|
||||
PreparedStatement statement = connection.prepareStatement(query);
|
||||
ResultSet rs = statement.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
String value = rs.getString(2);
|
||||
value = value == null ? nullReplacement : value;
|
||||
result.put(rs.getString(1), value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
private static Object getSingleResult(Connection connection, String query, DATATYPE type) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(query);
|
||||
ResultSet rs = statement.executeQuery()) {
|
||||
if (rs.next()) {
|
||||
switch (type) {
|
||||
case LONG:
|
||||
return rs.getLong(1);
|
||||
case TIMESTAMP:
|
||||
return rs.getTimestamp(1);
|
||||
case STRING:
|
||||
return rs.getString(1);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,218 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Arrays;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import io.debezium.annotation.ThreadSafe;
|
||||
import io.debezium.connector.common.CdcSourceTaskContext;
|
||||
import io.debezium.metrics.Metrics;
|
||||
|
||||
/**
|
||||
* This class contains methods to be exposed via MBean server
|
||||
*
|
||||
*/
|
||||
@ThreadSafe
|
||||
public class LogMinerMetrics extends Metrics implements LogMinerMetricsMXBean {
|
||||
private AtomicLong currentScn = new AtomicLong();
|
||||
private AtomicInteger capturedDmlCount = new AtomicInteger();
|
||||
private AtomicReference<String[]> currentLogFileName;
|
||||
private AtomicReference<String[]> redoLogStatus;
|
||||
private AtomicInteger switchCounter = new AtomicInteger();
|
||||
private AtomicReference<Duration> lastLogMinerQueryDuration = new AtomicReference<>();
|
||||
private AtomicReference<Duration> averageLogMinerQueryDuration = new AtomicReference<>();
|
||||
private AtomicInteger logMinerQueryCount = new AtomicInteger();
|
||||
private AtomicReference<Duration> lastProcessedCapturedBatchDuration = new AtomicReference<>();
|
||||
private AtomicInteger processedCapturedBatchCount = new AtomicInteger();
|
||||
private AtomicReference<Duration> averageProcessedCapturedBatchDuration = new AtomicReference<>();
|
||||
private AtomicInteger batchSize = new AtomicInteger();
|
||||
private AtomicInteger millisecondToSleepBetweenMiningQuery = new AtomicInteger();
|
||||
|
||||
private final int MAX_SLEEP_TIME = 3_000;
|
||||
private final int DEFAULT_SLEEP_TIME = 1_000;
|
||||
private final int MIN_SLEEP_TIME = 100;
|
||||
|
||||
private final int MIN_BATCH_SIZE = 1_000;
|
||||
private final int MAX_BATCH_SIZE = 100_000;
|
||||
private final int DEFAULT_BATCH_SIZE = 5_000;
|
||||
|
||||
private final int SLEEP_TIME_INCREMENT = 200;
|
||||
|
||||
LogMinerMetrics(CdcSourceTaskContext taskContext) {
|
||||
super(taskContext, "log-miner");
|
||||
|
||||
batchSize.set(DEFAULT_BATCH_SIZE);
|
||||
millisecondToSleepBetweenMiningQuery.set(DEFAULT_SLEEP_TIME);
|
||||
|
||||
currentScn.set(-1);
|
||||
capturedDmlCount.set(0);
|
||||
currentLogFileName = new AtomicReference<>();
|
||||
redoLogStatus = new AtomicReference<>();
|
||||
switchCounter.set(0);
|
||||
averageLogMinerQueryDuration.set(Duration.ZERO);
|
||||
lastLogMinerQueryDuration.set(Duration.ZERO);
|
||||
logMinerQueryCount.set(0);
|
||||
lastProcessedCapturedBatchDuration.set(Duration.ZERO);
|
||||
processedCapturedBatchCount.set(0);
|
||||
averageProcessedCapturedBatchDuration.set(Duration.ZERO);
|
||||
}
|
||||
|
||||
// setters
|
||||
public void setCurrentScn(Long scn) {
|
||||
currentScn.set(scn);
|
||||
}
|
||||
|
||||
public void incrementCapturedDmlCount() {
|
||||
capturedDmlCount.incrementAndGet();
|
||||
}
|
||||
|
||||
public void setCurrentLogFileName(Set<String> names) {
|
||||
currentLogFileName.set(names.stream().toArray(String[]::new));
|
||||
}
|
||||
|
||||
public void setRedoLogStatus(Map<String, String> status) {
|
||||
String[] statusArray = status.entrySet().stream().map(e -> e.getKey() + " | " + e.getValue()).toArray(String[]::new);
|
||||
redoLogStatus.set(statusArray);
|
||||
}
|
||||
|
||||
public void setSwitchCount(int counter) {
|
||||
switchCounter.set(counter);
|
||||
}
|
||||
|
||||
public void setLastLogMinerQueryDuration(Duration fetchDuration) {
|
||||
setDurationMetrics(fetchDuration, lastLogMinerQueryDuration, logMinerQueryCount, averageLogMinerQueryDuration);
|
||||
}
|
||||
|
||||
public void setProcessedCapturedBatchDuration(Duration processDuration) {
|
||||
setDurationMetrics(processDuration, lastProcessedCapturedBatchDuration, processedCapturedBatchCount, averageProcessedCapturedBatchDuration);
|
||||
}
|
||||
|
||||
// implemented getters
|
||||
@Override
|
||||
public Long getCurrentScn() {
|
||||
return currentScn.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getCapturedDmlCount() {
|
||||
return capturedDmlCount.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] getCurrentRedoLogFileName() {
|
||||
return currentLogFileName.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] getRedoLogStatus() {
|
||||
return redoLogStatus.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getSwitchCounter() {
|
||||
return switchCounter.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getLastLogMinerQueryDuration() {
|
||||
return lastLogMinerQueryDuration.get() == null ? 0 : lastLogMinerQueryDuration.get().toMillis();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getAverageLogMinerQueryDuration() {
|
||||
return averageLogMinerQueryDuration.get() == null ? 0 : averageLogMinerQueryDuration.get().toMillis();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getLastProcessedCapturedBatchDuration() {
|
||||
return lastProcessedCapturedBatchDuration.get() == null ? 0 : lastProcessedCapturedBatchDuration.get().toMillis();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getLogMinerQueryCount() {
|
||||
return logMinerQueryCount.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getProcessedCapturedBatchCount() {
|
||||
return processedCapturedBatchCount.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getAverageProcessedCapturedBatchDuration() {
|
||||
return averageProcessedCapturedBatchDuration.get() == null ? 0 : averageProcessedCapturedBatchDuration.get().toMillis();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getBatchSize() {
|
||||
return batchSize.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer getMillisecondToSleepBetweenMiningQuery() {
|
||||
return millisecondToSleepBetweenMiningQuery.get();
|
||||
}
|
||||
|
||||
// MBean accessible setters
|
||||
@Override
|
||||
public void setBatchSize(int size) {
|
||||
if (size >= MIN_BATCH_SIZE && size <= MAX_BATCH_SIZE) {
|
||||
batchSize.set(size);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setMillisecondToSleepBetweenMiningQuery(Integer milliseconds) {
|
||||
if (milliseconds != null && milliseconds >= MIN_SLEEP_TIME && milliseconds < MAX_SLEEP_TIME) {
|
||||
millisecondToSleepBetweenMiningQuery.set(milliseconds);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void changeSleepingTime(boolean increment) {
|
||||
int sleepTime = millisecondToSleepBetweenMiningQuery.get();
|
||||
int change = increment ? SLEEP_TIME_INCREMENT : -SLEEP_TIME_INCREMENT;
|
||||
if (sleepTime >= MIN_SLEEP_TIME && sleepTime < MAX_SLEEP_TIME) {
|
||||
millisecondToSleepBetweenMiningQuery.getAndAdd(change);
|
||||
}
|
||||
}
|
||||
|
||||
// private methods
|
||||
private void setDurationMetrics(Duration duration, AtomicReference<Duration> lastDuration, AtomicInteger counter,
|
||||
AtomicReference<Duration> averageDuration) {
|
||||
if (duration != null) {
|
||||
lastDuration.set(duration);
|
||||
int count = counter.incrementAndGet();
|
||||
Duration currentAverage = averageDuration.get() == null ? Duration.ZERO : averageDuration.get();
|
||||
averageDuration.set(currentAverage.multipliedBy(count - 1).plus(duration).dividedBy(count));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LogMinerMetrics{" +
|
||||
"currentEndScn=" + currentScn.get() +
|
||||
", currentLogFileNames=" + Arrays.toString(currentLogFileName.get()) +
|
||||
", redoLogStatus=" + Arrays.toString(redoLogStatus.get()) +
|
||||
", capturedDmlCount=" + capturedDmlCount.get() +
|
||||
", switchCounter=" + switchCounter.get() +
|
||||
", lastLogMinerQueryDuration=" + lastLogMinerQueryDuration.get() +
|
||||
", logMinerQueryCount=" + logMinerQueryCount.get() +
|
||||
", averageLogMinerQueryDuration=" + averageLogMinerQueryDuration.get() +
|
||||
", lastProcessedCapturedBatchDuration=" + lastProcessedCapturedBatchDuration.get() +
|
||||
", processedCapturedBatchCount=" + processedCapturedBatchCount.get() +
|
||||
", averageProcessedCapturedBatchDuration=" + averageProcessedCapturedBatchDuration.get() +
|
||||
", millisecondToSleepBetweenMiningQuery=" + millisecondToSleepBetweenMiningQuery.get() +
|
||||
", batchSize=" + batchSize.get() +
|
||||
'}';
|
||||
}
|
||||
}
|
@ -0,0 +1,107 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
/**
|
||||
* This interface is exposed for JMX
|
||||
*/
|
||||
public interface LogMinerMetricsMXBean {
|
||||
|
||||
/**
|
||||
* Exposes current SCN in the database. This is very efficient query and will not affect overall performance
|
||||
*
|
||||
* @return current SCN
|
||||
*/
|
||||
Long getCurrentScn();
|
||||
|
||||
/**
|
||||
* Exposes current redo log file. This is very efficient query and will not affect overall performance
|
||||
*
|
||||
* @return full path or NULL if an exception occurs.
|
||||
*/
|
||||
String[] getCurrentRedoLogFileName();
|
||||
|
||||
/**
|
||||
* Exposes states of redo logs: current, active, inactive, unused ...
|
||||
* @return array of: (redo log name | status) elements
|
||||
*/
|
||||
String[] getRedoLogStatus();
|
||||
|
||||
/**
|
||||
* fetches counter of redo switches for the last day.
|
||||
* If this number is high , like once in 3 minutes, the troubleshooting on the database level is required.
|
||||
* @return counter
|
||||
*/
|
||||
int getSwitchCounter();
|
||||
|
||||
/**
|
||||
* @return number of milliseconds last Log Miner query took
|
||||
*/
|
||||
Long getLastLogMinerQueryDuration();
|
||||
|
||||
/**
|
||||
* @return number of captured DML since the connector is up
|
||||
*/
|
||||
int getCapturedDmlCount();
|
||||
|
||||
/**
|
||||
* @return number of Log Miner view queries since the connector is up
|
||||
*/
|
||||
int getLogMinerQueryCount();
|
||||
|
||||
/**
|
||||
* @return average duration of Log Miner view query
|
||||
*/
|
||||
Long getAverageLogMinerQueryDuration();
|
||||
|
||||
/**
|
||||
* Log Miner view query returns number of captured DML , Commit and Rollback. This is what we call a batch.
|
||||
* @return duration of the last batch processing, which includes parsing and dispatching
|
||||
*/
|
||||
Long getLastProcessedCapturedBatchDuration();
|
||||
|
||||
/**
|
||||
* Log Miner view query returns number of captured DML , Commit and Rollback. This is what we call a batch.
|
||||
* @return number of all processed batches , which includes parsing and dispatching
|
||||
*/
|
||||
int getProcessedCapturedBatchCount();
|
||||
|
||||
/**
|
||||
* @return average time of processing captured batch from Log Miner view
|
||||
*/
|
||||
Long getAverageProcessedCapturedBatchDuration();
|
||||
|
||||
/**
|
||||
* Maximum number of entries in Log Miner view to fetch. This is used to set the diapason of the SCN in mining query.
|
||||
* If difference between "start SCN" and "end SCN" to mine exceeds this limit, end SCN will be set to "start SCN" + batchSize
|
||||
* @return the limit
|
||||
*/
|
||||
int getBatchSize();
|
||||
|
||||
/**
|
||||
* this gives ability to manipulate number of entries in Log Miner view to fetch.
|
||||
* It has limits to prevent abnormal values
|
||||
* @param size limit
|
||||
*/
|
||||
void setBatchSize(int size);
|
||||
|
||||
/**
|
||||
* @return number of milliseconds for connector to sleep before fetching another batch from the Log Miner view
|
||||
*/
|
||||
Integer getMillisecondToSleepBetweenMiningQuery();
|
||||
|
||||
/**
|
||||
* sets number of milliseconds for connector to sleep before fetching another batch from the Log Miner view
|
||||
* @param milliseconds to sleep
|
||||
*/
|
||||
void setMillisecondToSleepBetweenMiningQuery(Integer milliseconds);
|
||||
|
||||
/**
|
||||
* change sleeping time
|
||||
* @param increment true to add, false to deduct
|
||||
*/
|
||||
void changeSleepingTime(boolean increment);
|
||||
}
|
@ -0,0 +1,245 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.connector.oracle.OracleDatabaseSchema;
|
||||
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||
import io.debezium.connector.oracle.jsqlparser.SimpleDmlParser;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||
import io.debezium.data.Envelope;
|
||||
import io.debezium.pipeline.EventDispatcher;
|
||||
import io.debezium.pipeline.source.spi.ChangeEventSource;
|
||||
import io.debezium.relational.Table;
|
||||
import io.debezium.relational.TableId;
|
||||
import io.debezium.util.Clock;
|
||||
|
||||
/**
|
||||
* This class process entries obtained from LogMiner view.
|
||||
* It parses each entry.
|
||||
* On each DML it registers a callback in TransactionalBuffer.
|
||||
* On rollback it removes registered entries from TransactionalBuffer.
|
||||
* On commit it executes all registered callbacks, which dispatch ChangeRecords.
|
||||
* This also calculates metrics
|
||||
*/
|
||||
class LogMinerQueryResultProcessor {
|
||||
|
||||
private final ChangeEventSource.ChangeEventSourceContext context;
|
||||
private final LogMinerMetrics metrics;
|
||||
private final TransactionalBuffer transactionalBuffer;
|
||||
private final SimpleDmlParser dmlParser;
|
||||
private final OracleOffsetContext offsetContext;
|
||||
private final OracleDatabaseSchema schema;
|
||||
private final EventDispatcher<TableId> dispatcher;
|
||||
private final TransactionalBufferMetrics transactionalBufferMetrics;
|
||||
private final String catalogName;
|
||||
private final Clock clock;
|
||||
private final Logger LOGGER = LoggerFactory.getLogger(LogMinerQueryResultProcessor.class);
|
||||
private long currentOffsetScn = 0;
|
||||
private long currentOffsetCommitScn = 0;
|
||||
private long stuckScnCounter = 0;
|
||||
|
||||
LogMinerQueryResultProcessor(ChangeEventSource.ChangeEventSourceContext context, LogMinerMetrics metrics,
|
||||
TransactionalBuffer transactionalBuffer, SimpleDmlParser dmlParser,
|
||||
OracleOffsetContext offsetContext, OracleDatabaseSchema schema,
|
||||
EventDispatcher<TableId> dispatcher, TransactionalBufferMetrics transactionalBufferMetrics,
|
||||
String catalogName, Clock clock) {
|
||||
this.context = context;
|
||||
this.metrics = metrics;
|
||||
this.transactionalBuffer = transactionalBuffer;
|
||||
this.dmlParser = dmlParser;
|
||||
this.offsetContext = offsetContext;
|
||||
this.schema = schema;
|
||||
this.dispatcher = dispatcher;
|
||||
this.transactionalBufferMetrics = transactionalBufferMetrics;
|
||||
this.catalogName = catalogName;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method does all the job
|
||||
* @param resultSet the info from Log Miner view
|
||||
* @return number of processed DMLs from the given resultSet
|
||||
*/
|
||||
int processResult(ResultSet resultSet) {
|
||||
int dmlCounter = 0;
|
||||
int commitCounter = 0;
|
||||
int rollbackCounter = 0;
|
||||
Duration cumulativeCommitTime = Duration.ZERO;
|
||||
Duration cumulativeParseTime = Duration.ZERO;
|
||||
Duration cumulativeOtherTime = Duration.ZERO;
|
||||
Instant startTime = Instant.now();
|
||||
while (true) {
|
||||
try {
|
||||
if (!resultSet.next()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
LogMinerHelper.logError(transactionalBufferMetrics, "Closed resultSet");
|
||||
return 0;
|
||||
}
|
||||
|
||||
Instant iterationStart = Instant.now();
|
||||
|
||||
BigDecimal scn = RowMapper.getScn(transactionalBufferMetrics, resultSet);
|
||||
String redo_sql = RowMapper.getSqlRedo(transactionalBufferMetrics, resultSet);
|
||||
String tableName = RowMapper.getTableName(transactionalBufferMetrics, resultSet);
|
||||
String segOwner = RowMapper.getSegOwner(transactionalBufferMetrics, resultSet);
|
||||
int operationCode = RowMapper.getOperationCode(transactionalBufferMetrics, resultSet);
|
||||
Timestamp changeTime = RowMapper.getChangeTime(transactionalBufferMetrics, resultSet);
|
||||
String txId = RowMapper.getTransactionId(transactionalBufferMetrics, resultSet);
|
||||
String operation = RowMapper.getOperation(transactionalBufferMetrics, resultSet);
|
||||
String userName = RowMapper.getUsername(transactionalBufferMetrics, resultSet);
|
||||
|
||||
LOGGER.trace("scn={}, operationCode={}, operation={}, table={}, segOwner={}, userName={}", scn, operationCode, operation, tableName, segOwner, userName);
|
||||
|
||||
String logMessage = String.format("transactionId = %s, SCN= %s, table_name= %s, segOwner= %s, operationCode=%s, offsetSCN= %s, " +
|
||||
" commitOffsetSCN= %s", txId, scn, tableName, segOwner, operationCode, offsetContext.getScn(), offsetContext.getCommitScn());
|
||||
|
||||
if (scn == null) {
|
||||
LogMinerHelper.logWarn(transactionalBufferMetrics, "Scn is null for {}", logMessage);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Commit
|
||||
if (operationCode == RowMapper.COMMIT) {
|
||||
if (transactionalBuffer.commit(txId, scn, offsetContext, changeTime, context, logMessage)) {
|
||||
LOGGER.trace("COMMIT, {}", logMessage);
|
||||
commitCounter++;
|
||||
cumulativeCommitTime = cumulativeCommitTime.plus(Duration.between(iterationStart, Instant.now()));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Rollback
|
||||
if (operationCode == RowMapper.ROLLBACK) {
|
||||
if (transactionalBuffer.rollback(txId, logMessage)) {
|
||||
LOGGER.trace("ROLLBACK, {}", logMessage);
|
||||
rollbackCounter++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// DDL
|
||||
if (operationCode == RowMapper.DDL) {
|
||||
// todo: DDL operations are not yet supported during streaming while using Log Miner.
|
||||
LOGGER.info("DDL: {}, REDO_SQL: {}", logMessage, redo_sql);
|
||||
continue;
|
||||
}
|
||||
|
||||
// MISSING_SCN
|
||||
if (operationCode == RowMapper.MISSING_SCN) {
|
||||
LogMinerHelper.logWarn(transactionalBufferMetrics, "Missing SCN, {}", logMessage);
|
||||
continue;
|
||||
}
|
||||
|
||||
// DML
|
||||
if (operationCode == RowMapper.INSERT || operationCode == RowMapper.DELETE || operationCode == RowMapper.UPDATE) {
|
||||
LOGGER.trace("DML, {}, sql {}", logMessage, redo_sql);
|
||||
dmlCounter++;
|
||||
metrics.incrementCapturedDmlCount();
|
||||
iterationStart = Instant.now();
|
||||
LogMinerDmlEntry dmlEntry = dmlParser.parse(redo_sql, schema.getTables(), txId);
|
||||
cumulativeParseTime = cumulativeParseTime.plus(Duration.between(iterationStart, Instant.now()));
|
||||
iterationStart = Instant.now();
|
||||
|
||||
if (dmlEntry == null || redo_sql == null) {
|
||||
LOGGER.trace("Following statement was not parsed: {}, details: {}", redo_sql, logMessage);
|
||||
continue;
|
||||
}
|
||||
|
||||
// this will happen for instance on a blacklisted column change, we will omit this update
|
||||
if (dmlEntry.getCommandType().equals(Envelope.Operation.UPDATE)
|
||||
&& dmlEntry.getOldValues().size() == dmlEntry.getNewValues().size()
|
||||
&& dmlEntry.getNewValues().containsAll(dmlEntry.getOldValues())) {
|
||||
LOGGER.trace("Following DML was skipped, " +
|
||||
"most likely because of ignored blacklisted column change: {}, details: {}", redo_sql, logMessage);
|
||||
continue;
|
||||
}
|
||||
|
||||
dmlEntry.setObjectOwner(segOwner);
|
||||
dmlEntry.setSourceTime(changeTime);
|
||||
dmlEntry.setTransactionId(txId);
|
||||
dmlEntry.setObjectName(tableName);
|
||||
dmlEntry.setScn(scn);
|
||||
|
||||
try {
|
||||
TableId tableId = RowMapper.getTableId(catalogName, resultSet);
|
||||
transactionalBuffer.registerCommitCallback(txId, scn, changeTime.toInstant(), redo_sql, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
// update SCN in offset context only if processed SCN less than SCN among other transactions
|
||||
if (smallestScn == null || scn.compareTo(smallestScn) < 0) {
|
||||
offsetContext.setScn(scn.longValue());
|
||||
transactionalBufferMetrics.setOldestScn(scn.longValue());
|
||||
}
|
||||
offsetContext.setTransactionId(txId);
|
||||
offsetContext.setSourceTime(timestamp.toInstant());
|
||||
offsetContext.setTableId(tableId);
|
||||
if (counter == 0) {
|
||||
offsetContext.setCommitScn(commitScn.longValue());
|
||||
}
|
||||
Table table = schema.tableFor(tableId);
|
||||
LOGGER.trace("Processing DML event {} scn {}", dmlEntry.toString(), scn);
|
||||
|
||||
dispatcher.dispatchDataChangeEvent(tableId,
|
||||
new LogMinerChangeRecordEmitter(offsetContext, dmlEntry, table, clock));
|
||||
});
|
||||
cumulativeOtherTime = cumulativeOtherTime.plus(Duration.between(iterationStart, Instant.now()));
|
||||
|
||||
}
|
||||
catch (Exception e) {
|
||||
LogMinerHelper.logError(transactionalBufferMetrics, "Following dmlEntry: {} cannot be dispatched due to the : {}", dmlEntry, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
metrics.setProcessedCapturedBatchDuration(Duration.between(startTime, Instant.now()));
|
||||
if (dmlCounter > 0 || commitCounter > 0 || rollbackCounter > 0) {
|
||||
warnStuckScn();
|
||||
currentOffsetScn = offsetContext.getScn();
|
||||
if (offsetContext.getCommitScn() != null) {
|
||||
currentOffsetCommitScn = offsetContext.getCommitScn();
|
||||
}
|
||||
LOGGER.debug("{} DMLs, {} Commits, {} Rollbacks. Millis - (total:{}, commit:{}, parse:{}, other:{}). " +
|
||||
"Lag:{}. Offset scn:{}. Offset commit scn:{}. Active transactions:{}. Sleep time:{}",
|
||||
dmlCounter, commitCounter, rollbackCounter, (Duration.between(startTime, Instant.now()).toMillis()),
|
||||
cumulativeCommitTime.toMillis(), cumulativeParseTime.toMillis(), cumulativeOtherTime.toMillis(),
|
||||
transactionalBufferMetrics.getLagFromSource(), offsetContext.getScn(), offsetContext.getCommitScn(),
|
||||
transactionalBufferMetrics.getNumberOfActiveTransactions(), metrics.getMillisecondToSleepBetweenMiningQuery());
|
||||
}
|
||||
return dmlCounter;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is warning if a long running transaction is discovered and could be abandoned in the future.
|
||||
* The criteria is the offset SCN remains the same in five mining cycles
|
||||
*/
|
||||
private void warnStuckScn() {
|
||||
if (offsetContext != null && offsetContext.getCommitScn() != null) {
|
||||
if (currentOffsetScn == offsetContext.getScn() && currentOffsetCommitScn != offsetContext.getCommitScn()) {
|
||||
stuckScnCounter++;
|
||||
// logWarn only once
|
||||
if (stuckScnCounter == 5) {
|
||||
LogMinerHelper.logWarn(transactionalBufferMetrics,
|
||||
"Offset SCN {} did not change in five mining cycles, hence the oldest transaction was not committed. Offset commit SCN: {}", currentOffsetScn,
|
||||
offsetContext.getCommitScn());
|
||||
transactionalBufferMetrics.incrementScnFreezeCounter();
|
||||
}
|
||||
}
|
||||
else {
|
||||
stuckScnCounter = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import io.debezium.connector.oracle.BaseOracleSchemaChangeEventEmitter;
|
||||
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDdlEntry;
|
||||
import io.debezium.pipeline.spi.SchemaChangeEventEmitter;
|
||||
import io.debezium.relational.TableId;
|
||||
|
||||
/**
|
||||
* {@link SchemaChangeEventEmitter} implementation based on Oracle LogMiner utility.
|
||||
*/
|
||||
public class LogMinerSchemaChangeEventEmitter extends BaseOracleSchemaChangeEventEmitter {
|
||||
|
||||
public LogMinerSchemaChangeEventEmitter(OracleOffsetContext offsetContext, TableId tableId, LogMinerDdlEntry ddlLcr) {
|
||||
super(offsetContext,
|
||||
tableId,
|
||||
tableId.catalog(), // todo tableId should be enough
|
||||
tableId.schema(), // todo same here
|
||||
ddlLcr.getDdlText(),
|
||||
ddlLcr.getCommandType());
|
||||
}
|
||||
}
|
@ -0,0 +1,263 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.buildDataDictionary;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.checkSupplementalLogging;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.createAuditTable;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.endMining;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.getCurrentRedoLogFiles;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.getEndScn;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.getFirstOnlineLogScn;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.getLastScnFromTheOldestOnlineRedo;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.getTimeDifference;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.logError;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.logWarn;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.setNlsSessionParameters;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.setRedoLogFilesForMining;
|
||||
import static io.debezium.connector.oracle.logminer.LogMinerHelper.startOnlineMining;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.SQLRecoverableException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.connector.oracle.OracleConnection;
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||
import io.debezium.connector.oracle.OracleDatabaseSchema;
|
||||
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||
import io.debezium.connector.oracle.OracleTaskContext;
|
||||
import io.debezium.connector.oracle.jsqlparser.SimpleDmlParser;
|
||||
import io.debezium.pipeline.ErrorHandler;
|
||||
import io.debezium.pipeline.EventDispatcher;
|
||||
import io.debezium.pipeline.source.spi.StreamingChangeEventSource;
|
||||
import io.debezium.relational.TableId;
|
||||
import io.debezium.util.Clock;
|
||||
import io.debezium.util.Metronome;
|
||||
|
||||
import oracle.net.ns.NetException;
|
||||
|
||||
/**
|
||||
* A {@link StreamingChangeEventSource} based on Oracle's LogMiner utility.
|
||||
* The event handler loop is executed in a separate executor.
|
||||
*/
|
||||
public class LogMinerStreamingChangeEventSource implements StreamingChangeEventSource {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LogMinerStreamingChangeEventSource.class);
|
||||
|
||||
private final OracleConnection jdbcConnection;
|
||||
private final EventDispatcher<TableId> dispatcher;
|
||||
private final Clock clock;
|
||||
private final OracleDatabaseSchema schema;
|
||||
private final OracleOffsetContext offsetContext;
|
||||
private final SimpleDmlParser dmlParser;
|
||||
private final String catalogName;
|
||||
private OracleConnectorConfig connectorConfig;
|
||||
private TransactionalBufferMetrics transactionalBufferMetrics;
|
||||
private LogMinerMetrics logMinerMetrics;
|
||||
private TransactionalBuffer transactionalBuffer;
|
||||
private final OracleConnectorConfig.LogMiningStrategy strategy;
|
||||
private final OracleTaskContext taskContext;
|
||||
private final ErrorHandler errorHandler;
|
||||
private final boolean isContinuousMining;
|
||||
private long startScn;
|
||||
private long endScn;
|
||||
|
||||
public LogMinerStreamingChangeEventSource(OracleConnectorConfig connectorConfig, OracleOffsetContext offsetContext,
|
||||
OracleConnection jdbcConnection, EventDispatcher<TableId> dispatcher,
|
||||
ErrorHandler errorHandler, Clock clock, OracleDatabaseSchema schema,
|
||||
OracleTaskContext taskContext) {
|
||||
this.jdbcConnection = jdbcConnection;
|
||||
this.dispatcher = dispatcher;
|
||||
this.clock = clock;
|
||||
this.schema = schema;
|
||||
this.offsetContext = offsetContext;
|
||||
OracleChangeRecordValueConverter converters = new OracleChangeRecordValueConverter(jdbcConnection);
|
||||
this.connectorConfig = connectorConfig;
|
||||
this.catalogName = (connectorConfig.getPdbName() != null) ? connectorConfig.getPdbName() : connectorConfig.getDatabaseName();
|
||||
this.dmlParser = new SimpleDmlParser(catalogName, connectorConfig.getSchemaName(), converters);
|
||||
this.strategy = connectorConfig.getLogMiningStrategy();
|
||||
this.isContinuousMining = connectorConfig.isContinuousMining();
|
||||
this.errorHandler = errorHandler;
|
||||
this.taskContext = taskContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* This is the loop to get changes from LogMiner
|
||||
*
|
||||
* @param context change event source context
|
||||
*/
|
||||
@Override
|
||||
public void execute(ChangeEventSourceContext context) {
|
||||
Metronome metronome;
|
||||
|
||||
this.transactionalBufferMetrics = new TransactionalBufferMetrics(taskContext);
|
||||
this.transactionalBufferMetrics.register(LOGGER);
|
||||
|
||||
this.transactionalBuffer = new TransactionalBuffer(connectorConfig.getLogicalName(), errorHandler, transactionalBufferMetrics, connectorConfig.getMaxQueueSize());
|
||||
|
||||
this.logMinerMetrics = new LogMinerMetrics(taskContext);
|
||||
this.logMinerMetrics.register(LOGGER);
|
||||
|
||||
// The top outer loop gives the resiliency on the network disconnections. This is critical for cloud deployment.
|
||||
while (context.isRunning()) {
|
||||
try (Connection connection = jdbcConnection.connection(false);
|
||||
PreparedStatement fetchFromMiningView = connection
|
||||
.prepareStatement(SqlUtils.queryLogMinerContents(connectorConfig.getSchemaName(), jdbcConnection.username(), schema))) {
|
||||
|
||||
startScn = offsetContext.getScn();
|
||||
createAuditTable(connection);
|
||||
LOGGER.trace("current millis {}, db time {}", System.currentTimeMillis(), getTimeDifference(connection));
|
||||
transactionalBufferMetrics.setTimeDifference(new AtomicLong(getTimeDifference(connection)));
|
||||
|
||||
if (!isContinuousMining && startScn < getFirstOnlineLogScn(connection)) {
|
||||
throw new RuntimeException("Online REDO LOG files don't contain the offset SCN. Clean offset and start over");
|
||||
}
|
||||
|
||||
// 1. Configure Log Miner to mine online redo logs
|
||||
setNlsSessionParameters(jdbcConnection);
|
||||
checkSupplementalLogging(jdbcConnection, connectorConfig.getPdbName());
|
||||
|
||||
if (strategy == OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO) {
|
||||
buildDataDictionary(connection);
|
||||
}
|
||||
|
||||
if (!isContinuousMining) {
|
||||
setRedoLogFilesForMining(connection, startScn);
|
||||
}
|
||||
|
||||
LogMinerQueryResultProcessor processor = new LogMinerQueryResultProcessor(context, logMinerMetrics, transactionalBuffer,
|
||||
dmlParser, offsetContext, schema, dispatcher, transactionalBufferMetrics, catalogName, clock);
|
||||
|
||||
// 2. Querying LogMiner view while running
|
||||
Set<String> currentRedoLogFiles = getCurrentRedoLogFiles(connection, logMinerMetrics);
|
||||
while (context.isRunning()) {
|
||||
|
||||
endScn = getEndScn(connection, startScn, logMinerMetrics);
|
||||
// LOGGER.trace("startScn: {}, endScn: {}", startScn, endScn);
|
||||
|
||||
metronome = Metronome.sleeper(Duration.ofMillis(logMinerMetrics.getMillisecondToSleepBetweenMiningQuery()), clock);
|
||||
metronome.pause();
|
||||
|
||||
Set<String> possibleNewCurrentLogFile = getCurrentRedoLogFiles(connection, logMinerMetrics);
|
||||
if (!currentRedoLogFiles.equals(possibleNewCurrentLogFile)) {
|
||||
LOGGER.debug("\n\n***** SWITCH occurred *****\n" + " from:{} , to:{} \n\n", currentRedoLogFiles, possibleNewCurrentLogFile);
|
||||
|
||||
// This is the way to mitigate PGA leak.
|
||||
// With one mining session it grows and maybe there is another way to flush PGA, but at this point we use new mining session
|
||||
endMining(connection);
|
||||
|
||||
if (!isContinuousMining) {
|
||||
if (strategy == OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO) {
|
||||
buildDataDictionary(connection);
|
||||
}
|
||||
|
||||
abandonOldTransactionsIfExist(connection);
|
||||
setRedoLogFilesForMining(connection, startScn);
|
||||
}
|
||||
|
||||
currentRedoLogFiles = getCurrentRedoLogFiles(connection, logMinerMetrics);
|
||||
}
|
||||
|
||||
startOnlineMining(connection, startScn, endScn, strategy, isContinuousMining);
|
||||
|
||||
Instant startTime = Instant.now();
|
||||
fetchFromMiningView.setFetchSize(10_000);
|
||||
fetchFromMiningView.setLong(1, startScn);
|
||||
fetchFromMiningView.setLong(2, endScn);
|
||||
|
||||
ResultSet res = fetchFromMiningView.executeQuery();
|
||||
logMinerMetrics.setLastLogMinerQueryDuration(Duration.between(startTime, Instant.now()));
|
||||
processor.processResult(res);
|
||||
|
||||
updateStartScn();
|
||||
// LOGGER.trace("largest scn = {}", transactionalBuffer.getLargestScn());
|
||||
|
||||
// update SCN in offset context only if buffer is empty, otherwise we update offset in TransactionalBuffer
|
||||
if (transactionalBuffer.isEmpty()) {
|
||||
offsetContext.setScn(startScn);
|
||||
transactionalBuffer.resetLargestScn(null);
|
||||
}
|
||||
|
||||
res.close();
|
||||
// we don't do it for other modes to save time on building data dictionary
|
||||
// if (strategy == OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG) {
|
||||
// endMining(connection);
|
||||
// updateRedoLogMetrics(connection, logMinerMetrics);
|
||||
// currentRedoLogFiles = getCurrentRedoLogFiles(connection, logMinerMetrics);
|
||||
// }
|
||||
}
|
||||
}
|
||||
catch (Throwable e) {
|
||||
if (connectionProblem(e)) {
|
||||
logWarn(transactionalBufferMetrics, "Disconnection occurred. {} ", e.toString());
|
||||
continue;
|
||||
}
|
||||
logError(transactionalBufferMetrics, "Mining session was stopped due to the {} ", e.toString());
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
finally {
|
||||
LOGGER.info("startScn={}, endScn={}, offsetContext.getScn()={}", startScn, endScn, offsetContext.getScn());
|
||||
LOGGER.info("Transactional buffer metrics dump: {}", transactionalBufferMetrics.toString());
|
||||
LOGGER.info("Transactional buffer dump: {}", transactionalBuffer.toString());
|
||||
LOGGER.info("LogMiner metrics dump: {}", logMinerMetrics.toString());
|
||||
}
|
||||
}
|
||||
|
||||
logMinerMetrics.unregister(LOGGER);
|
||||
transactionalBufferMetrics.unregister(LOGGER);
|
||||
}
|
||||
|
||||
private void abandonOldTransactionsIfExist(Connection connection) throws SQLException {
|
||||
Optional<Long> lastScnToAbandonTransactions = getLastScnFromTheOldestOnlineRedo(connection, offsetContext.getScn());
|
||||
lastScnToAbandonTransactions.ifPresent(thresholdScn -> {
|
||||
logWarn(transactionalBufferMetrics, "All transactions with first SCN <= {} will be abandoned, offset: {}", thresholdScn, offsetContext.getScn());
|
||||
transactionalBuffer.abandonLongTransactions(thresholdScn);
|
||||
offsetContext.setScn(thresholdScn);
|
||||
updateStartScn();
|
||||
});
|
||||
}
|
||||
|
||||
private void updateStartScn() {
|
||||
long nextStartScn = transactionalBuffer.getLargestScn().equals(BigDecimal.ZERO) ? endScn : transactionalBuffer.getLargestScn().longValue();
|
||||
if (nextStartScn <= startScn) {
|
||||
// When system is idle, largest SCN may stay unchanged, move it forward then
|
||||
transactionalBuffer.resetLargestScn(endScn);
|
||||
}
|
||||
startScn = endScn;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void commitOffset(Map<String, ?> offset) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
private boolean connectionProblem(Throwable e) {
|
||||
if (e.getMessage() == null || e.getCause() == null) {
|
||||
return false;
|
||||
}
|
||||
return e.getMessage().startsWith("ORA-03135") || // connection lost contact
|
||||
e.getMessage().startsWith("ORA-12543") || // TNS:destination host unreachable
|
||||
e.getMessage().startsWith("ORA-00604") || // error occurred at recursive SQL level 1
|
||||
e.getMessage().startsWith("ORA-01089") || // Oracle immediate shutdown in progress
|
||||
e.getCause() instanceof IOException ||
|
||||
e instanceof SQLRecoverableException ||
|
||||
e.getMessage().toUpperCase().startsWith("NO MORE DATA TO READ FROM SOCKET") ||
|
||||
e.getCause().getCause() instanceof NetException;
|
||||
}
|
||||
}
|
@ -0,0 +1,569 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import static io.debezium.util.NumberConversions.BYTE_FALSE;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Types;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.DateTimeFormatterBuilder;
|
||||
import java.time.temporal.ChronoField;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.kafka.connect.data.Field;
|
||||
import org.apache.kafka.connect.data.SchemaBuilder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.config.CommonConnectorConfig.BinaryHandlingMode;
|
||||
import io.debezium.data.SpecialValueDecimal;
|
||||
import io.debezium.data.VariableScaleDecimal;
|
||||
import io.debezium.jdbc.JdbcConnection;
|
||||
import io.debezium.jdbc.JdbcValueConverters;
|
||||
import io.debezium.relational.Column;
|
||||
import io.debezium.relational.ValueConverter;
|
||||
import io.debezium.time.Date;
|
||||
import io.debezium.time.MicroDuration;
|
||||
import io.debezium.time.ZonedTimestamp;
|
||||
import io.debezium.util.NumberConversions;
|
||||
import io.debezium.util.Strings;
|
||||
|
||||
import oracle.jdbc.OracleTypes;
|
||||
import oracle.sql.BINARY_DOUBLE;
|
||||
import oracle.sql.BINARY_FLOAT;
|
||||
import oracle.sql.BLOB;
|
||||
import oracle.sql.CHAR;
|
||||
import oracle.sql.CLOB;
|
||||
import oracle.sql.DATE;
|
||||
import oracle.sql.INTERVALDS;
|
||||
import oracle.sql.INTERVALYM;
|
||||
import oracle.sql.NUMBER;
|
||||
import oracle.sql.TIMESTAMP;
|
||||
import oracle.sql.TIMESTAMPLTZ;
|
||||
import oracle.sql.TIMESTAMPTZ;
|
||||
|
||||
/**
|
||||
* This class is used as a value converter by the DML parser.
|
||||
* todo this class is a replica of OracleValueConverters which is used by XStream and Logminer for snapshot mode
|
||||
*/
|
||||
public class OracleChangeRecordValueConverter extends JdbcValueConverters {
|
||||
|
||||
private static final Pattern INTERVAL_DAY_SECOND_PATTERN = Pattern.compile("([+\\-])?(\\d+) (\\d+):(\\d+):(\\d+).(\\d+)");
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(OracleChangeRecordValueConverter.class);
|
||||
|
||||
private static final DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder()
|
||||
.parseCaseInsensitive()
|
||||
.appendPattern("yyyy-MM-dd")
|
||||
.toFormatter();
|
||||
private static final DateTimeFormatter TIMESTAMP_FORMATTER = new DateTimeFormatterBuilder()
|
||||
.parseCaseInsensitive()
|
||||
.appendPattern("yyyy-MM-dd HH:mm:ss")
|
||||
.optionalStart()
|
||||
.appendPattern(".")
|
||||
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, false)
|
||||
.optionalEnd()
|
||||
.toFormatter();
|
||||
private static final DateTimeFormatter TIMESTAMP_TZ_FORMATTER = new DateTimeFormatterBuilder()
|
||||
.parseCaseInsensitive()
|
||||
.appendPattern("yyyy-MM-dd HH:mm:ss")
|
||||
.optionalStart()
|
||||
.appendPattern(".")
|
||||
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, false)
|
||||
.optionalEnd()
|
||||
.appendPattern(" XXX")
|
||||
.toFormatter();
|
||||
|
||||
private final JdbcConnection connection;
|
||||
|
||||
public OracleChangeRecordValueConverter(JdbcConnection connection) {
|
||||
this.connection = connection;
|
||||
}
|
||||
|
||||
// todo some methods are just a clones from OracleValueConverters, remove or implement differently
|
||||
|
||||
@Override
|
||||
public SchemaBuilder schemaBuilder(Column column) {
|
||||
if (column == null) { // todo: we will address it if happens
|
||||
LOGGER.warn("Column is null, investigate");
|
||||
return null;
|
||||
}
|
||||
LOGGER.trace("Building schema for column {} of type {} named {} with constraints ({},{})",
|
||||
column.name(),
|
||||
column.jdbcType(),
|
||||
column.typeName(),
|
||||
column.length(),
|
||||
column.scale());
|
||||
|
||||
switch (column.jdbcType()) {
|
||||
// Oracle's float is not float as in Java but a NUMERIC without scale
|
||||
// case Types.FLOAT:
|
||||
// return VariableScaleDecimal.builder();
|
||||
case Types.NUMERIC:
|
||||
return getNumericSchema(column);
|
||||
case OracleTypes.BINARY_FLOAT:
|
||||
return SchemaBuilder.float32();
|
||||
case OracleTypes.BINARY_DOUBLE:
|
||||
return SchemaBuilder.float64();
|
||||
// case OracleTypes.TIMESTAMP:
|
||||
// return io.debezium.time.Timestamp.builder();
|
||||
case OracleTypes.TIMESTAMPTZ:
|
||||
case OracleTypes.TIMESTAMPLTZ:
|
||||
return ZonedTimestamp.builder();
|
||||
case OracleTypes.INTERVALYM:
|
||||
case OracleTypes.INTERVALDS:
|
||||
return MicroDuration.builder();
|
||||
case OracleTypes.STRUCT:
|
||||
return SchemaBuilder.string();
|
||||
default:
|
||||
return super.schemaBuilder(column);
|
||||
}
|
||||
}
|
||||
|
||||
private SchemaBuilder getNumericSchema(Column column) {
|
||||
if (column.scale().isPresent()) {
|
||||
// return sufficiently sized int schema for non-floating point types
|
||||
Integer scale = column.scale().get();
|
||||
|
||||
// a negative scale means rounding, e.g. NUMBER(10, -2) would be rounded to hundreds
|
||||
if (scale <= 0) {
|
||||
// Boolean represtented as Number(1,0)
|
||||
if (scale == 0 && column.length() == 1) {
|
||||
return SchemaBuilder.bool();
|
||||
}
|
||||
|
||||
int width = column.length() - scale;
|
||||
|
||||
if (width < 3) {
|
||||
return SchemaBuilder.int8();
|
||||
}
|
||||
else if (width < 5) {
|
||||
return SchemaBuilder.int16();
|
||||
}
|
||||
else if (width < 10 || (width == 10 && scale == 0)) {
|
||||
return SchemaBuilder.int32();
|
||||
}
|
||||
else if (width < 19) {
|
||||
return SchemaBuilder.int64();
|
||||
}
|
||||
}
|
||||
|
||||
// larger non-floating point types and floating point types use Decimal
|
||||
return super.schemaBuilder(column);
|
||||
}
|
||||
else {
|
||||
return VariableScaleDecimal.builder();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ValueConverter converter(Column column, Field fieldDefn) {
|
||||
switch (column.jdbcType()) {
|
||||
case Types.CHAR:
|
||||
case Types.VARCHAR:
|
||||
case Types.NCHAR:
|
||||
case Types.NVARCHAR:
|
||||
case OracleTypes.STRUCT:
|
||||
case Types.CLOB:
|
||||
return data -> convertString(column, fieldDefn, data);
|
||||
case Types.BLOB:
|
||||
return data -> convertBinary(column, fieldDefn, data, binaryMode);
|
||||
case OracleTypes.BINARY_FLOAT:
|
||||
return data -> convertFloat(column, fieldDefn, data);
|
||||
case OracleTypes.BINARY_DOUBLE:
|
||||
return data -> convertDouble(column, fieldDefn, data);
|
||||
case Types.NUMERIC:
|
||||
return getNumericConverter(column, fieldDefn);
|
||||
case Types.FLOAT:
|
||||
return data -> getFloatConverter(column, fieldDefn, data);
|
||||
case OracleTypes.TIMESTAMP:
|
||||
return data -> convertToLocalDateTime(column, fieldDefn, data);
|
||||
case OracleTypes.TIMESTAMPTZ:
|
||||
case OracleTypes.TIMESTAMPLTZ:
|
||||
return (data) -> convertTimestampWithZone(column, fieldDefn, data);
|
||||
case OracleTypes.INTERVALYM:
|
||||
return (data) -> convertIntervalYearMonth(column, fieldDefn, data);
|
||||
case OracleTypes.INTERVALDS:
|
||||
return (data) -> convertIntervalDaySecond(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
return super.converter(column, fieldDefn);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a string object for an object type of {@link LocalDateTime}.
|
||||
* If the column definition allows null and default value is 0000-00-00 00:00:00, we need return null,
|
||||
* else 0000-00-00 00:00:00 will be replaced with 1970-01-01 00:00:00;
|
||||
*
|
||||
* @param column the column definition describing the {@code data} value; never null
|
||||
* @param fieldDefn field definition
|
||||
* @param value the string object to be converted into a {@link LocalDateTime} type;
|
||||
* @return the converted value;
|
||||
*/
|
||||
private Object convertToLocalDateTime(Column column, Field fieldDefn, Object value) {
|
||||
|
||||
// todo make it better
|
||||
String dateText;
|
||||
if (value instanceof String) {
|
||||
String valueString = (String) value;
|
||||
if (valueString.toLowerCase().startsWith("to_timestamp")) {
|
||||
dateText = valueString.substring("to_timestamp".length() + 2, valueString.length() - 2);
|
||||
LocalDateTime dateTime = LocalDateTime.from(TIMESTAMP_FORMATTER.parse(dateText.trim()));
|
||||
// todo: DBZ-137 this originally converted to ZoneId.systemDefault() but this should be GMT
|
||||
// this is so that this behavior is compatible with the Xstreams implementation
|
||||
return dateTime.atZone(ZoneId.of("GMT")).toInstant().toEpochMilli() * 1000; // timestamp(6) is converted as microTimestamp
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private ValueConverter getNumericConverter(Column column, Field fieldDefn) {
|
||||
if (column.scale().isPresent()) {
|
||||
Integer scale = column.scale().get();
|
||||
|
||||
if (scale <= 0) {
|
||||
// Boolean represtented as Number(1,0)
|
||||
if (scale == 0 && column.length() == 1) {
|
||||
return data -> convertBoolean(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
int width = column.length() - scale;
|
||||
|
||||
if (width < 3) {
|
||||
return data -> convertNumericAsTinyInt(column, fieldDefn, data);
|
||||
}
|
||||
else if (width < 5) {
|
||||
return data -> convertNumericAsSmallInt(column, fieldDefn, data);
|
||||
}
|
||||
else if (width < 10 || (width == 10 && scale == 0)) {
|
||||
return data -> convertNumericAsInteger(column, fieldDefn, data);
|
||||
}
|
||||
else if (width < 19) {
|
||||
return data -> convertNumericAsBigInteger(column, fieldDefn, data);
|
||||
}
|
||||
}
|
||||
|
||||
// larger non-floating point types and floating point types use Decimal
|
||||
return data -> convertNumeric(column, fieldDefn, data);
|
||||
}
|
||||
else {
|
||||
return data -> convertVariableScale(column, fieldDefn, data);
|
||||
}
|
||||
}
|
||||
|
||||
private Object getFloatConverter(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof String) {
|
||||
return Float.parseFloat((String) data);
|
||||
}
|
||||
return convertVariableScale(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertString(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof CHAR) {
|
||||
return ((CHAR) data).stringValue();
|
||||
}
|
||||
if (data instanceof CLOB) {
|
||||
return ((CLOB) data).toString();
|
||||
}
|
||||
|
||||
return super.convertString(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertBinary(Column column, Field fieldDefn, Object data, BinaryHandlingMode mode) {
|
||||
if (data instanceof BLOB) {
|
||||
return ((BLOB) data).getBytes();
|
||||
}
|
||||
return super.convertBinary(column, fieldDefn, data, mode);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertInteger(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof NUMBER) {
|
||||
try {
|
||||
data = ((NUMBER) data).intValue();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||
}
|
||||
}
|
||||
|
||||
return super.convertInteger(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertFloat(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof Float) {
|
||||
return data;
|
||||
}
|
||||
else if (data instanceof NUMBER) {
|
||||
return ((NUMBER) data).floatValue();
|
||||
}
|
||||
else if (data instanceof BINARY_FLOAT) {
|
||||
try {
|
||||
return ((BINARY_FLOAT) data).floatValue();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||
}
|
||||
}
|
||||
|
||||
return super.convertFloat(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertDouble(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof BINARY_DOUBLE) {
|
||||
try {
|
||||
return ((BINARY_DOUBLE) data).doubleValue();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||
}
|
||||
}
|
||||
|
||||
return super.convertDouble(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertDecimal(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof NUMBER) {
|
||||
try {
|
||||
data = ((NUMBER) data).bigDecimalValue();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||
}
|
||||
}
|
||||
|
||||
// adjust scale to column's scale if the column's scale is larger than the one from
|
||||
// the value (e.g. 4.4444 -> 4.444400)
|
||||
if (data instanceof BigDecimal) {
|
||||
data = withScaleAdjustedIfNeeded(column, (BigDecimal) data);
|
||||
}
|
||||
|
||||
return super.convertDecimal(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
protected BigDecimal withScaleAdjustedIfNeeded(Column column, BigDecimal data) {
|
||||
if (column.scale().isPresent() && column.scale().get() > data.scale()) {
|
||||
data = data.setScale(column.scale().get());
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertNumeric(Column column, Field fieldDefn, Object data) {
|
||||
return convertDecimal(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
private Object convertNumericAsTinyInt(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof NUMBER) {
|
||||
try {
|
||||
data = ((NUMBER) data).byteValue();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||
}
|
||||
}
|
||||
|
||||
return convertTinyInt(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
private Object convertNumericAsSmallInt(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof NUMBER) {
|
||||
try {
|
||||
data = ((NUMBER) data).shortValue();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||
}
|
||||
}
|
||||
|
||||
return super.convertSmallInt(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
private Object convertNumericAsInteger(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof NUMBER) {
|
||||
try {
|
||||
data = ((NUMBER) data).intValue();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||
}
|
||||
}
|
||||
|
||||
return super.convertInteger(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
private Object convertNumericAsBigInteger(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof NUMBER) {
|
||||
try {
|
||||
data = ((NUMBER) data).longValue();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||
}
|
||||
}
|
||||
|
||||
return super.convertBigInt(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a value object for an expected JDBC type of {@link Types#BOOLEAN}.
|
||||
*
|
||||
* @param column the column definition describing the {@code data} value; never null
|
||||
* @param fieldDefn the field definition; never null
|
||||
* @param data the data object to be converted into a {@link Date Kafka Connect date} type; never null
|
||||
* @return the converted value, or null if the conversion could not be made and the column allows nulls
|
||||
* @throws IllegalArgumentException if the value could not be converted but the column does not allow nulls
|
||||
*/
|
||||
@Override
|
||||
protected Object convertBoolean(Column column, Field fieldDefn, Object data) {
|
||||
if (data instanceof String) {
|
||||
return Byte.parseByte((String) data) == 0 ? Boolean.FALSE : Boolean.TRUE;
|
||||
}
|
||||
return super.convertBoolean(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertTinyInt(Column column, Field fieldDefn, Object data) {
|
||||
return convertValue(column, fieldDefn, data, BYTE_FALSE, (r) -> {
|
||||
if (data instanceof Byte) {
|
||||
r.deliver(data);
|
||||
}
|
||||
else if (data instanceof Number) {
|
||||
Number value = (Number) data;
|
||||
r.deliver(value.byteValue());
|
||||
}
|
||||
else if (data instanceof Boolean) {
|
||||
r.deliver(NumberConversions.getByte((boolean) data));
|
||||
}
|
||||
else if (data instanceof String) {
|
||||
r.deliver(Byte.parseByte((String) data));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private Object convertVariableScale(Column column, Field fieldDefn, Object data) {
|
||||
data = convertNumeric(column, fieldDefn, data); // provides default value
|
||||
|
||||
if (data == null) {
|
||||
return null;
|
||||
}
|
||||
// TODO Need to handle special values, it is not supported in variable scale decimal
|
||||
else if (data instanceof SpecialValueDecimal) {
|
||||
return VariableScaleDecimal.fromLogical(fieldDefn.schema(), (SpecialValueDecimal) data);
|
||||
}
|
||||
else if (data instanceof BigDecimal) {
|
||||
return VariableScaleDecimal.fromLogical(fieldDefn.schema(), new SpecialValueDecimal((BigDecimal) data));
|
||||
}
|
||||
return handleUnknownData(column, fieldDefn, data);
|
||||
}
|
||||
|
||||
private Object fromOracleTimeClasses(Column column, Object data) {
|
||||
try {
|
||||
if (data instanceof TIMESTAMP) {
|
||||
data = ((TIMESTAMP) data).timestampValue();
|
||||
}
|
||||
else if (data instanceof DATE) {
|
||||
data = ((DATE) data).timestampValue();
|
||||
}
|
||||
else if (data instanceof TIMESTAMPTZ) {
|
||||
final TIMESTAMPTZ ts = (TIMESTAMPTZ) data;
|
||||
data = ZonedDateTime.ofInstant(ts.timestampValue(connection.connection()).toInstant(), ts.getTimeZone().toZoneId());
|
||||
}
|
||||
else if (data instanceof TIMESTAMPLTZ) {
|
||||
// JDBC driver throws an exception
|
||||
// final TIMESTAMPLTZ ts = (TIMESTAMPLTZ)data;
|
||||
// data = ts.offsetDateTimeValue(connection.connection());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertTimestampToEpochMicros(Column column, Field fieldDefn, Object data) {
|
||||
return super.convertTimestampToEpochMicros(column, fieldDefn, fromOracleTimeClasses(column, data));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertTimestampToEpochMillis(Column column, Field fieldDefn, Object data) {
|
||||
return super.convertTimestampToEpochMillis(column, fieldDefn, fromOracleTimeClasses(column, data));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertTimestampToEpochNanos(Column column, Field fieldDefn, Object data) {
|
||||
return super.convertTimestampToEpochNanos(column, fieldDefn, fromOracleTimeClasses(column, data));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object convertTimestampWithZone(Column column, Field fieldDefn, Object data) {
|
||||
return super.convertTimestampWithZone(column, fieldDefn, fromOracleTimeClasses(column, data));
|
||||
}
|
||||
|
||||
private Object convertIntervalYearMonth(Column column, Field fieldDefn, Object data) {
|
||||
return convertValue(column, fieldDefn, data, NumberConversions.DOUBLE_FALSE, (r) -> {
|
||||
if (data instanceof Number) {
|
||||
// we expect to get back from the plugin a double value
|
||||
r.deliver(((Number) data).doubleValue());
|
||||
}
|
||||
else if (data instanceof INTERVALYM) {
|
||||
final String interval = ((INTERVALYM) data).stringValue();
|
||||
int sign = 1;
|
||||
int start = 0;
|
||||
if (interval.charAt(0) == '-') {
|
||||
sign = -1;
|
||||
start = 1;
|
||||
}
|
||||
for (int i = 1; i < interval.length(); i++) {
|
||||
if (interval.charAt(i) == '-') {
|
||||
final int year = sign * Integer.parseInt(interval.substring(start, i));
|
||||
final int month = sign * Integer.parseInt(interval.substring(i + 1));
|
||||
r.deliver(MicroDuration.durationMicros(year, month, 0, 0,
|
||||
0, 0, MicroDuration.DAYS_PER_MONTH_AVG));
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private Object convertIntervalDaySecond(Column column, Field fieldDefn, Object data) {
|
||||
return convertValue(column, fieldDefn, data, NumberConversions.DOUBLE_FALSE, (r) -> {
|
||||
if (data instanceof Number) {
|
||||
// we expect to get back from the plugin a double value
|
||||
r.deliver(((Number) data).doubleValue());
|
||||
}
|
||||
else if (data instanceof INTERVALDS) {
|
||||
final String interval = ((INTERVALDS) data).stringValue();
|
||||
final Matcher m = INTERVAL_DAY_SECOND_PATTERN.matcher(interval);
|
||||
if (m.matches()) {
|
||||
final int sign = "-".equals(m.group(1)) ? -1 : 1;
|
||||
r.deliver(MicroDuration.durationMicros(
|
||||
0,
|
||||
0,
|
||||
sign * Integer.valueOf(m.group(2)),
|
||||
sign * Integer.valueOf(m.group(3)),
|
||||
sign * Integer.valueOf(m.group(4)),
|
||||
sign * Integer.valueOf(m.group(5)),
|
||||
sign * Integer.valueOf(Strings.pad(m.group(6), 6, '0')),
|
||||
MicroDuration.DAYS_PER_MONTH_AVG));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
@ -0,0 +1,175 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Instant;
|
||||
|
||||
import javax.xml.bind.DatatypeConverter;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.relational.TableId;
|
||||
|
||||
/**
|
||||
* A utility class to map LogMiner content resultSet values.
|
||||
* This class gracefully logs errors, loosing an entry is not critical.
|
||||
* The loss will be logged
|
||||
*/
|
||||
public class RowMapper {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(RowMapper.class);
|
||||
|
||||
// operations
|
||||
public static final int INSERT = 1;
|
||||
public static final int DELETE = 2;
|
||||
public static final int UPDATE = 3;
|
||||
public static final int DDL = 5;
|
||||
public static final int COMMIT = 7;
|
||||
public static final int MISSING_SCN = 34;
|
||||
public static final int ROLLBACK = 36;
|
||||
|
||||
private static final int SCN = 1;
|
||||
private static final int SQL_REDO = 2;
|
||||
private static final int OPERATION_CODE = 3;
|
||||
private static final int CHANGE_TIME = 4;
|
||||
private static final int TX_ID = 5;
|
||||
private static final int CSF = 6;
|
||||
private static final int TABLE_NAME = 7;
|
||||
private static final int SEG_OWNER = 8;
|
||||
private static final int OPERATION = 9;
|
||||
private static final int USERNAME = 10;
|
||||
|
||||
public static String getOperation(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||
try {
|
||||
return rs.getString(OPERATION);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
logError(metrics, e, "OPERATION");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static String getUsername(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||
try {
|
||||
return rs.getString(USERNAME);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
logError(metrics, e, "USERNAME");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static int getOperationCode(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||
try {
|
||||
return rs.getInt(OPERATION_CODE);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
logError(metrics, e, "OPERATION_CODE");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public static String getTableName(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||
try {
|
||||
return rs.getString(TABLE_NAME);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
logError(metrics, e, "TABLE_NAME");
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
public static String getSegOwner(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||
try {
|
||||
return rs.getString(SEG_OWNER);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
logError(metrics, e, "SEG_OWNER");
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
public static Timestamp getChangeTime(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||
try {
|
||||
return rs.getTimestamp(CHANGE_TIME);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
logError(metrics, e, "CHANGE_TIME");
|
||||
return new Timestamp(Instant.now().getEpochSecond());
|
||||
}
|
||||
}
|
||||
|
||||
public static BigDecimal getScn(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||
try {
|
||||
return rs.getBigDecimal(SCN);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
logError(metrics, e, "SCN");
|
||||
return new BigDecimal(-1);
|
||||
}
|
||||
}
|
||||
|
||||
public static String getTransactionId(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||
try {
|
||||
return DatatypeConverter.printHexBinary(rs.getBytes(TX_ID));
|
||||
}
|
||||
catch (SQLException e) {
|
||||
logError(metrics, e, "TX_ID");
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* It constructs REDO_SQL. If REDO_SQL is in a few lines, it truncates after first 40_000 characters
|
||||
* @param metrics metrics
|
||||
* @param rs result set
|
||||
* @return REDO_SQL
|
||||
*/
|
||||
public static String getSqlRedo(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||
int lobLimitCounter = 9; // todo : decide on approach ( XStream chunk option) and Lob limit
|
||||
StringBuilder result = new StringBuilder(4000);
|
||||
try {
|
||||
String redo_sql = rs.getString(SQL_REDO);
|
||||
if (redo_sql == null) {
|
||||
return null;
|
||||
}
|
||||
result = new StringBuilder(redo_sql);
|
||||
|
||||
int csf = rs.getInt(CSF);
|
||||
// 0 - indicates SQL_REDO is contained within the same row
|
||||
// 1 - indicates that either SQL_REDO is greater than 4000 bytes in size and is continued in
|
||||
// the next row returned by the ResultSet
|
||||
while (csf == 1) {
|
||||
rs.next();
|
||||
if (lobLimitCounter-- == 0) {
|
||||
LOGGER.warn("LOB value was truncated due to the connector limitation of {} MB", 40);
|
||||
break;
|
||||
}
|
||||
result.append(rs.getString(SQL_REDO));
|
||||
csf = rs.getInt(CSF);
|
||||
}
|
||||
|
||||
}
|
||||
catch (SQLException e) {
|
||||
logError(metrics, e, "SQL_REDO");
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
private static void logError(TransactionalBufferMetrics metrics, SQLException e, String s) {
|
||||
LogMinerHelper.logError(metrics, "Cannot get {}. This entry from log miner will be lost due to the {}", s, e);
|
||||
}
|
||||
|
||||
public static TableId getTableId(String catalogName, ResultSet rs) throws SQLException {
|
||||
return new TableId(catalogName.toUpperCase(), rs.getString(SEG_OWNER).toUpperCase(), rs.getString(TABLE_NAME).toUpperCase());
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,134 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.StringJoiner;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||
import io.debezium.connector.oracle.OracleDatabaseSchema;
|
||||
import io.debezium.relational.TableId;
|
||||
|
||||
/**
|
||||
* This utility class contains SQL statements to configure, manage and query Oracle LogMiner
|
||||
*/
|
||||
class SqlUtils {
|
||||
|
||||
private static final String LOGMNR_CONTENTS_VIEW = "V$LOGMNR_CONTENTS";
|
||||
private static final String LOGMNR_AUDIT_TABLE = "LOG_MINING_AUDIT";
|
||||
|
||||
static final String BUILD_DICTIONARY = "BEGIN DBMS_LOGMNR_D.BUILD (options => DBMS_LOGMNR_D.STORE_IN_REDO_LOGS); END;";
|
||||
static final String CURRENT_SCN = "SELECT CURRENT_SCN FROM V$DATABASE";
|
||||
static final String CURRENT_TIMESTAMP = "select current_timestamp from dual";
|
||||
static final String END_LOGMNR = "BEGIN SYS.DBMS_LOGMNR.END_LOGMNR(); END;";
|
||||
static final String OLDEST_FIRST_CHANGE = "SELECT MIN(FIRST_CHANGE#) FROM V$LOG";
|
||||
static final String ALL_ONLINE_LOGS = "SELECT MIN(F.MEMBER) AS FILE_NAME, L.NEXT_CHANGE# AS NEXT_CHANGE, F.GROUP# " +
|
||||
" FROM V$LOG L, V$LOGFILE F " +
|
||||
" WHERE F.GROUP# = L.GROUP# AND L.NEXT_CHANGE# > 0 " +
|
||||
" GROUP BY F.GROUP#, L.NEXT_CHANGE# ORDER BY 3";
|
||||
|
||||
static final String REDO_LOGS_STATUS = "SELECT F.MEMBER, R.STATUS FROM V$LOGFILE F, V$LOG R WHERE F.GROUP# = R.GROUP# ORDER BY 2";
|
||||
static final String SWITCH_HISTORY_TOTAL_COUNT = "select 'total', count(1) from v$archived_log where first_time > trunc(sysdate)" +
|
||||
" and dest_id = (select dest_id from V$ARCHIVE_DEST_STATUS where status='VALID' and type='LOCAL')";
|
||||
static final String CURRENT_REDO_LOG_NAME = "select f.member from v$log log, v$logfile f where log.group#=f.group# and log.status='CURRENT'";
|
||||
static final String AUDIT_TABLE_EXISTS = "SELECT '1' AS ONE FROM USER_TABLES WHERE TABLE_NAME = '" + LOGMNR_AUDIT_TABLE + "'";
|
||||
static final String AUDIT_TABLE_RECORD_EXISTS = "SELECT '1' AS ONE FROM " + LOGMNR_AUDIT_TABLE;
|
||||
static final String CREATE_AUDIT_TABLE = "CREATE TABLE " + LOGMNR_AUDIT_TABLE + "(LAST_SCN NUMBER(19,0))";
|
||||
static final String INSERT_AUDIT_TABLE = "INSERT INTO " + LOGMNR_AUDIT_TABLE + " VALUES(0)";
|
||||
static final String UPDATE_AUDIT_TABLE = "UPDATE " + LOGMNR_AUDIT_TABLE + " SET LAST_SCN =";
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SqlUtils.class);
|
||||
|
||||
// todo handle INVALID file member (report somehow and continue to work with valid file), handle adding multiplexed files,
|
||||
// todo SELECT name, value FROM v$sysstat WHERE name = 'redo wastage';
|
||||
// todo SELECT GROUP#, STATUS, MEMBER FROM V$LOGFILE WHERE STATUS='INVALID'; (drop and recreate? or do it manually?)
|
||||
// todo SELECT BLOCKSIZE FROM V$LOG;
|
||||
|
||||
static final String NLS_SESSION_PARAMETERS = "ALTER SESSION SET "
|
||||
+ " NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS'"
|
||||
+ " NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF'"
|
||||
+ " NLS_TIMESTAMP_TZ_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM'"
|
||||
+ " NLS_NUMERIC_CHARACTERS = '.,'";
|
||||
|
||||
/**
|
||||
* This returns statement to build log miner view for online redo log files
|
||||
* @param startScn mine from
|
||||
* @param endScn mine till
|
||||
* @param strategy Log Mining strategy
|
||||
* @return statement todo: handle corruption. STATUS (Double) — value of 0 indicates it is executable
|
||||
*/
|
||||
static String getStartLogMinerStatement(Long startScn, Long endScn, OracleConnectorConfig.LogMiningStrategy strategy, boolean isContinuousMining) {
|
||||
String miningStrategy;
|
||||
if (strategy.equals(OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO)) {
|
||||
miningStrategy = "DBMS_LOGMNR.DICT_FROM_REDO_LOGS + DBMS_LOGMNR.DDL_DICT_TRACKING ";
|
||||
}
|
||||
else {
|
||||
miningStrategy = "DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG ";
|
||||
}
|
||||
if (isContinuousMining) {
|
||||
miningStrategy += " + DBMS_LOGMNR.CONTINUOUS_MINE ";
|
||||
}
|
||||
return "BEGIN sys.dbms_logmnr.start_logmnr(" +
|
||||
"startScn => '" + startScn + "', " +
|
||||
"endScn => '" + endScn + "', " +
|
||||
"OPTIONS => " + miningStrategy +
|
||||
" + DBMS_LOGMNR.NO_ROWID_IN_STMT);" +
|
||||
"END;";
|
||||
}
|
||||
|
||||
/**
|
||||
* This is the query from the log miner view to get changes. Columns of the view we using are:
|
||||
* NOTE. Currently we do not capture changes from other schemas
|
||||
* SCN - The SCN at which a change was made
|
||||
* COMMIT_SCN - The SCN at which a change was committed
|
||||
* USERNAME - Name of the user who executed the transaction
|
||||
* SQL_REDO Reconstructed SQL statement that is equivalent to the original SQL statement that made the change
|
||||
* OPERATION_CODE - Number of the operation code.
|
||||
* TABLE_NAME - Name of the modified table
|
||||
* TIMESTAMP - Timestamp when the database change was made
|
||||
*
|
||||
* @param schemaName user name
|
||||
* @param logMinerUser log mining session user name
|
||||
* @param schema schema
|
||||
* @return the query
|
||||
*/
|
||||
static String queryLogMinerContents(String schemaName, String logMinerUser, OracleDatabaseSchema schema) {
|
||||
List<String> whiteListTableNames = schema.tableIds().stream().map(TableId::table).collect(Collectors.toList());
|
||||
|
||||
String sorting = "ORDER BY SCN";
|
||||
return "SELECT SCN, SQL_REDO, OPERATION_CODE, TIMESTAMP, XID, CSF, TABLE_NAME, SEG_OWNER, OPERATION, USERNAME " +
|
||||
" FROM " + LOGMNR_CONTENTS_VIEW + " WHERE OPERATION_CODE in (1,2,3,5) " + // 5 - DDL
|
||||
" AND SEG_OWNER = '" + schemaName.toUpperCase() + "' " +
|
||||
buildTableInPredicate(whiteListTableNames) +
|
||||
" AND SCN >= ? AND SCN < ? " +
|
||||
" OR (OPERATION_CODE IN (5,7,34,36) AND USERNAME NOT IN ('SYS','SYSTEM','" + logMinerUser.toUpperCase() + "'))" + sorting; // todo username = schemaName?
|
||||
}
|
||||
|
||||
static String getAddLogFileStatement(String option, String fileName) {
|
||||
return "BEGIN sys.dbms_logmnr.add_logfile(LOGFILENAME => '" + fileName + "', OPTIONS => " + option + ");END;";
|
||||
}
|
||||
|
||||
/**
|
||||
* This method builds table_name IN predicate, filtering out non whitelisted tables from Log Mining.
|
||||
* It limits joining condition over 1000 tables, Oracle will throw exception in such predicate.
|
||||
* @param tables white listed table names
|
||||
* @return IN predicate or empty string if number of whitelisted tables exceeds 1000
|
||||
*/
|
||||
private static String buildTableInPredicate(List<String> tables) {
|
||||
if (tables.size() == 0 || tables.size() > 1000) {
|
||||
LOGGER.warn(" Cannot apply {} whitelisted tables condition", tables.size());
|
||||
return "";
|
||||
}
|
||||
|
||||
StringJoiner tableNames = new StringJoiner(",");
|
||||
tables.forEach(table -> tableNames.add("'" + table + "'"));
|
||||
return " AND table_name IN (" + tableNames + ") AND SEG_NAME IN (" + tableNames + ") ";
|
||||
}
|
||||
}
|
@ -0,0 +1,399 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.apache.kafka.connect.errors.DataException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.annotation.NotThreadSafe;
|
||||
import io.debezium.connector.oracle.OracleConnector;
|
||||
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||
import io.debezium.pipeline.ErrorHandler;
|
||||
import io.debezium.pipeline.source.spi.ChangeEventSource;
|
||||
import io.debezium.util.Threads;
|
||||
|
||||
/**
|
||||
* @author Andrey Pustovetov
|
||||
* <p>
|
||||
* Transactional buffer is designed to register callbacks, to execute them when transaction commits and to clear them
|
||||
* when transaction rollbacks.
|
||||
*/
|
||||
@NotThreadSafe
|
||||
public final class TransactionalBuffer {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TransactionalBuffer.class);
|
||||
|
||||
private final Map<String, Transaction> transactions;
|
||||
private final ExecutorService executor;
|
||||
private final AtomicInteger taskCounter;
|
||||
private final ErrorHandler errorHandler;
|
||||
private final Supplier<Integer> commitQueueCapacity;
|
||||
private TransactionalBufferMetrics metrics;
|
||||
private final Set<String> abandonedTransactionIds;
|
||||
|
||||
// storing rolledBackTransactionIds is for debugging purposes to check what was rolled back to research, todo delete in future releases
|
||||
private final Set<String> rolledBackTransactionIds;
|
||||
|
||||
// It holds the latest captured SCN.
|
||||
// This number tracks starting point for the next mining cycle.
|
||||
private BigDecimal largestScn;
|
||||
private BigDecimal lastCommittedScn;
|
||||
|
||||
/**
|
||||
* Constructor to create a new instance.
|
||||
*
|
||||
* @param logicalName logical name
|
||||
* @param errorHandler logError handler
|
||||
* @param metrics metrics MBean
|
||||
* @param inCommitQueueCapacity commit queue capacity. On overflow, caller runs task
|
||||
*/
|
||||
TransactionalBuffer(String logicalName, ErrorHandler errorHandler, TransactionalBufferMetrics metrics, int inCommitQueueCapacity) {
|
||||
this.transactions = new HashMap<>();
|
||||
final BlockingQueue<Runnable> workQueue = new ArrayBlockingQueue<>(inCommitQueueCapacity);
|
||||
executor = new ThreadPoolExecutor(1, 1,
|
||||
Integer.MAX_VALUE, TimeUnit.MILLISECONDS,
|
||||
workQueue,
|
||||
Threads.threadFactory(OracleConnector.class, logicalName, "transactional-buffer", true, false),
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
commitQueueCapacity = workQueue::remainingCapacity;
|
||||
this.taskCounter = new AtomicInteger();
|
||||
this.errorHandler = errorHandler;
|
||||
this.metrics = metrics;
|
||||
largestScn = BigDecimal.ZERO;
|
||||
lastCommittedScn = BigDecimal.ZERO;
|
||||
this.abandonedTransactionIds = new HashSet<>();
|
||||
this.rolledBackTransactionIds = new HashSet<>();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return largest last SCN in the buffer among all transactions
|
||||
*/
|
||||
BigDecimal getLargestScn() {
|
||||
return largestScn;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return rolled back transactions
|
||||
*/
|
||||
Set<String> getRolledBackTransactionIds() {
|
||||
return new HashSet<>(rolledBackTransactionIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset Largest SCN
|
||||
*/
|
||||
void resetLargestScn(Long value) {
|
||||
if (value != null) {
|
||||
largestScn = new BigDecimal(value);
|
||||
}
|
||||
else {
|
||||
largestScn = BigDecimal.ZERO;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers callback to execute when transaction commits.
|
||||
*
|
||||
* @param transactionId transaction identifier
|
||||
* @param scn SCN
|
||||
* @param changeTime time of DML parsing completion
|
||||
* @param redoSql statement from redo
|
||||
* @param callback callback to execute when transaction commits
|
||||
*/
|
||||
void registerCommitCallback(String transactionId, BigDecimal scn, Instant changeTime, String redoSql, CommitCallback callback) {
|
||||
if (abandonedTransactionIds.contains(transactionId)) {
|
||||
LogMinerHelper.logWarn(metrics, "Another DML for an abandoned transaction {} : {}, ignored", transactionId, redoSql);
|
||||
return;
|
||||
}
|
||||
|
||||
transactions.computeIfAbsent(transactionId, s -> new Transaction(scn));
|
||||
|
||||
metrics.setActiveTransactions(transactions.size());
|
||||
metrics.incrementCapturedDmlCounter();
|
||||
metrics.calculateLagMetrics(changeTime);
|
||||
|
||||
// The transaction object is not a lightweight object anymore having all REDO_SQL stored.
|
||||
Transaction transaction = transactions.get(transactionId);
|
||||
if (transaction != null) {
|
||||
|
||||
// todo this should never happen, delete when tested and confirmed
|
||||
if (rolledBackTransactionIds.contains(transactionId)) {
|
||||
LogMinerHelper.logWarn(metrics, "Ignore DML for rolled back transaction: SCN={}, REDO_SQL={}", scn, redoSql);
|
||||
return;
|
||||
}
|
||||
|
||||
transaction.commitCallbacks.add(callback);
|
||||
transaction.addRedoSql(scn, redoSql);
|
||||
}
|
||||
|
||||
if (scn.compareTo(largestScn) > 0) {
|
||||
largestScn = scn;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* If the commit executor queue is full, back-pressure will be applied by letting execution of the callback
|
||||
* be performed by the calling thread.
|
||||
*
|
||||
* @param transactionId transaction identifier
|
||||
* @param scn SCN of the commit.
|
||||
* @param offsetContext Oracle offset
|
||||
* @param timestamp commit timestamp
|
||||
* @param context context to check that source is running
|
||||
* @param debugMessage message
|
||||
* @return true if committed transaction is in the buffer, was not processed yet and processed now
|
||||
*/
|
||||
boolean commit(String transactionId, BigDecimal scn, OracleOffsetContext offsetContext, Timestamp timestamp,
|
||||
ChangeEventSource.ChangeEventSourceContext context, String debugMessage) {
|
||||
|
||||
Transaction transaction = transactions.get(transactionId);
|
||||
if (transaction == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
calculateLargestScn();
|
||||
transaction = transactions.remove(transactionId);
|
||||
BigDecimal smallestScn = calculateSmallestScn();
|
||||
|
||||
taskCounter.incrementAndGet();
|
||||
abandonedTransactionIds.remove(transactionId);
|
||||
|
||||
// On the restarting connector, we start from SCN in the offset. There is possibility to commit a transaction(s) which were already committed.
|
||||
// Currently we cannot use ">=", because we may lose normal commit which may happen at the same time. TODO use audit table to prevent duplications
|
||||
if ((offsetContext.getCommitScn() != null && offsetContext.getCommitScn() > scn.longValue()) || lastCommittedScn.longValue() > scn.longValue()) {
|
||||
LogMinerHelper.logWarn(metrics,
|
||||
"Transaction {} was already processed, ignore. Committed SCN in offset is {}, commit SCN of the transaction is {}, last committed SCN is {}",
|
||||
transactionId, offsetContext.getCommitScn(), scn, lastCommittedScn);
|
||||
metrics.setActiveTransactions(transactions.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
List<CommitCallback> commitCallbacks = transaction.commitCallbacks;
|
||||
LOGGER.trace("COMMIT, {}, smallest SCN: {}, largest SCN {}", debugMessage, smallestScn, largestScn);
|
||||
executor.execute(() -> {
|
||||
try {
|
||||
int counter = commitCallbacks.size();
|
||||
for (CommitCallback callback : commitCallbacks) {
|
||||
if (!context.isRunning()) {
|
||||
return;
|
||||
}
|
||||
callback.execute(timestamp, smallestScn, scn, --counter);
|
||||
}
|
||||
|
||||
lastCommittedScn = new BigDecimal(scn.longValue());
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
LogMinerHelper.logError(metrics, "Thread interrupted during running", e);
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
catch (Exception e) {
|
||||
errorHandler.setProducerThrowable(e);
|
||||
}
|
||||
finally {
|
||||
metrics.incrementCommittedTransactions();
|
||||
metrics.setActiveTransactions(transactions.size());
|
||||
metrics.incrementCommittedDmlCounter(commitCallbacks.size());
|
||||
metrics.setCommittedScn(scn.longValue());
|
||||
metrics.setCommitQueueCapacity(commitQueueCapacity.get());
|
||||
taskCounter.decrementAndGet();
|
||||
}
|
||||
});
|
||||
metrics.setCommitQueueCapacity(commitQueueCapacity.get());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears registered callbacks for given transaction identifier.
|
||||
*
|
||||
* @param transactionId transaction id
|
||||
* @param debugMessage message
|
||||
* @return true if the rollback is for a transaction in the buffer
|
||||
*/
|
||||
boolean rollback(String transactionId, String debugMessage) {
|
||||
|
||||
Transaction transaction = transactions.get(transactionId);
|
||||
if (transaction != null) {
|
||||
LOGGER.debug("Transaction rolled back, {} , Statements: {}", debugMessage, transaction.redoSqlMap.values().toArray());
|
||||
|
||||
calculateLargestScn(); // in case if largest SCN was in this transaction
|
||||
transactions.remove(transactionId);
|
||||
|
||||
abandonedTransactionIds.remove(transactionId);
|
||||
rolledBackTransactionIds.add(transactionId);
|
||||
|
||||
metrics.setActiveTransactions(transactions.size());
|
||||
metrics.incrementRolledBackTransactions();
|
||||
metrics.addRolledBackTransactionId(transactionId); // todo decide if we need both metrics
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* If for some reason the connector got restarted, the offset will point to the beginning of the oldest captured transaction.
|
||||
* Taking in consideration offset flush interval, the offset could be even older.
|
||||
* If that transaction was lasted for a long time, let say > 30 minutes, the offset will be not accessible after restart,
|
||||
* because we don't mine archived logs, neither rely on continuous_mine configuration option.
|
||||
* Hence we have to address these cases manually.
|
||||
* <p>
|
||||
* It is limited by following condition:
|
||||
* allOnlineRedoLogFiles.size() - currentlyMinedLogFiles.size() <= 1
|
||||
* <p>
|
||||
* If each redo lasts for 10 minutes and 7 redo group have been configured, any transaction cannot lasts longer than 1 hour.
|
||||
* <p>
|
||||
* In case of an abandonment, all DMLs/Commits/Rollbacs for this transaction will be ignored
|
||||
* <p>
|
||||
* In other words connector will not send any part of this transaction to Kafka
|
||||
*
|
||||
* @param thresholdScn the smallest SVN of any transaction to keep in the buffer. All others will be removed.
|
||||
*/
|
||||
void abandonLongTransactions(Long thresholdScn) {
|
||||
BigDecimal threshold = new BigDecimal(thresholdScn);
|
||||
Iterator<Map.Entry<String, Transaction>> iter = transactions.entrySet().iterator();
|
||||
while (iter.hasNext()) {
|
||||
Map.Entry<String, Transaction> transaction = iter.next();
|
||||
if (transaction.getValue().firstScn.compareTo(threshold) <= 0) {
|
||||
LogMinerHelper.logWarn(metrics, "Following long running transaction {} will be abandoned and ignored: {} ", transaction.getKey(),
|
||||
transaction.getValue().toString());
|
||||
abandonedTransactionIds.add(transaction.getKey());
|
||||
iter.remove();
|
||||
calculateLargestScn();
|
||||
|
||||
metrics.addAbandonedTransactionId(transaction.getKey());
|
||||
metrics.setActiveTransactions(transactions.size());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private BigDecimal calculateSmallestScn() {
|
||||
BigDecimal scn = transactions.isEmpty() ? null
|
||||
: transactions.values()
|
||||
.stream()
|
||||
.map(transaction -> transaction.firstScn)
|
||||
.min(BigDecimal::compareTo)
|
||||
.orElseThrow(() -> new DataException("Cannot calculate smallest SCN"));
|
||||
metrics.setOldestScn(scn == null ? -1 : scn.longValue());
|
||||
return scn;
|
||||
}
|
||||
|
||||
private void calculateLargestScn() {
|
||||
largestScn = transactions.isEmpty() ? BigDecimal.ZERO
|
||||
: transactions.values()
|
||||
.stream()
|
||||
.map(transaction -> transaction.lastScn)
|
||||
.max(BigDecimal::compareTo)
|
||||
.orElseThrow(() -> new DataException("Cannot calculate largest SCN"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns {@code true} if buffer is empty, otherwise {@code false}.
|
||||
*
|
||||
* @return {@code true} if buffer is empty, otherwise {@code false}
|
||||
*/
|
||||
boolean isEmpty() {
|
||||
return transactions.isEmpty() && taskCounter.get() == 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder result = new StringBuilder();
|
||||
this.transactions.values().forEach(t -> result.append(t.toString()));
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes buffer.
|
||||
*/
|
||||
void close() {
|
||||
transactions.clear();
|
||||
executor.shutdown();
|
||||
try {
|
||||
if (!executor.awaitTermination(1000L, TimeUnit.MILLISECONDS)) {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
LogMinerHelper.logError(metrics, "Thread interrupted during shutdown", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Callback is designed to execute when transaction commits.
|
||||
*/
|
||||
public interface CommitCallback {
|
||||
|
||||
/**
|
||||
* Executes callback.
|
||||
*
|
||||
* @param timestamp commit timestamp
|
||||
* @param smallestScn smallest SCN among other transactions
|
||||
* @param commitScn commit SCN
|
||||
* @param callbackNumber number of the callback in the transaction
|
||||
*/
|
||||
void execute(Timestamp timestamp, BigDecimal smallestScn, BigDecimal commitScn, int callbackNumber) throws InterruptedException;
|
||||
}
|
||||
|
||||
@NotThreadSafe
|
||||
private static final class Transaction {
|
||||
|
||||
private final BigDecimal firstScn;
|
||||
private BigDecimal lastScn;
|
||||
private final List<CommitCallback> commitCallbacks;
|
||||
private final Map<BigDecimal, List<String>> redoSqlMap;
|
||||
|
||||
private Transaction(BigDecimal firstScn) {
|
||||
this.firstScn = firstScn;
|
||||
this.commitCallbacks = new ArrayList<>();
|
||||
this.redoSqlMap = new HashMap<>();
|
||||
this.lastScn = firstScn;
|
||||
}
|
||||
|
||||
private void addRedoSql(BigDecimal scn, String redoSql) {
|
||||
this.lastScn = scn;
|
||||
|
||||
List<String> sqlList = redoSqlMap.get(scn);
|
||||
if (sqlList == null) {
|
||||
redoSqlMap.put(scn, new ArrayList<>(Collections.singletonList(redoSql)));
|
||||
}
|
||||
else {
|
||||
sqlList.add(redoSql);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Transaction{" +
|
||||
"firstScn=" + firstScn +
|
||||
", lastScn=" + lastScn +
|
||||
", redoSqls=" + Arrays.toString(redoSqlMap.values().toArray()) +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,284 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import io.debezium.annotation.ThreadSafe;
|
||||
import io.debezium.connector.common.CdcSourceTaskContext;
|
||||
import io.debezium.metrics.Metrics;
|
||||
|
||||
/**
|
||||
* This class contains MBean methods
|
||||
*/
|
||||
@ThreadSafe
|
||||
public class TransactionalBufferMetrics extends Metrics implements TransactionalBufferMetricsMXBean {
|
||||
private AtomicLong oldestScn = new AtomicLong();
|
||||
private AtomicLong committedScn = new AtomicLong();
|
||||
private AtomicReference<Duration> lagFromTheSource = new AtomicReference<>();
|
||||
private AtomicInteger activeTransactions = new AtomicInteger();
|
||||
private AtomicLong rolledBackTransactions = new AtomicLong();
|
||||
private AtomicLong committedTransactions = new AtomicLong();
|
||||
private AtomicLong capturedDmlCounter = new AtomicLong();
|
||||
private AtomicLong committedDmlCounter = new AtomicLong();
|
||||
private AtomicInteger commitQueueCapacity = new AtomicInteger();
|
||||
private AtomicReference<Duration> maxLagFromTheSource = new AtomicReference<>();
|
||||
private AtomicReference<Duration> minLagFromTheSource = new AtomicReference<>();
|
||||
private AtomicReference<Duration> averageLagsFromTheSource = new AtomicReference<>();
|
||||
private AtomicReference<Set<String>> abandonedTransactionIds = new AtomicReference<>();
|
||||
private AtomicReference<Set<String>> rolledBackTransactionIds = new AtomicReference<>();
|
||||
private Instant startTime;
|
||||
private static long MILLIS_PER_SECOND = 1000L;
|
||||
private AtomicLong timeDifference = new AtomicLong();
|
||||
private AtomicInteger errorCounter = new AtomicInteger();
|
||||
private AtomicInteger warningCounter = new AtomicInteger();
|
||||
private AtomicInteger scnFreezeCounter = new AtomicInteger();
|
||||
|
||||
TransactionalBufferMetrics(CdcSourceTaskContext taskContext) {
|
||||
super(taskContext, "log-miner-transactional-buffer");
|
||||
startTime = Instant.now();
|
||||
oldestScn.set(-1);
|
||||
committedScn.set(-1);
|
||||
timeDifference.set(0);
|
||||
reset();
|
||||
}
|
||||
|
||||
// setters
|
||||
void setOldestScn(Long scn) {
|
||||
oldestScn.set(scn);
|
||||
}
|
||||
|
||||
public void setCommittedScn(Long scn) {
|
||||
committedScn.set(scn);
|
||||
}
|
||||
|
||||
public void setTimeDifference(AtomicLong timeDifference) {
|
||||
this.timeDifference = timeDifference;
|
||||
}
|
||||
|
||||
void calculateLagMetrics(Instant changeTime) {
|
||||
if (changeTime != null) {
|
||||
Instant correctedChangeTime = changeTime.plus(Duration.ofMillis(timeDifference.longValue()));
|
||||
lagFromTheSource.set(Duration.between(correctedChangeTime, Instant.now()).abs());
|
||||
|
||||
if (maxLagFromTheSource.get().toMillis() < lagFromTheSource.get().toMillis()) {
|
||||
maxLagFromTheSource.set(lagFromTheSource.get());
|
||||
}
|
||||
if (minLagFromTheSource.get().toMillis() > lagFromTheSource.get().toMillis()) {
|
||||
minLagFromTheSource.set(lagFromTheSource.get());
|
||||
}
|
||||
|
||||
if (averageLagsFromTheSource.get().isZero()) {
|
||||
averageLagsFromTheSource.set(lagFromTheSource.get());
|
||||
}
|
||||
else {
|
||||
averageLagsFromTheSource.set(averageLagsFromTheSource.get().plus(lagFromTheSource.get()).dividedBy(2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void setActiveTransactions(Integer counter) {
|
||||
if (counter != null) {
|
||||
activeTransactions.set(counter);
|
||||
}
|
||||
}
|
||||
|
||||
void incrementRolledBackTransactions() {
|
||||
rolledBackTransactions.incrementAndGet();
|
||||
}
|
||||
|
||||
void incrementCommittedTransactions() {
|
||||
committedTransactions.incrementAndGet();
|
||||
}
|
||||
|
||||
void incrementCapturedDmlCounter() {
|
||||
capturedDmlCounter.incrementAndGet();
|
||||
}
|
||||
|
||||
void incrementCommittedDmlCounter(int counter) {
|
||||
committedDmlCounter.getAndAdd(counter);
|
||||
}
|
||||
|
||||
void addAbandonedTransactionId(String transactionId) {
|
||||
if (transactionId != null) {
|
||||
abandonedTransactionIds.get().add(transactionId);
|
||||
}
|
||||
}
|
||||
|
||||
void addRolledBackTransactionId(String transactionId) {
|
||||
if (transactionId != null) {
|
||||
rolledBackTransactionIds.get().add(transactionId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This is to increase logged logError counter.
|
||||
* There are other ways to monitor the log, but this is just to check if there are any.
|
||||
*/
|
||||
void incrementErrorCounter() {
|
||||
errorCounter.incrementAndGet();
|
||||
}
|
||||
|
||||
/**
|
||||
* This is to increase logged warning counter
|
||||
* There are other ways to monitor the log, but this is just to check if there are any.
|
||||
*/
|
||||
void incrementWarningCounter() {
|
||||
warningCounter.incrementAndGet();
|
||||
}
|
||||
|
||||
/**
|
||||
* This counter to accumulate number of encountered observations when SCN does not change in the offset.
|
||||
* This call indicates an uncommitted oldest transaction in the buffer.
|
||||
*/
|
||||
void incrementScnFreezeCounter() {
|
||||
scnFreezeCounter.incrementAndGet();
|
||||
}
|
||||
|
||||
// implemented getters
|
||||
@Override
|
||||
public Long getOldestScn() {
|
||||
return oldestScn.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getCommittedScn() {
|
||||
return committedScn.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumberOfActiveTransactions() {
|
||||
return activeTransactions.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberOfRolledBackTransactions() {
|
||||
return rolledBackTransactions.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberOfCommittedTransactions() {
|
||||
return committedTransactions.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getCommitThroughput() {
|
||||
long timeSpent = Duration.between(startTime, Instant.now()).isZero() ? 1 : Duration.between(startTime, Instant.now()).toMillis();
|
||||
return committedTransactions.get() * MILLIS_PER_SECOND / timeSpent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getCapturedDmlThroughput() {
|
||||
long timeSpent = Duration.between(startTime, Instant.now()).isZero() ? 1 : Duration.between(startTime, Instant.now()).toMillis();
|
||||
return committedDmlCounter.get() * MILLIS_PER_SECOND / timeSpent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getCapturedDmlCount() {
|
||||
return capturedDmlCounter.longValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getLagFromSource() {
|
||||
return lagFromTheSource.get().toMillis();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxLagFromSource() {
|
||||
return maxLagFromTheSource.get().toMillis();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMinLagFromSource() {
|
||||
return minLagFromTheSource.get().toMillis();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getAverageLagFromSource() {
|
||||
return averageLagsFromTheSource.get().toMillis();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getAbandonedTransactionIds() {
|
||||
return abandonedTransactionIds.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getRolledBackTransactionIds() {
|
||||
return rolledBackTransactionIds.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getErrorCounter() {
|
||||
return errorCounter.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getWarningCounter() {
|
||||
return warningCounter.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getScnFreezeCounter() {
|
||||
return scnFreezeCounter.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getCommitQueueCapacity() {
|
||||
return commitQueueCapacity.get();
|
||||
}
|
||||
|
||||
void setCommitQueueCapacity(int commitQueueCapacity) {
|
||||
this.commitQueueCapacity.set(commitQueueCapacity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reset() {
|
||||
maxLagFromTheSource.set(Duration.ZERO);
|
||||
minLagFromTheSource.set(Duration.ZERO);
|
||||
averageLagsFromTheSource.set(Duration.ZERO);
|
||||
activeTransactions.set(0);
|
||||
rolledBackTransactions.set(0);
|
||||
committedTransactions.set(0);
|
||||
capturedDmlCounter.set(0);
|
||||
committedDmlCounter.set(0);
|
||||
abandonedTransactionIds.set(new HashSet<>());
|
||||
rolledBackTransactionIds.set(new HashSet<>());
|
||||
lagFromTheSource.set(Duration.ZERO);
|
||||
errorCounter.set(0);
|
||||
warningCounter.set(0);
|
||||
scnFreezeCounter.set(0);
|
||||
commitQueueCapacity.set(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TransactionalBufferMetrics{" +
|
||||
"oldestScn=" + oldestScn.get() +
|
||||
", committedScn=" + committedScn.get() +
|
||||
", lagFromTheSource=" + lagFromTheSource.get() +
|
||||
", activeTransactions=" + activeTransactions.get() +
|
||||
", rolledBackTransactions=" + rolledBackTransactions.get() +
|
||||
", committedTransactions=" + committedTransactions.get() +
|
||||
", capturedDmlCounter=" + capturedDmlCounter.get() +
|
||||
", committedDmlCounter=" + committedDmlCounter.get() +
|
||||
", maxLagFromTheSource=" + maxLagFromTheSource.get() +
|
||||
", minLagFromTheSource=" + minLagFromTheSource.get() +
|
||||
", averageLagsFromTheSource=" + averageLagsFromTheSource.get() +
|
||||
", abandonedTransactionIds=" + abandonedTransactionIds.get() +
|
||||
", errorCounter=" + errorCounter.get() +
|
||||
", warningCounter=" + warningCounter.get() +
|
||||
", scnFreezeCounter=" + scnFreezeCounter.get() +
|
||||
", commitQueueCapacity=" + commitQueueCapacity.get() +
|
||||
'}';
|
||||
}
|
||||
}
|
@ -0,0 +1,146 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* This interface exposes TransactionalBuffer metrics
|
||||
*/
|
||||
public interface TransactionalBufferMetricsMXBean {
|
||||
|
||||
/**
|
||||
* Exposes number of rolled back transactions
|
||||
*
|
||||
* @return number of rolled back transaction in the in-memory buffer
|
||||
*/
|
||||
long getNumberOfRolledBackTransactions();
|
||||
|
||||
/**
|
||||
* Exposes number of committed transactions
|
||||
*
|
||||
* @return number of committed transaction in the in-memory buffer
|
||||
*/
|
||||
long getNumberOfCommittedTransactions();
|
||||
|
||||
/**
|
||||
* Exposes average number of committed transactions per second
|
||||
*
|
||||
* @return average number of committed transactions per second in the in-memory buffer
|
||||
*/
|
||||
long getCommitThroughput();
|
||||
|
||||
/**
|
||||
* Exposes average number of captured and parsed DML per second
|
||||
*
|
||||
* @return average number of captured and parsed DML per second in the in-memory buffer
|
||||
*/
|
||||
long getCapturedDmlThroughput();
|
||||
|
||||
/**
|
||||
* exposes total number of captured DMLs
|
||||
*
|
||||
* @return captured DML count
|
||||
*/
|
||||
long getCapturedDmlCount();
|
||||
|
||||
/**
|
||||
* Exposes number of transaction, buffered in memory
|
||||
*
|
||||
* @return number of currently buffered transactions
|
||||
*/
|
||||
int getNumberOfActiveTransactions();
|
||||
|
||||
/**
|
||||
* Exposes the oldest(smallest) SCN in the Transactional Buffer
|
||||
*
|
||||
* @return oldest SCN
|
||||
*/
|
||||
Long getOldestScn();
|
||||
|
||||
/**
|
||||
* It shows last committed SCN
|
||||
*
|
||||
* @return committed SCN
|
||||
*/
|
||||
Long getCommittedScn();
|
||||
|
||||
/**
|
||||
* This is to get the lag between latest captured change timestamp in REDO LOG and time of it's placement in the buffer
|
||||
*
|
||||
* @return lag in milliseconds
|
||||
*/
|
||||
long getLagFromSource();
|
||||
|
||||
/**
|
||||
* This is to get max value of the time difference between logging of source DB records into redo log and capturing it by Log Miner
|
||||
*
|
||||
* @return value in milliseconds
|
||||
*/
|
||||
long getMaxLagFromSource();
|
||||
|
||||
/**
|
||||
* This is to get min value of the time difference between logging of source DB records into redo log and capturing it by Log Miner
|
||||
*
|
||||
* @return value in milliseconds
|
||||
*/
|
||||
long getMinLagFromSource();
|
||||
|
||||
/**
|
||||
* This is to get average value of the time difference between logging of source DB records into redo log and capturing it by Log Miner.
|
||||
* Average is calculated as summary of all lags / number of captured DB changes
|
||||
*
|
||||
* @return value in milliseconds
|
||||
*/
|
||||
long getAverageLagFromSource();
|
||||
|
||||
/**
|
||||
* This is to get list of removed transactions from the Transactional Buffer
|
||||
*
|
||||
* @return count abandoned transaction ids
|
||||
*/
|
||||
Set<String> getAbandonedTransactionIds();
|
||||
|
||||
/**
|
||||
* See which transactions were rolled back
|
||||
*
|
||||
* @return set of transaction IDs
|
||||
*/
|
||||
Set<String> getRolledBackTransactionIds();
|
||||
|
||||
/**
|
||||
* Gets commit queue capacity. As the queue fills up, this reduces to zero
|
||||
*
|
||||
* @return the commit queue capacity
|
||||
*/
|
||||
int getCommitQueueCapacity();
|
||||
|
||||
/**
|
||||
* action to reset some metrics
|
||||
*/
|
||||
void reset();
|
||||
|
||||
/**
|
||||
* This is to get logged logError counter.
|
||||
*
|
||||
* @return the error counter
|
||||
*/
|
||||
int getErrorCounter();
|
||||
|
||||
/**
|
||||
* This is to get logged warning counter
|
||||
*
|
||||
* @return the warning counter
|
||||
*/
|
||||
int getWarningCounter();
|
||||
|
||||
/**
|
||||
* Get counter of encountered observations when SCN does not change in the offset.
|
||||
*
|
||||
* @return the scn freeze counter
|
||||
*/
|
||||
int getScnFreezeCounter();
|
||||
}
|
@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer.valueholder;
|
||||
|
||||
public interface LogMinerColumnValue {
|
||||
|
||||
/**
|
||||
* @return value of the database record
|
||||
* with exception of LOB types
|
||||
*/
|
||||
Object getColumnData();
|
||||
|
||||
/**
|
||||
* @return column name
|
||||
*/
|
||||
String getColumnName();
|
||||
|
||||
/**
|
||||
* This sets the database record value with the exception of LOBs
|
||||
* @param columnData data
|
||||
*/
|
||||
void setColumnData(Object columnData);
|
||||
}
|
@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer.valueholder;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import io.debezium.connector.oracle.antlr.listener.ParserUtils;
|
||||
|
||||
/**
|
||||
* This class stores parsed column info
|
||||
*
|
||||
*/
|
||||
public class LogMinerColumnValueImpl implements LogMinerColumnValue {
|
||||
|
||||
private String columnName;
|
||||
private Object columnData;
|
||||
private int columnType;
|
||||
|
||||
public LogMinerColumnValueImpl(String columnName, int columnType) {
|
||||
this.columnName = columnName;
|
||||
this.columnType = columnType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getColumnData() {
|
||||
return columnData;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getColumnName() {
|
||||
return columnName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setColumnData(Object columnData) {
|
||||
if (columnData instanceof String) {
|
||||
this.columnData = ParserUtils.replaceDoubleBackSlashes((String) columnData);
|
||||
}
|
||||
else {
|
||||
this.columnData = columnData;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
LogMinerColumnValueImpl that = (LogMinerColumnValueImpl) o;
|
||||
return columnType == that.columnType &&
|
||||
Objects.equals(columnName, that.columnName) &&
|
||||
Objects.equals(columnData, that.columnData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(columnName, columnData, columnType);
|
||||
}
|
||||
}
|
@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer.valueholder;
|
||||
|
||||
/**
|
||||
* This class is a wrapper class which holds LogMinerColumnValue
|
||||
* and the indicator if the column was processed by a parser listener.
|
||||
* The "processed" is "true" means a listener has parsed a value.
|
||||
* The "false" value means the this value was not parsed yet
|
||||
* The "processed" flag helps to filter the resulting collection of "new" and "old" values.
|
||||
*
|
||||
*/
|
||||
public class LogMinerColumnValueWrapper {
|
||||
|
||||
private boolean processed;
|
||||
private final LogMinerColumnValue columnValue;
|
||||
|
||||
public LogMinerColumnValueWrapper(LogMinerColumnValue columnValue) {
|
||||
this.columnValue = columnValue;
|
||||
}
|
||||
|
||||
public LogMinerColumnValue getColumnValue() {
|
||||
return columnValue;
|
||||
}
|
||||
|
||||
public boolean isProcessed() {
|
||||
return processed;
|
||||
}
|
||||
|
||||
public void setProcessed(boolean processed) {
|
||||
this.processed = processed;
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer.valueholder;
|
||||
|
||||
/**
|
||||
* This class is a placeholder of DDL data
|
||||
*
|
||||
*/
|
||||
public interface LogMinerDdlEntry {
|
||||
/**
|
||||
* @return text of the DDL statement
|
||||
*/
|
||||
String getDdlText();
|
||||
|
||||
/**
|
||||
* @return string such as "CREATE TABLE", "ALTER TABLE", "DROP TABLE"
|
||||
*/
|
||||
String getCommandType();
|
||||
}
|
@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer.valueholder;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.List;
|
||||
|
||||
import io.debezium.data.Envelope;
|
||||
|
||||
public interface LogMinerDmlEntry {
|
||||
/**
|
||||
* This getter
|
||||
* @return old(current) values of the database record.
|
||||
* They represent values in WHERE clauses
|
||||
*/
|
||||
List<LogMinerColumnValue> getOldValues();
|
||||
|
||||
/**
|
||||
* this getter
|
||||
* @return new values to be applied to the database record
|
||||
* Those values are applicable for INSERT and UPDATE statements
|
||||
*/
|
||||
List<LogMinerColumnValue> getNewValues();
|
||||
|
||||
/**
|
||||
* this getter
|
||||
* @return Envelope.Operation enum
|
||||
*/
|
||||
Envelope.Operation getCommandType();
|
||||
|
||||
/**
|
||||
* the scn obtained from a Log Miner entry.
|
||||
* This SCN is not a final SCN, just a candidate.
|
||||
* The actual SCN will be assigned after commit
|
||||
* @return it's value
|
||||
*/
|
||||
BigDecimal getScn();
|
||||
|
||||
/**
|
||||
* @return transaction ID
|
||||
*/
|
||||
String getTransactionId();
|
||||
|
||||
/**
|
||||
* @return schema name
|
||||
*/
|
||||
String getObjectOwner();
|
||||
|
||||
/**
|
||||
* @return table name
|
||||
*/
|
||||
String getObjectName();
|
||||
|
||||
/**
|
||||
* @return database change time of this logical record
|
||||
*/
|
||||
Timestamp getSourceTime();
|
||||
|
||||
/**
|
||||
* sets scn obtained from a Log Miner entry
|
||||
* @param scn it's value
|
||||
*/
|
||||
void setScn(BigDecimal scn);
|
||||
|
||||
/**
|
||||
* Sets table name
|
||||
* @param name table name
|
||||
*/
|
||||
void setObjectName(String name);
|
||||
|
||||
/**
|
||||
* Sets schema owner
|
||||
* @param name schema owner
|
||||
*/
|
||||
void setObjectOwner(String name);
|
||||
|
||||
/**
|
||||
* Sets the time of the database change
|
||||
* @param changeTime the time of the change
|
||||
*/
|
||||
void setSourceTime(Timestamp changeTime);
|
||||
|
||||
/**
|
||||
* @param id unique transaction ID
|
||||
*/
|
||||
void setTransactionId(String id);
|
||||
|
||||
}
|
@ -0,0 +1,119 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer.valueholder;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import io.debezium.data.Envelope;
|
||||
|
||||
/**
|
||||
* This class holds one parsed DML Log Miner record details
|
||||
*
|
||||
*/
|
||||
public class LogMinerDmlEntryImpl implements LogMinerDmlEntry {
|
||||
|
||||
private Envelope.Operation commandType;
|
||||
private List<LogMinerColumnValue> newLmColumnValues;
|
||||
private List<LogMinerColumnValue> oldLmColumnValues;
|
||||
private String objectOwner;
|
||||
private String objectName;
|
||||
private Timestamp sourceTime;
|
||||
private String transactionId;
|
||||
private BigDecimal scn;
|
||||
|
||||
public LogMinerDmlEntryImpl(Envelope.Operation commandType, List<LogMinerColumnValue> newLmColumnValues, List<LogMinerColumnValue> oldLmColumnValues) {
|
||||
this.commandType = commandType;
|
||||
this.newLmColumnValues = newLmColumnValues;
|
||||
this.oldLmColumnValues = oldLmColumnValues;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Envelope.Operation getCommandType() {
|
||||
return commandType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<LogMinerColumnValue> getOldValues() {
|
||||
return oldLmColumnValues;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<LogMinerColumnValue> getNewValues() {
|
||||
return newLmColumnValues;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTransactionId() {
|
||||
return transactionId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getObjectOwner() {
|
||||
return objectOwner;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getObjectName() {
|
||||
return objectName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Timestamp getSourceTime() {
|
||||
return sourceTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setObjectName(String name) {
|
||||
this.objectName = name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setObjectOwner(String name) {
|
||||
this.objectOwner = name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setSourceTime(Timestamp changeTime) {
|
||||
this.sourceTime = changeTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTransactionId(String id) {
|
||||
this.transactionId = id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BigDecimal getScn() {
|
||||
return scn;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setScn(BigDecimal scn) {
|
||||
this.scn = scn;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
LogMinerDmlEntryImpl that = (LogMinerDmlEntryImpl) o;
|
||||
return commandType == that.commandType &&
|
||||
Objects.equals(newLmColumnValues, that.newLmColumnValues) &&
|
||||
Objects.equals(oldLmColumnValues, that.oldLmColumnValues);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(commandType, newLmColumnValues, oldLmColumnValues);
|
||||
}
|
||||
}
|
@ -3,11 +3,12 @@
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle;
|
||||
package io.debezium.connector.oracle.xstream;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||
import io.debezium.pipeline.ErrorHandler;
|
||||
import io.debezium.pipeline.EventDispatcher;
|
||||
import io.debezium.relational.RelationalDatabaseSchema;
|
||||
@ -28,7 +29,7 @@
|
||||
*/
|
||||
class LcrEventHandler implements XStreamLCRCallbackHandler {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(OracleStreamingChangeEventSource.class);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(XstreamStreamingChangeEventSource.class);
|
||||
|
||||
private final ErrorHandler errorHandler;
|
||||
private final EventDispatcher<TableId> dispatcher;
|
||||
@ -114,7 +115,7 @@ private void dispatchSchemaChangeEvent(DDLLCR ddlLcr) throws InterruptedExceptio
|
||||
|
||||
dispatcher.dispatchSchemaChangeEvent(
|
||||
tableId,
|
||||
new OracleSchemaChangeEventEmitter(offsetContext, tableId, ddlLcr));
|
||||
new XStreamSchemaChangeEventEmitter(offsetContext, tableId, ddlLcr));
|
||||
}
|
||||
|
||||
private TableId getTableId(LCR lcr) {
|
@ -3,7 +3,7 @@
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle;
|
||||
package io.debezium.connector.oracle.xstream;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.Arrays;
|
||||
@ -49,7 +49,7 @@ public static LcrPosition valueOf(String rawPosition) {
|
||||
return new LcrPosition(Strings.hexStringToByteArray(rawPosition));
|
||||
}
|
||||
|
||||
byte[] getRawPosition() {
|
||||
public byte[] getRawPosition() {
|
||||
return rawPosition;
|
||||
}
|
||||
|
||||
@ -77,10 +77,7 @@ public boolean equals(Object obj) {
|
||||
return false;
|
||||
}
|
||||
LcrPosition other = (LcrPosition) obj;
|
||||
if (!Arrays.equals(rawPosition, other.rawPosition)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
return Arrays.equals(rawPosition, other.rawPosition);
|
||||
}
|
||||
|
||||
@Override
|
@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.xstream;
|
||||
|
||||
import io.debezium.config.EnumeratedValue;
|
||||
|
||||
import oracle.streams.XStreamUtility;
|
||||
|
||||
/**
|
||||
* This enum class includes Oracle 11 and 12 major versions
|
||||
* It returns position version based on the Oracle release.
|
||||
* Position version get used in conversion SCN (system change number) into position
|
||||
*/
|
||||
public enum OracleVersion implements EnumeratedValue {
|
||||
|
||||
V11("11"),
|
||||
V12Plus("12+");
|
||||
|
||||
private final String version;
|
||||
|
||||
OracleVersion(String version) {
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getValue() {
|
||||
return version;
|
||||
}
|
||||
|
||||
public int getPosVersion() {
|
||||
switch (version) {
|
||||
case "11":
|
||||
return XStreamUtility.POS_VERSION_V1;
|
||||
case "12+":
|
||||
return XStreamUtility.POS_VERSION_V2;
|
||||
default:
|
||||
return XStreamUtility.POS_VERSION_V2;
|
||||
}
|
||||
}
|
||||
|
||||
public static OracleVersion parse(String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
value = value.trim();
|
||||
|
||||
for (OracleVersion option : OracleVersion.values()) {
|
||||
if (option.getValue().equalsIgnoreCase(value)) {
|
||||
return option;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public static OracleVersion parse(String value, String defaultValue) {
|
||||
OracleVersion option = parse(value);
|
||||
|
||||
if (option == null && defaultValue != null) {
|
||||
option = parse(defaultValue);
|
||||
}
|
||||
|
||||
return option;
|
||||
}
|
||||
}
|
@ -3,11 +3,11 @@
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle;
|
||||
package io.debezium.connector.oracle.xstream;
|
||||
|
||||
import io.debezium.connector.oracle.BaseChangeRecordEmitter;
|
||||
import io.debezium.data.Envelope.Operation;
|
||||
import io.debezium.pipeline.spi.OffsetContext;
|
||||
import io.debezium.relational.RelationalChangeRecordEmitter;
|
||||
import io.debezium.relational.Table;
|
||||
import io.debezium.util.Clock;
|
||||
|
||||
@ -19,16 +19,13 @@
|
||||
*
|
||||
* @author Gunnar Morling
|
||||
*/
|
||||
public class XStreamChangeRecordEmitter extends RelationalChangeRecordEmitter {
|
||||
public class XStreamChangeRecordEmitter extends BaseChangeRecordEmitter<ColumnValue> {
|
||||
|
||||
private final RowLCR lcr;
|
||||
private final Table table;
|
||||
|
||||
public XStreamChangeRecordEmitter(OffsetContext offset, RowLCR lcr, Table table, Clock clock) {
|
||||
super(offset, clock);
|
||||
|
||||
super(offset, table, clock);
|
||||
this.lcr = lcr;
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -55,14 +52,13 @@ protected Object[] getNewColumnValues() {
|
||||
return getColumnValues(lcr.getNewValues());
|
||||
}
|
||||
|
||||
private Object[] getColumnValues(ColumnValue[] columnValues) {
|
||||
Object[] values = new Object[table.columns().size()];
|
||||
|
||||
for (ColumnValue columnValue : columnValues) {
|
||||
int index = table.columnWithName(columnValue.getColumnName()).position() - 1;
|
||||
values[index] = columnValue.getColumnData();
|
||||
@Override
|
||||
protected String getColumnName(ColumnValue columnValue) {
|
||||
return columnValue.getColumnName();
|
||||
}
|
||||
|
||||
return values;
|
||||
@Override
|
||||
protected Object getColumnData(ColumnValue columnValue) {
|
||||
return columnValue.getColumnData();
|
||||
}
|
||||
}
|
@ -0,0 +1,30 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.xstream;
|
||||
|
||||
import io.debezium.connector.oracle.BaseOracleSchemaChangeEventEmitter;
|
||||
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||
import io.debezium.pipeline.spi.SchemaChangeEventEmitter;
|
||||
import io.debezium.relational.TableId;
|
||||
|
||||
import oracle.streams.DDLLCR;
|
||||
|
||||
/**
|
||||
* {@link SchemaChangeEventEmitter} implementation based on Oracle.
|
||||
*
|
||||
* @author Gunnar Morling
|
||||
*/
|
||||
public class XStreamSchemaChangeEventEmitter extends BaseOracleSchemaChangeEventEmitter {
|
||||
|
||||
public XStreamSchemaChangeEventEmitter(OracleOffsetContext offsetContext, TableId tableId, DDLLCR ddlLcr) {
|
||||
super(offsetContext,
|
||||
tableId,
|
||||
ddlLcr.getSourceDatabaseName(),
|
||||
ddlLcr.getObjectOwner(),
|
||||
ddlLcr.getDDLText(),
|
||||
ddlLcr.getCommandType());
|
||||
}
|
||||
}
|
@ -3,13 +3,17 @@
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle;
|
||||
package io.debezium.connector.oracle.xstream;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||
import io.debezium.connector.oracle.OracleDatabaseSchema;
|
||||
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||
import io.debezium.connector.oracle.SourceInfo;
|
||||
import io.debezium.jdbc.JdbcConnection;
|
||||
import io.debezium.pipeline.ErrorHandler;
|
||||
import io.debezium.pipeline.EventDispatcher;
|
||||
@ -29,9 +33,9 @@
|
||||
*
|
||||
* @author Gunnar Morling
|
||||
*/
|
||||
public class OracleStreamingChangeEventSource implements StreamingChangeEventSource {
|
||||
public class XstreamStreamingChangeEventSource implements StreamingChangeEventSource {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(OracleStreamingChangeEventSource.class);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(XstreamStreamingChangeEventSource.class);
|
||||
|
||||
private final JdbcConnection jdbcConnection;
|
||||
private final EventDispatcher<TableId> dispatcher;
|
||||
@ -44,7 +48,7 @@ public class OracleStreamingChangeEventSource implements StreamingChangeEventSou
|
||||
private final boolean tablenameCaseInsensitive;
|
||||
private final int posVersion;
|
||||
|
||||
public OracleStreamingChangeEventSource(OracleConnectorConfig connectorConfig, OracleOffsetContext offsetContext, JdbcConnection jdbcConnection,
|
||||
public XstreamStreamingChangeEventSource(OracleConnectorConfig connectorConfig, OracleOffsetContext offsetContext, JdbcConnection jdbcConnection,
|
||||
EventDispatcher<TableId> dispatcher, ErrorHandler errorHandler, Clock clock, OracleDatabaseSchema schema) {
|
||||
this.jdbcConnection = jdbcConnection;
|
||||
this.dispatcher = dispatcher;
|
@ -228,7 +228,7 @@ protected List<String> getAllTables() {
|
||||
protected abstract Builder connectorConfig();
|
||||
|
||||
private static void streamTable(String table) throws SQLException {
|
||||
connection.execute("GRANT SELECT ON " + table + " to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON " + table + " to " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE " + table + " ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
}
|
||||
|
||||
|
@ -7,7 +7,12 @@
|
||||
|
||||
import static org.fest.assertions.Assertions.assertThat;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@ -16,10 +21,15 @@
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
|
||||
import io.debezium.config.Configuration;
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIs;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||
import io.debezium.connector.oracle.util.TestHelper;
|
||||
import io.debezium.data.VerifyRecord;
|
||||
import io.debezium.embedded.AbstractConnectorTest;
|
||||
@ -35,6 +45,9 @@ public class OracleConnectorFilterIT extends AbstractConnectorTest {
|
||||
private static OracleConnection connection;
|
||||
private static OracleConnection adminConnection;
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws SQLException {
|
||||
connection = TestHelper.testConnection();
|
||||
@ -71,8 +84,8 @@ public void before() throws SQLException {
|
||||
"CREATE TABLE debezium2.table2 (id NUMERIC(9,0) NOT NULL, name VARCHAR2(1000), PRIMARY KEY (id))",
|
||||
"CREATE TABLE debezium2.nopk (id NUMERIC(9,0) NOT NULL)",
|
||||
"GRANT ALL PRIVILEGES ON debezium2.table2 TO debezium",
|
||||
"GRANT SELECT ON debezium2.table2 TO " + TestHelper.CONNECTOR_USER,
|
||||
"GRANT SELECT ON debezium2.nopk TO " + TestHelper.CONNECTOR_USER,
|
||||
"GRANT SELECT ON debezium2.table2 TO " + TestHelper.getConnectorUserName(),
|
||||
"GRANT SELECT ON debezium2.nopk TO " + TestHelper.getConnectorUserName(),
|
||||
"ALTER TABLE debezium2.table2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
String ddl = "CREATE TABLE debezium.table1 (" +
|
||||
" id NUMERIC(9,0) NOT NULL, " +
|
||||
@ -81,7 +94,7 @@ public void before() throws SQLException {
|
||||
")";
|
||||
|
||||
connection.execute(ddl);
|
||||
connection.execute("GRANT SELECT ON debezium.table1 TO " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.table1 TO " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.table1 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
|
||||
ddl = "CREATE TABLE debezium.table2 (" +
|
||||
@ -91,7 +104,7 @@ public void before() throws SQLException {
|
||||
")";
|
||||
|
||||
connection.execute(ddl);
|
||||
connection.execute("GRANT SELECT ON debezium.table2 TO " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.table2 TO " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.table2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
|
||||
initializeConnectorTestFramework();
|
||||
@ -99,10 +112,11 @@ public void before() throws SQLException {
|
||||
}
|
||||
|
||||
@Test
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Log Miner does not support DDL during streaming")
|
||||
public void shouldApplyTableWhitelistConfiguration() throws Exception {
|
||||
Configuration config = TestHelper.defaultConfig()
|
||||
.with(
|
||||
OracleConnectorConfig.TABLE_WHITELIST,
|
||||
RelationalDatabaseConnectorConfig.TABLE_WHITELIST,
|
||||
"DEBEZIUM2\\.TABLE2,DEBEZIUM\\.TABLE1,DEBEZIUM\\.TABLE3")
|
||||
.with(OracleConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL_SCHEMA_ONLY)
|
||||
.build();
|
||||
@ -111,6 +125,7 @@ public void shouldApplyTableWhitelistConfiguration() throws Exception {
|
||||
assertConnectorIsRunning();
|
||||
|
||||
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||
waitForStreamingRunning(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||
|
||||
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
||||
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
||||
@ -151,6 +166,7 @@ public void shouldApplyTableWhitelistConfiguration() throws Exception {
|
||||
}
|
||||
|
||||
@Test
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Log Miner does not support DDL during streaming")
|
||||
public void shouldApplyTableIncludeListConfiguration() throws Exception {
|
||||
Configuration config = TestHelper.defaultConfig()
|
||||
.with(
|
||||
@ -163,6 +179,7 @@ public void shouldApplyTableIncludeListConfiguration() throws Exception {
|
||||
assertConnectorIsRunning();
|
||||
|
||||
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||
waitForStreamingRunning(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||
|
||||
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
||||
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
||||
@ -175,8 +192,8 @@ public void shouldApplyTableIncludeListConfiguration() throws Exception {
|
||||
")";
|
||||
|
||||
connection.execute(ddl);
|
||||
connection.execute("GRANT SELECT ON debezium.table3 TO c##xstrm");
|
||||
|
||||
connection.execute("GRANT SELECT ON debezium.table3 TO " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.table3 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
connection.execute("INSERT INTO debezium.table3 VALUES (3, 'Text-3')");
|
||||
connection.execute("COMMIT");
|
||||
|
||||
@ -203,6 +220,7 @@ public void shouldApplyTableIncludeListConfiguration() throws Exception {
|
||||
}
|
||||
|
||||
@Test
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Log Miner does not support DDL during streaming")
|
||||
public void shouldApplyTableBlacklistConfiguration() throws Exception {
|
||||
Configuration config = TestHelper.defaultConfig()
|
||||
.with(
|
||||
@ -215,6 +233,7 @@ public void shouldApplyTableBlacklistConfiguration() throws Exception {
|
||||
assertConnectorIsRunning();
|
||||
|
||||
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||
waitForStreamingRunning(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||
|
||||
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
||||
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
||||
@ -227,7 +246,7 @@ public void shouldApplyTableBlacklistConfiguration() throws Exception {
|
||||
")";
|
||||
|
||||
connection.execute(ddl);
|
||||
connection.execute("GRANT SELECT ON debezium.table3 TO " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.table3 TO " + TestHelper.getConnectorUserName());
|
||||
|
||||
connection.execute("INSERT INTO debezium.table3 VALUES (3, 'Text-3')");
|
||||
connection.execute("COMMIT");
|
||||
@ -255,6 +274,7 @@ public void shouldApplyTableBlacklistConfiguration() throws Exception {
|
||||
}
|
||||
|
||||
@Test
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Log Miner does not support DDL during streaming")
|
||||
public void shouldApplyTableExcludeListConfiguration() throws Exception {
|
||||
Configuration config = TestHelper.defaultConfig()
|
||||
.with(
|
||||
@ -267,6 +287,7 @@ public void shouldApplyTableExcludeListConfiguration() throws Exception {
|
||||
assertConnectorIsRunning();
|
||||
|
||||
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||
waitForStreamingRunning(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||
|
||||
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
||||
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
||||
@ -279,7 +300,7 @@ public void shouldApplyTableExcludeListConfiguration() throws Exception {
|
||||
")";
|
||||
|
||||
connection.execute(ddl);
|
||||
connection.execute("GRANT SELECT ON debezium.table3 TO " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.table3 TO " + TestHelper.getConnectorUserName());
|
||||
|
||||
connection.execute("INSERT INTO debezium.table3 VALUES (3, 'Text-3')");
|
||||
connection.execute("COMMIT");
|
||||
@ -305,4 +326,21 @@ public void shouldApplyTableExcludeListConfiguration() throws Exception {
|
||||
assertThat(after.get("ID")).isEqualTo(3);
|
||||
assertThat(after.get("NAME")).isEqualTo("Text-3");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SkipWhenAdapterNameIsNot(value = SkipWhenAdapterNameIsNot.AdapterName.LOGMINER)
|
||||
public void shouldTakeTimeDifference() throws Exception {
|
||||
Testing.Print.enable();
|
||||
String stmt = "select current_timestamp from dual";
|
||||
try (Connection conn = connection.connection(true);
|
||||
PreparedStatement ps = conn.prepareStatement(stmt);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
rs.next();
|
||||
java.sql.Timestamp ts = rs.getTimestamp(1);
|
||||
Instant fromDb = ts.toInstant();
|
||||
Instant now = Instant.now();
|
||||
long diff = Duration.between(fromDb, now).toMillis();
|
||||
Testing.print("diff: " + diff);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -26,10 +26,14 @@
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
|
||||
import io.debezium.config.Configuration;
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIs;
|
||||
import io.debezium.connector.oracle.util.TestHelper;
|
||||
import io.debezium.data.Envelope;
|
||||
import io.debezium.data.VerifyRecord;
|
||||
@ -48,11 +52,19 @@ public class OracleConnectorIT extends AbstractConnectorTest {
|
||||
private static final long MICROS_PER_SECOND = TimeUnit.SECONDS.toMicros(1);
|
||||
private static final String SNAPSHOT_COMPLETED_KEY = "snapshot_completed";
|
||||
|
||||
@Rule
|
||||
public final TestRule skipAdapterRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
private static OracleConnection connection;
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws SQLException {
|
||||
if (TestHelper.adapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
|
||||
connection = TestHelper.logMinerPdbConnection();
|
||||
}
|
||||
else {
|
||||
connection = TestHelper.testConnection();
|
||||
}
|
||||
|
||||
TestHelper.dropTable(connection, "debezium.customer");
|
||||
TestHelper.dropTable(connection, "debezium.masked_hashed_column_table");
|
||||
@ -68,7 +80,7 @@ public static void beforeClass() throws SQLException {
|
||||
")";
|
||||
|
||||
connection.execute(ddl);
|
||||
connection.execute("GRANT SELECT ON debezium.customer to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.customer to " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.customer ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
|
||||
String ddl2 = "create table debezium.masked_hashed_column_table (" +
|
||||
@ -80,7 +92,7 @@ public static void beforeClass() throws SQLException {
|
||||
")";
|
||||
|
||||
connection.execute(ddl2);
|
||||
connection.execute("GRANT SELECT ON debezium.masked_hashed_column_table to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.masked_hashed_column_table to " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.masked_hashed_column_table ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
|
||||
String ddl3 = "create table debezium.truncated_column_table (" +
|
||||
@ -90,7 +102,7 @@ public static void beforeClass() throws SQLException {
|
||||
")";
|
||||
|
||||
connection.execute(ddl3);
|
||||
connection.execute("GRANT SELECT ON debezium.truncated_column_table to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.truncated_column_table to " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.truncated_column_table ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
|
||||
String ddl4 = "create table dt_table (" +
|
||||
@ -105,7 +117,7 @@ public static void beforeClass() throws SQLException {
|
||||
")";
|
||||
|
||||
connection.execute(ddl4);
|
||||
connection.execute("GRANT SELECT ON debezium.dt_table to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.dt_table to " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.dt_table ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
}
|
||||
|
||||
@ -139,7 +151,7 @@ public void shouldSnapshotAndStreamWithHyphenedTableName() throws Exception {
|
||||
" primary key (id))";
|
||||
|
||||
connection.execute(ddl);
|
||||
connection.execute("GRANT SELECT ON debezium.\"my-table\" to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.\"my-table\" to " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.\"my-table\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
connection.execute("INSERT INTO debezium.\"my-table\" VALUES (1, 25, 'Test')");
|
||||
connection.execute("COMMIT");
|
||||
@ -300,6 +312,7 @@ public void shouldContinueWithStreamingAfterSnapshot() throws Exception {
|
||||
|
||||
@Test
|
||||
@FixFor("DBZ-1223")
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "sendTxBatch randomly fails")
|
||||
public void shouldStreamTransaction() throws Exception {
|
||||
Configuration config = TestHelper.defaultConfig()
|
||||
.with(OracleConnectorConfig.TABLE_INCLUDE_LIST, "DEBEZIUM\\.CUSTOMER")
|
||||
@ -346,20 +359,28 @@ public void shouldStreamTransaction() throws Exception {
|
||||
|
||||
expectedRecordCount = 30;
|
||||
connection.setAutoCommit(false);
|
||||
sendTxBatch(expectedRecordCount, 100);
|
||||
sendTxBatch(expectedRecordCount, 200);
|
||||
sendTxBatch(config, expectedRecordCount, 100);
|
||||
sendTxBatch(config, expectedRecordCount, 200);
|
||||
}
|
||||
|
||||
private void sendTxBatch(int expectedRecordCount, int offset) throws SQLException, InterruptedException {
|
||||
private void sendTxBatch(Configuration config, int expectedRecordCount, int offset) throws SQLException, InterruptedException {
|
||||
boolean isAutoCommit = false;
|
||||
if (connection.connection().getAutoCommit()) {
|
||||
isAutoCommit = true;
|
||||
connection.connection().setAutoCommit(false);
|
||||
}
|
||||
for (int i = offset; i < expectedRecordCount + offset; i++) {
|
||||
connection.executeWithoutCommitting(String.format("INSERT INTO debezium.customer VALUES (%s, 'Brian%s', 2345.67, null)", i, i));
|
||||
}
|
||||
connection.connection().commit();
|
||||
|
||||
assertTxBatch(expectedRecordCount, offset);
|
||||
if (isAutoCommit) {
|
||||
connection.connection().setAutoCommit(true);
|
||||
}
|
||||
|
||||
private void assertTxBatch(int expectedRecordCount, int offset) throws InterruptedException {
|
||||
assertTxBatch(config, expectedRecordCount, offset);
|
||||
}
|
||||
|
||||
private void assertTxBatch(Configuration config, int expectedRecordCount, int offset) throws InterruptedException {
|
||||
SourceRecords records;
|
||||
List<SourceRecord> testTableRecords;
|
||||
Struct after;
|
||||
@ -367,6 +388,7 @@ private void assertTxBatch(int expectedRecordCount, int offset) throws Interrupt
|
||||
records = consumeRecordsByTopic(expectedRecordCount);
|
||||
testTableRecords = records.recordsForTopic("server1.DEBEZIUM.CUSTOMER");
|
||||
assertThat(testTableRecords).hasSize(expectedRecordCount);
|
||||
final String adapter = config.getString(OracleConnectorConfig.CONNECTOR_ADAPTER);
|
||||
|
||||
for (int i = 0; i < expectedRecordCount; i++) {
|
||||
SourceRecord record3 = testTableRecords.get(i);
|
||||
@ -376,13 +398,19 @@ private void assertTxBatch(int expectedRecordCount, int offset) throws Interrupt
|
||||
|
||||
assertThat(record3.sourceOffset().containsKey(SourceInfo.SNAPSHOT_KEY)).isFalse();
|
||||
assertThat(record3.sourceOffset().containsKey(SNAPSHOT_COMPLETED_KEY)).isFalse();
|
||||
|
||||
if (!"LogMiner".equalsIgnoreCase(adapter)) {
|
||||
assertThat(record3.sourceOffset().containsKey(SourceInfo.LCR_POSITION_KEY)).isTrue();
|
||||
assertThat(record3.sourceOffset().containsKey(SourceInfo.SCN_KEY)).isFalse();
|
||||
}
|
||||
|
||||
source = (Struct) ((Struct) record3.value()).get("source");
|
||||
assertThat(source.get(SourceInfo.SNAPSHOT_KEY)).isEqualTo("false");
|
||||
assertThat(source.get(SourceInfo.SCN_KEY)).isNotNull();
|
||||
if (!"LogMiner".equalsIgnoreCase(adapter)) {
|
||||
assertThat(source.get(SourceInfo.LCR_POSITION_KEY)).isNotNull();
|
||||
}
|
||||
|
||||
assertThat(source.get(SourceInfo.SERVER_NAME_KEY)).isEqualTo("server1");
|
||||
assertThat(source.get(SourceInfo.DEBEZIUM_VERSION_KEY)).isNotNull();
|
||||
assertThat(source.get(SourceInfo.TXID_KEY)).isNotNull();
|
||||
@ -391,6 +419,7 @@ private void assertTxBatch(int expectedRecordCount, int offset) throws Interrupt
|
||||
}
|
||||
|
||||
@Test
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Test randomly fails in sendTxBatch")
|
||||
public void shouldStreamAfterRestart() throws Exception {
|
||||
Configuration config = TestHelper.defaultConfig()
|
||||
.with(OracleConnectorConfig.TABLE_INCLUDE_LIST, "DEBEZIUM\\.CUSTOMER")
|
||||
@ -406,14 +435,16 @@ public void shouldStreamAfterRestart() throws Exception {
|
||||
start(OracleConnector.class, config);
|
||||
assertConnectorIsRunning();
|
||||
|
||||
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||
|
||||
SourceRecords records = consumeRecordsByTopic(expectedRecordCount);
|
||||
List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.CUSTOMER");
|
||||
assertThat(testTableRecords).hasSize(expectedRecordCount);
|
||||
|
||||
expectedRecordCount = 30;
|
||||
connection.setAutoCommit(false);
|
||||
sendTxBatch(expectedRecordCount, 100);
|
||||
sendTxBatch(expectedRecordCount, 200);
|
||||
sendTxBatch(config, expectedRecordCount, 100);
|
||||
sendTxBatch(config, expectedRecordCount, 200);
|
||||
|
||||
stopConnector();
|
||||
final int OFFSET = 300;
|
||||
@ -425,9 +456,11 @@ public void shouldStreamAfterRestart() throws Exception {
|
||||
start(OracleConnector.class, config);
|
||||
assertConnectorIsRunning();
|
||||
|
||||
assertTxBatch(expectedRecordCount, 300);
|
||||
sendTxBatch(expectedRecordCount, 400);
|
||||
sendTxBatch(expectedRecordCount, 500);
|
||||
waitForStreamingRunning(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||
|
||||
assertTxBatch(config, expectedRecordCount, 300);
|
||||
sendTxBatch(config, expectedRecordCount, 400);
|
||||
sendTxBatch(config, expectedRecordCount, 500);
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -459,11 +492,14 @@ public void shouldStreamAfterRestartAfterSnapshot() throws Exception {
|
||||
}
|
||||
connection.connection().commit();
|
||||
|
||||
connection.setAutoCommit(true);
|
||||
|
||||
Testing.print("=== Starting connector second time ===");
|
||||
start(OracleConnector.class, config);
|
||||
assertConnectorIsRunning();
|
||||
|
||||
assertTxBatch(expectedRecordCount, 100);
|
||||
sendTxBatch(expectedRecordCount, 200);
|
||||
assertTxBatch(config, expectedRecordCount, 100);
|
||||
sendTxBatch(config, expectedRecordCount, 200);
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -599,6 +635,7 @@ public void deleteWithoutTombstone() throws Exception {
|
||||
}
|
||||
|
||||
@Test
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Seems to get caught in loop?")
|
||||
public void shouldReadChangeStreamForTableCreatedWhileStreaming() throws Exception {
|
||||
TestHelper.dropTable(connection, "debezium.customer2");
|
||||
|
||||
@ -620,7 +657,7 @@ public void shouldReadChangeStreamForTableCreatedWhileStreaming() throws Excepti
|
||||
")";
|
||||
|
||||
connection.execute(ddl);
|
||||
connection.execute("GRANT SELECT ON debezium.customer2 to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.customer2 to " + TestHelper.getConnectorUserName());
|
||||
|
||||
connection.execute("INSERT INTO debezium.customer2 VALUES (2, 'Billie-Bob', 1234.56, TO_DATE('2018/02/22', 'yyyy-mm-dd'))");
|
||||
connection.execute("COMMIT");
|
||||
@ -640,6 +677,7 @@ public void shouldReadChangeStreamForTableCreatedWhileStreaming() throws Excepti
|
||||
|
||||
@Test
|
||||
@FixFor("DBZ-800")
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Log Miner does not yet support DDL during streaming")
|
||||
public void shouldReceiveHeartbeatAlsoWhenChangingTableIncludeListTables() throws Exception {
|
||||
TestHelper.dropTable(connection, "debezium.dbz800a");
|
||||
TestHelper.dropTable(connection, "debezium.dbz800b");
|
||||
|
@ -28,7 +28,8 @@
|
||||
*/
|
||||
public class OracleDdlParserTest {
|
||||
|
||||
private static final String TABLE_NAME = "DEBEZIUM";
|
||||
private static final String TABLE_NAME = "TEST";
|
||||
private static final String PDB_NAME = "ORCLPDB1";
|
||||
|
||||
private OracleDdlParser parser;
|
||||
private Tables tables;
|
||||
@ -47,7 +48,7 @@ public void shouldParseCreateAndAlterTable() throws Exception {
|
||||
Table table = tables.forTable(new TableId(null, null, TABLE_NAME));
|
||||
|
||||
assertThat(tables.size()).isEqualTo(1);
|
||||
assertThat(table.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL7", "COL8", "COL9", "COL10");
|
||||
assertThat(table.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL8", "COL9", "COL10", "COL11", "COL12");
|
||||
// ID, primary key
|
||||
assertThat(table.columnWithName("ID").position()).isEqualTo(1);
|
||||
assertThat(table.isPrimaryKeyColumn("ID"));
|
||||
@ -64,26 +65,31 @@ public void shouldParseCreateAndAlterTable() throws Exception {
|
||||
testColumn(table, "COL5", true, Types.NCHAR, "NCHAR", 1, 0, true, null);
|
||||
// float(126)
|
||||
testColumn(table, "COL6", true, Types.FLOAT, "FLOAT", 126, 0, true, null);
|
||||
// todo: DBZ-137 removed
|
||||
// date
|
||||
testColumn(table, "COL7", true, Types.TIMESTAMP, "DATE", -1, null, true, null);
|
||||
// testColumn(table, "COL7", true, Types.TIMESTAMP, "DATE", -1, null, true, null);
|
||||
// timestamp
|
||||
testColumn(table, "COL8", true, Types.TIMESTAMP, "TIMESTAMP", 6, null, true, null);
|
||||
// blob
|
||||
testColumn(table, "COL9", true, Types.BLOB, "BLOB", -1, null, true, null);
|
||||
// clob
|
||||
testColumn(table, "COL10", true, Types.CLOB, "CLOB", -1, null, true, null);
|
||||
// todo sdo_geometry
|
||||
// testColumn(table, "col12", true, Types.STRUCT, "MDSYS.SDO_GEOMETRY", -1, null,true);
|
||||
// sdo_geometry
|
||||
testColumn(table, "col11", true, Types.STRUCT, "MDSYS.SDO_GEOMETRY", -1, null, true, null);
|
||||
// number(1,0)
|
||||
testColumn(table, "col12", true, Types.NUMERIC, "NUMBER", 1, 0, true, null);
|
||||
|
||||
String ddl = "alter table " + TABLE_NAME + " add (col21 varchar2(20), col22 number(19));";
|
||||
parser.parse(ddl, tables);
|
||||
Table alteredTable = tables.forTable(new TableId(null, null, TABLE_NAME));
|
||||
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL7", "COL8", "COL9", "COL10", "COL21",
|
||||
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL8", "COL9", "COL10", "COL11", "COL12",
|
||||
"COL21",
|
||||
"COL22");
|
||||
// varchar2(255)
|
||||
testColumn(alteredTable, "COL21", true, Types.VARCHAR, "VARCHAR2", 20, null, true, null);
|
||||
testColumn(alteredTable, "COL22", true, Types.NUMERIC, "NUMBER", 19, 0, true, null);
|
||||
|
||||
// todo check real LogMiner entry, maybe this entry never happens
|
||||
ddl = "alter table " + TABLE_NAME + " add col23 varchar2(20);";
|
||||
try {
|
||||
parser.parse(ddl, tables);
|
||||
@ -95,14 +101,16 @@ public void shouldParseCreateAndAlterTable() throws Exception {
|
||||
ddl = "alter table " + TABLE_NAME + " add (col23 varchar2(20) not null);";
|
||||
parser.parse(ddl, tables);
|
||||
alteredTable = tables.forTable(new TableId(null, null, TABLE_NAME));
|
||||
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL7", "COL8", "COL9", "COL10", "COL21",
|
||||
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL8", "COL9", "COL10", "COL11", "COL12",
|
||||
"COL21",
|
||||
"COL22", "COL23");
|
||||
testColumn(alteredTable, "COL23", false, Types.VARCHAR, "VARCHAR2", 20, null, false, null);
|
||||
|
||||
ddl = "alter table " + TABLE_NAME + " drop (col22, col23);";
|
||||
parser.parse(ddl, tables);
|
||||
alteredTable = tables.forTable(new TableId(null, null, TABLE_NAME));
|
||||
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL7", "COL8", "COL9", "COL10", "COL21");
|
||||
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL8", "COL9", "COL10", "COL11", "COL12",
|
||||
"COL21");
|
||||
|
||||
ddl = "drop table " + TABLE_NAME + ";";
|
||||
parser.parse(ddl, tables);
|
||||
@ -120,8 +128,8 @@ public void shouldParseCreateAndAlterTable() throws Exception {
|
||||
@Test
|
||||
public void shouldParseCreateTable() {
|
||||
|
||||
parser.setCurrentDatabase("ORCLPDB1");
|
||||
parser.setCurrentSchema(TABLE_NAME);
|
||||
parser.setCurrentDatabase(PDB_NAME);
|
||||
parser.setCurrentSchema("DEBEZIUM");
|
||||
|
||||
String CREATE_SIMPLE_TABLE = "create table debezium.customer (" +
|
||||
" id int not null, " +
|
||||
@ -131,7 +139,7 @@ public void shouldParseCreateTable() {
|
||||
" primary key (id)" +
|
||||
");";
|
||||
parser.parse(CREATE_SIMPLE_TABLE, tables);
|
||||
Table table = tables.forTable(new TableId("ORCLPDB1", "DEBEZIUM", "CUSTOMER"));
|
||||
Table table = tables.forTable(new TableId(PDB_NAME, "DEBEZIUM", "CUSTOMER"));
|
||||
|
||||
assertThat(table).isNotNull();
|
||||
|
||||
|
@ -47,11 +47,11 @@ public void before() throws SQLException {
|
||||
"CREATE TABLE debezium.tableb (id numeric(9,0) not null, colb varchar2(30), primary key(id))",
|
||||
"CREATE TABLE debezium.tablec (id numeric(9,0) not null, colc varchar2(30), primary key(id))");
|
||||
|
||||
connection.execute("GRANT SELECT ON debezium.tablea to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.tablea to " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.tablea ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
connection.execute("GRANT SELECT ON debezium.tableb to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.tableb to " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.tableb ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
connection.execute("GRANT SELECT ON debezium.tablec to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.tablec to " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.tablec ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
|
||||
initializeConnectorTestFramework();
|
||||
|
@ -59,6 +59,7 @@ public void schemaIsCorrect() {
|
||||
.field("table", Schema.STRING_SCHEMA)
|
||||
.field("txId", Schema.OPTIONAL_STRING_SCHEMA)
|
||||
.field("scn", Schema.OPTIONAL_INT64_SCHEMA)
|
||||
.field("commit_scn", Schema.OPTIONAL_INT64_SCHEMA)
|
||||
.field("lcr_position", Schema.OPTIONAL_STRING_SCHEMA)
|
||||
.build();
|
||||
|
||||
|
@ -9,10 +9,14 @@
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.rules.TestRule;
|
||||
|
||||
import io.debezium.config.Configuration;
|
||||
import io.debezium.config.Configuration.Builder;
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIs;
|
||||
import io.debezium.connector.oracle.util.TestHelper;
|
||||
import io.debezium.util.Testing;
|
||||
|
||||
@ -21,8 +25,12 @@
|
||||
*
|
||||
* @author Jiri Pechanec
|
||||
*/
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Implementation does not support creating/updating schema during streaming")
|
||||
public class StreamingDatatypesIT extends AbstractOracleDatatypesTest {
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
setConsumeTimeout(TestHelper.defaultMessageConsumerPollTimeout(), TimeUnit.SECONDS);
|
||||
|
@ -17,10 +17,14 @@
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
|
||||
import io.debezium.config.Configuration;
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIs;
|
||||
import io.debezium.connector.oracle.util.TestHelper;
|
||||
import io.debezium.data.VerifyRecord;
|
||||
import io.debezium.embedded.AbstractConnectorTest;
|
||||
@ -36,6 +40,9 @@ public class TransactionMetadataIT extends AbstractConnectorTest {
|
||||
|
||||
private static OracleConnection connection;
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws SQLException {
|
||||
connection = TestHelper.testConnection();
|
||||
@ -51,7 +58,7 @@ public static void beforeClass() throws SQLException {
|
||||
")";
|
||||
|
||||
connection.execute(ddl);
|
||||
connection.execute("GRANT SELECT ON debezium.customer to " + TestHelper.CONNECTOR_USER);
|
||||
connection.execute("GRANT SELECT ON debezium.customer to " + TestHelper.getConnectorUserName());
|
||||
connection.execute("ALTER TABLE debezium.customer ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||
}
|
||||
|
||||
@ -71,6 +78,7 @@ public void before() throws SQLException {
|
||||
}
|
||||
|
||||
@Test
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "End transaction is not emitted")
|
||||
public void transactionMetadata() throws Exception {
|
||||
Configuration config = TestHelper.defaultConfig()
|
||||
.with(OracleConnectorConfig.TABLE_INCLUDE_LIST, "DEBEZIUM\\.CUSTOMER")
|
||||
|
@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.junit;
|
||||
|
||||
import org.junit.runner.Description;
|
||||
import org.junit.runners.model.Statement;
|
||||
|
||||
import io.debezium.connector.oracle.util.TestHelper;
|
||||
import io.debezium.junit.AnnotationBasedTestRule;
|
||||
|
||||
/**
|
||||
* JUnit rule that skips a test based on the {@link SkipWhenAdapterNameIs} annotation on
|
||||
* either a test method or test class.
|
||||
*
|
||||
* @author Chris Cranford
|
||||
*/
|
||||
public class SkipTestDependingOnAdapterNameRule extends AnnotationBasedTestRule {
|
||||
|
||||
private static final String adapterName = determineAdapterName();
|
||||
|
||||
@Override
|
||||
public Statement apply(Statement base, Description description) {
|
||||
SkipWhenAdapterNameIs skipWhenAdpterName = hasAnnotation(description, SkipWhenAdapterNameIs.class);
|
||||
if (skipWhenAdpterName != null && skipWhenAdpterName.value().isEqualTo(adapterName)) {
|
||||
String reasonForSkipping = "Adapter name is " + skipWhenAdpterName.value() + System.lineSeparator() + skipWhenAdpterName.reason();
|
||||
return emptyStatement(reasonForSkipping, description);
|
||||
}
|
||||
|
||||
SkipWhenAdapterNameIsNot skipWhenAdapterNameNot = hasAnnotation(description, SkipWhenAdapterNameIsNot.class);
|
||||
if (skipWhenAdapterNameNot != null && skipWhenAdapterNameNot.value().isNotEqualTo(adapterName)) {
|
||||
String reasonForSkipping = "Adapter name is not " + skipWhenAdapterNameNot.value() + System.lineSeparator() + skipWhenAdapterNameNot.reason();
|
||||
return emptyStatement(reasonForSkipping, description);
|
||||
}
|
||||
|
||||
return base;
|
||||
}
|
||||
|
||||
public static String determineAdapterName() {
|
||||
return TestHelper.adapter().getValue();
|
||||
}
|
||||
}
|
@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.junit;
|
||||
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
/**
|
||||
* Marker annotation used together with the {@link SkipTestDependingOnAdapterNameRule} JUnit rule, that allows
|
||||
* tests to be skipped based on the adapter name that is being used for testing.
|
||||
*
|
||||
* @author Chris Cranford
|
||||
*/
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target({ ElementType.METHOD, ElementType.TYPE })
|
||||
public @interface SkipWhenAdapterNameIs {
|
||||
|
||||
SkipWhenAdapterNameIs.AdapterName value();
|
||||
|
||||
/**
|
||||
* Returns the reason why the test should be skipped.
|
||||
*/
|
||||
String reason() default "";
|
||||
|
||||
enum AdapterName {
|
||||
XSTREAM {
|
||||
@Override
|
||||
boolean isEqualTo(String adapterName) {
|
||||
return adapterName.equalsIgnoreCase("xstream");
|
||||
}
|
||||
},
|
||||
LOGMINER {
|
||||
@Override
|
||||
boolean isEqualTo(String adapterName) {
|
||||
return adapterName.equalsIgnoreCase("logminer");
|
||||
}
|
||||
};
|
||||
|
||||
abstract boolean isEqualTo(String adapterName);
|
||||
}
|
||||
}
|
@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.junit;
|
||||
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
/**
|
||||
* Marker annotation used togehter with {@link SkipTestDependingOnAdapterNameRule} JUnit rule, that allows
|
||||
* tests to not be skipped based on the adapter name that is being used for testing.
|
||||
*
|
||||
* @author Chris Cranford
|
||||
*/
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target({ ElementType.METHOD, ElementType.TYPE })
|
||||
public @interface SkipWhenAdapterNameIsNot {
|
||||
|
||||
SkipWhenAdapterNameIsNot.AdapterName value();
|
||||
|
||||
/**
|
||||
* Returns the reason why the test should be skipped.
|
||||
*/
|
||||
String reason() default "";
|
||||
|
||||
enum AdapterName {
|
||||
XSTREAM {
|
||||
@Override
|
||||
boolean isNotEqualTo(String adapterName) {
|
||||
return !adapterName.equalsIgnoreCase("xstream");
|
||||
}
|
||||
},
|
||||
LOGMINER {
|
||||
@Override
|
||||
boolean isNotEqualTo(String adapterName) {
|
||||
return !adapterName.equalsIgnoreCase("logminer");
|
||||
}
|
||||
};
|
||||
|
||||
abstract boolean isNotEqualTo(String adapterName);
|
||||
}
|
||||
}
|
@ -0,0 +1,96 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import static org.fest.assertions.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import io.debezium.connector.common.CdcSourceTaskContext;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||
|
||||
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||
public class LogMinerMetricsTest {
|
||||
|
||||
private LogMinerMetrics metrics;
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
CdcSourceTaskContext taskContext = mock(CdcSourceTaskContext.class);
|
||||
Mockito.when(taskContext.getConnectorName()).thenReturn("connector name");
|
||||
Mockito.when(taskContext.getConnectorType()).thenReturn("connector type");
|
||||
metrics = new LogMinerMetrics(taskContext);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMetrics() {
|
||||
|
||||
metrics.incrementCapturedDmlCount();
|
||||
assertThat(metrics.getCapturedDmlCount() == 1).isTrue();
|
||||
|
||||
metrics.setCurrentScn(1000L);
|
||||
assertThat(metrics.getCurrentScn() == 1000L).isTrue();
|
||||
|
||||
metrics.setBatchSize(10);
|
||||
assertThat(metrics.getBatchSize() == 5_000).isTrue();
|
||||
metrics.setBatchSize(1_000_000);
|
||||
assertThat(metrics.getBatchSize() == 5_000).isTrue();
|
||||
metrics.setBatchSize(6000);
|
||||
assertThat(metrics.getBatchSize() == 6_000).isTrue();
|
||||
|
||||
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 1000).isTrue();
|
||||
metrics.changeSleepingTime(true);
|
||||
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 1200).isTrue();
|
||||
metrics.changeSleepingTime(false);
|
||||
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 1000).isTrue();
|
||||
metrics.setMillisecondToSleepBetweenMiningQuery(20);
|
||||
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 1000).isTrue();
|
||||
metrics.setMillisecondToSleepBetweenMiningQuery(4000);
|
||||
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 1000).isTrue();
|
||||
metrics.setMillisecondToSleepBetweenMiningQuery(2000);
|
||||
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 2000).isTrue();
|
||||
|
||||
metrics.setLastLogMinerQueryDuration(Duration.ofMillis(100));
|
||||
assertThat(metrics.getLastLogMinerQueryDuration() == 100).isTrue();
|
||||
metrics.setLastLogMinerQueryDuration(Duration.ofMillis(200));
|
||||
assertThat(metrics.getLastLogMinerQueryDuration() == 200).isTrue();
|
||||
assertThat(metrics.getAverageLogMinerQueryDuration() == 150).isTrue();
|
||||
assertThat(metrics.getLogMinerQueryCount() == 2).isTrue();
|
||||
|
||||
metrics.setCurrentLogFileName(new HashSet<>(Arrays.asList("name", "name1")));
|
||||
assertThat(metrics.getCurrentRedoLogFileName()[0].equals("name")).isTrue();
|
||||
assertThat(metrics.getCurrentRedoLogFileName()[1].equals("name1")).isTrue();
|
||||
|
||||
metrics.setSwitchCount(5);
|
||||
assertThat(metrics.getSwitchCounter() == 5).isTrue();
|
||||
|
||||
metrics.setProcessedCapturedBatchDuration(Duration.ofMillis(1000));
|
||||
assertThat(metrics.getLastProcessedCapturedBatchDuration() == 1000).isTrue();
|
||||
assertThat(metrics.getProcessedCapturedBatchCount() == 1).isTrue();
|
||||
assertThat(metrics.getAverageProcessedCapturedBatchDuration() == 1000).isTrue();
|
||||
|
||||
metrics.setRedoLogStatus(Collections.singletonMap("name", "current"));
|
||||
assertThat(metrics.getRedoLogStatus()[0].equals("name | current")).isTrue();
|
||||
|
||||
assertThat(metrics.toString().contains("logMinerQueryCount"));
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,132 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import static org.fest.assertions.Assertions.assertThat;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||
import io.debezium.connector.oracle.OracleSnapshotChangeEventSource;
|
||||
import io.debezium.connector.oracle.antlr.OracleDdlParser;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||
import io.debezium.relational.Table;
|
||||
import io.debezium.relational.TableId;
|
||||
import io.debezium.relational.Tables;
|
||||
import io.debezium.util.IoUtil;
|
||||
|
||||
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||
public class LogMinerUtilsTest {
|
||||
|
||||
private static final BigDecimal SCN = BigDecimal.ONE;
|
||||
private static final BigDecimal OTHER_SCN = BigDecimal.TEN;
|
||||
private OracleDdlParser ddlParser;
|
||||
private Tables tables;
|
||||
private static final String TABLE_NAME = "TEST";
|
||||
private static final String CATALOG_NAME = "ORCLPDB1";
|
||||
private static final String SCHEMA_NAME = "DEBEZIUM";
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@Test
|
||||
public void testStartLogMinerStatement() {
|
||||
String statement = SqlUtils.getStartLogMinerStatement(SCN.longValue(), OTHER_SCN.longValue(), OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO, false);
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isTrue();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isTrue();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isFalse();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isFalse();
|
||||
statement = SqlUtils.getStartLogMinerStatement(SCN.longValue(), OTHER_SCN.longValue(), OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG, false);
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isFalse();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isFalse();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isTrue();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isFalse();
|
||||
statement = SqlUtils.getStartLogMinerStatement(SCN.longValue(), OTHER_SCN.longValue(), OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO, true);
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isTrue();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isTrue();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isFalse();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isTrue();
|
||||
statement = SqlUtils.getStartLogMinerStatement(SCN.longValue(), OTHER_SCN.longValue(), OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG, true);
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isFalse();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isFalse();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isTrue();
|
||||
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBlacklistFiltering() throws Exception {
|
||||
|
||||
ddlParser = new OracleDdlParser(true, CATALOG_NAME, SCHEMA_NAME);
|
||||
tables = new Tables();
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
Table table = tables.forTable(new TableId(CATALOG_NAME, SCHEMA_NAME, TABLE_NAME));
|
||||
|
||||
String prefix = CATALOG_NAME + "." + TABLE_NAME + ".";
|
||||
String blacklistedColumns = prefix + "COL2," + prefix + "COL3";
|
||||
String whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||
assertThat(whitelistedColumns.contains("COL2")).isFalse();
|
||||
assertThat(whitelistedColumns.contains("COL3")).isFalse();
|
||||
assertThat(whitelistedColumns.contains("COL4")).isTrue();
|
||||
|
||||
prefix = TABLE_NAME + ".";
|
||||
blacklistedColumns = prefix + "COL2," + prefix + "COL3";
|
||||
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns.toLowerCase(), table);
|
||||
assertThat(whitelistedColumns.contains("COL2")).isFalse();
|
||||
assertThat(whitelistedColumns.contains("COL3")).isFalse();
|
||||
assertThat(whitelistedColumns.contains("COL4")).isTrue();
|
||||
|
||||
prefix = "";
|
||||
blacklistedColumns = prefix + "COL2," + prefix + "COL3";
|
||||
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||
assertThat(whitelistedColumns.equals("*")).isTrue();
|
||||
|
||||
prefix = "NONEXISTINGTABLE.";
|
||||
blacklistedColumns = prefix + "COL2," + prefix + "COL3";
|
||||
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||
assertThat(whitelistedColumns.equals("*")).isTrue();
|
||||
|
||||
prefix = TABLE_NAME + ".";
|
||||
blacklistedColumns = prefix + "col2," + prefix + "CO77";
|
||||
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||
assertThat(whitelistedColumns.contains("COL2")).isFalse();
|
||||
assertThat(whitelistedColumns.contains("CO77")).isFalse();
|
||||
assertThat(whitelistedColumns.contains("COL4")).isTrue();
|
||||
|
||||
blacklistedColumns = "";
|
||||
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||
assertThat(whitelistedColumns.equals("*")).isTrue();
|
||||
|
||||
blacklistedColumns = null;
|
||||
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||
assertThat(whitelistedColumns.equals("*")).isTrue();
|
||||
}
|
||||
|
||||
// todo delete after replacement == -1 in the code
|
||||
@Test
|
||||
public void testConversion() {
|
||||
Map<String, String> map = new HashMap<>();
|
||||
map.put("one", "1001");
|
||||
map.put("two", "1002");
|
||||
map.put("three", "1007");
|
||||
map.put("four", "18446744073709551615");
|
||||
Map<String, Long> res = map.entrySet().stream()
|
||||
.filter(entry -> new BigDecimal(entry.getValue()).longValue() > 1003 || new BigDecimal(entry.getValue()).longValue() == -1).collect(Collectors
|
||||
.toMap(Map.Entry::getKey, e -> new BigDecimal(e.getValue()).longValue() == -1 ? Long.MAX_VALUE : new BigInteger(e.getValue()).longValue()));
|
||||
|
||||
assertThat(res).isNotEmpty();
|
||||
}
|
||||
}
|
@ -0,0 +1,454 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import static org.fest.assertions.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import io.debezium.connector.oracle.antlr.OracleDdlParser;
|
||||
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||
import io.debezium.connector.oracle.jsqlparser.SimpleDmlParser;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||
import io.debezium.data.Envelope;
|
||||
import io.debezium.relational.Tables;
|
||||
import io.debezium.util.IoUtil;
|
||||
|
||||
import net.sf.jsqlparser.statement.update.Update;
|
||||
|
||||
/**
|
||||
* This is the test suite for Oracle Antlr and jsqlparser DML parser unit testing
|
||||
*/
|
||||
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||
public class OracleDmlParserTest {
|
||||
|
||||
private OracleDdlParser ddlParser;
|
||||
private OracleDmlParser antlrDmlParser;
|
||||
private SimpleDmlParser sqlDmlParser;
|
||||
private Tables tables;
|
||||
private static final String TABLE_NAME = "TEST";
|
||||
private static final String CATALOG_NAME = "ORCLPDB1";
|
||||
private static final String SCHEMA_NAME = "DEBEZIUM";
|
||||
private static final String FULL_TABLE_NAME = SCHEMA_NAME + "\".\"" + TABLE_NAME;
|
||||
private static final String SPATIAL_DATA = "SDO_GEOMETRY(2003, NULL, NULL, SDO_ELEM_INFO_ARRAY(1, 1003, 1), SDO_ORDINATE_ARRAY" +
|
||||
"(102604.878, 85772.8286, 101994.879, 85773.6633, 101992.739, 84209.6648, 102602.738, 84208.83, 102604.878, 85772.8286))";
|
||||
private static final String SPATIAL_DATA_1 = "'unsupported type'";
|
||||
private static String CLOB_DATA;
|
||||
private static byte[] BLOB_DATA; // todo
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
OracleChangeRecordValueConverter converters = new OracleChangeRecordValueConverter(null);
|
||||
|
||||
ddlParser = new OracleDdlParser(true, CATALOG_NAME, SCHEMA_NAME);
|
||||
antlrDmlParser = new OracleDmlParser(true, CATALOG_NAME, SCHEMA_NAME, converters);
|
||||
sqlDmlParser = new SimpleDmlParser(CATALOG_NAME, SCHEMA_NAME, converters);
|
||||
tables = new Tables();
|
||||
|
||||
CLOB_DATA = StringUtils.repeat("clob_", 4000);
|
||||
String blobString = "blob_";
|
||||
BLOB_DATA = Arrays.copyOf(blobString.getBytes(), 8000); // todo doesn't support blob
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldParseAliasUpdate() throws Exception {
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
|
||||
String dml = "update \"" + FULL_TABLE_NAME + "\" a set a.\"col1\" = '9', a.col2 = 'diFFerent', a.col3 = 'anotheR', a.col4 = '123', a.col6 = 5.2, " +
|
||||
"a.col8 = TO_TIMESTAMP('2019-05-14 02:28:32.302000'), a.col10 = " + CLOB_DATA + ", a.col11 = null, a.col12 = '1' " +
|
||||
"where a.ID = 5 and a.COL1 = 6 and a.\"COL2\" = 'text' " +
|
||||
"and a.COL3 = 'text' and a.COL4 IS NULL and a.\"COL5\" IS NULL and a.COL6 IS NULL " +
|
||||
"and a.COL8 = TO_TIMESTAMP('2019-05-14 02:28:32.') and a.col11 is null;";
|
||||
|
||||
antlrDmlParser.parse(dml, tables);
|
||||
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||
verifyUpdate(record, false, true, 9);
|
||||
|
||||
record = sqlDmlParser.parse(dml, tables, "1");
|
||||
verifyUpdate(record, false, true, 9);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldParseAliasInsert() throws Exception {
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
|
||||
String dml = "insert into \"" + FULL_TABLE_NAME + "\" a (a.\"ID\",a.\"COL1\",a.\"COL2\",a.\"COL3\",a.\"COL4\",a.\"COL5\",a.\"COL6\",a.\"COL8\"," +
|
||||
"a.\"COL9\",a.\"COL10\") values ('5','4','tExt','text',NULL,NULL,NULL,NULL,EMPTY_BLOB(),EMPTY_CLOB());";
|
||||
antlrDmlParser.parse(dml, tables);
|
||||
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||
verifyInsert(record);
|
||||
|
||||
record = sqlDmlParser.parse(dml, tables, "1");
|
||||
verifyInsert(record);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldParseAliasDelete() throws Exception {
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
|
||||
String dml = "delete from \"" + FULL_TABLE_NAME +
|
||||
"\" a where a.\"id\" = 6 and a.\"col1\" = 2 and a.\"col2\" = 'text' and a.col3 = 'tExt' and a.col4 is null and a.col5 is null " +
|
||||
" and a.col6 is null and a.col8 is null and a.col9 is null and a.col10 is null and a.col11 is null and a.col12 is null";
|
||||
antlrDmlParser.parse(dml, tables);
|
||||
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||
verifyDelete(record, true);
|
||||
|
||||
record = sqlDmlParser.parse(dml, tables, "1");
|
||||
verifyDelete(record, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldParseNoWhereClause() throws Exception {
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
|
||||
String dml = "update \"" + FULL_TABLE_NAME
|
||||
+ "\" a set a.\"id\"=1, a.\"col1\" = '9', a.col2 = 'diFFerent', a.col3 = 'anotheR', a.col4 = '123', a.col5 = null, a.col6 = 5.2, " +
|
||||
"a.col8 = TO_TIMESTAMP('2019-05-14 02:28:32.302000'), a.col9=null, a.col10 = " + CLOB_DATA + ", a.col11 = null, a.col12 = '1'";
|
||||
|
||||
antlrDmlParser.parse(dml, tables);
|
||||
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||
verifyUpdate(record, false, false, 9);
|
||||
|
||||
record = sqlDmlParser.parse(dml, tables, "1");
|
||||
verifyUpdate(record, false, false, 9);
|
||||
|
||||
dml = "delete from \"" + FULL_TABLE_NAME + "\" a ";
|
||||
antlrDmlParser.parse(dml, tables);
|
||||
record = antlrDmlParser.getDmlEntry();
|
||||
verifyDelete(record, false);
|
||||
|
||||
record = sqlDmlParser.parse(dml, tables, "1");
|
||||
verifyDelete(record, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldParseInsertAndDeleteTable() throws Exception {
|
||||
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
|
||||
String dml = "insert into \"" + FULL_TABLE_NAME + "\"(\"ID\",\"COL1\",\"COL2\",\"COL3\",\"COL4\",\"COL5\",\"COL6\",\"COL8\"," +
|
||||
"\"COL9\",\"COL10\") values ('5','4','tExt','text',NULL,NULL,NULL,NULL,EMPTY_BLOB(),EMPTY_CLOB());";
|
||||
antlrDmlParser.parse(dml, tables);
|
||||
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||
verifyInsert(record);
|
||||
|
||||
record = sqlDmlParser.parse(dml, tables, "1");
|
||||
verifyInsert(record);
|
||||
|
||||
dml = "delete from \"" + FULL_TABLE_NAME +
|
||||
"\" where id = 6 and col1 = 2 and col2 = 'text' and col3 = 'tExt' and col4 is null and col5 is null " +
|
||||
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col12 is null";
|
||||
antlrDmlParser.parse(dml, tables);
|
||||
record = antlrDmlParser.getDmlEntry();
|
||||
verifyDelete(record, true);
|
||||
|
||||
record = sqlDmlParser.parse(dml, tables, "");
|
||||
verifyDelete(record, true);
|
||||
}
|
||||
|
||||
// todo encrypted columns and spatial will be represented as "Unsupported Type"
|
||||
@Test
|
||||
public void shouldParseUpdateTable() throws Exception {
|
||||
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
|
||||
String dml = "update \"" + FULL_TABLE_NAME + "\" set \"col1\" = '9', col2 = 'diFFerent', col3 = 'anotheR', col4 = '123', col6 = '5.2', " +
|
||||
"col8 = TO_TIMESTAMP('2019-05-14 02:28:32.302000'), col10='clob_', col12 = '1' " +
|
||||
"where ID = 5 and COL1 = 6 and \"COL2\" = 'text' " +
|
||||
"and COL3 = 'text' and COL4 IS NULL and \"COL5\" IS NULL and COL6 IS NULL " +
|
||||
"and COL8 = TO_TIMESTAMP('2019-05-14 02:28:32') and col11 = " + SPATIAL_DATA + ";";
|
||||
|
||||
antlrDmlParser.parse(dml, tables);
|
||||
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||
// verifyUpdate(record, true, true);
|
||||
|
||||
record = sqlDmlParser.parse(dml, tables, "");
|
||||
verifyUpdate(record, true, true, 9);
|
||||
|
||||
dml = "update \"" + FULL_TABLE_NAME
|
||||
+ "\" set \"col1\" = '9', col2 = '$2a$10$aHo.lQk.YAkGl5AkXbjJhODBqwNLkqF94slP5oZ3boNzm0d04WnE2', col3 = NULL, col4 = '123', col6 = '5.2', " +
|
||||
"col8 = TO_TIMESTAMP('2019-05-14 02:28:32.302000'), col10='clob_', col12 = '1' " +
|
||||
"where ID = 5 and COL1 = 6 and \"COL2\" = 'johan.philtjens@dpworld.com' " +
|
||||
"and COL3 = 'text' and COL4 IS NULL and \"COL5\" IS NULL and COL6 IS NULL " +
|
||||
"and COL8 = TO_TIMESTAMP('2019-05-14 02:28:32') and col11 = " + SPATIAL_DATA + ";";
|
||||
|
||||
record = sqlDmlParser.parse(dml, tables, "");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldParseUpdateNoChangesTable() throws Exception {
|
||||
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
|
||||
String dml = "update \"" + FULL_TABLE_NAME + "\" set \"col1\" = '6', col2 = 'text', col3 = 'text', col4 = NULL " +
|
||||
"where ID = 5 and COL1 = 6 and \"COL2\" = 'text' " +
|
||||
"and COL3 = Unsupported Type and COL4 IS NULL and \"COL5\" IS NULL and COL6 IS NULL and COL7 IS NULL and COL9 IS NULL and COL10 IS NULL and COL12 IS NULL "
|
||||
+
|
||||
"and COL8 = TO_TIMESTAMP('2019-05-14 02:28:32') and col11 = " + SPATIAL_DATA + ";";
|
||||
|
||||
LogMinerDmlEntry record = sqlDmlParser.parse(dml, tables, "");
|
||||
boolean pass = record.getCommandType().equals(Envelope.Operation.UPDATE)
|
||||
&& record.getOldValues().size() == record.getNewValues().size()
|
||||
&& record.getNewValues().containsAll(record.getOldValues());
|
||||
assertThat(pass);
|
||||
assertThat(record.getOldValues().get(4).getColumnData()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldParseSpecialCharacters() throws Exception {
|
||||
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
|
||||
String dml = "insert into \"" + FULL_TABLE_NAME + "\"(\"ID\",\"COL1\",\"COL2\",\"COL3\",\"COL4\",\"COL5\",\"COL6\",\"COL8\"," +
|
||||
"\"COL9\",\"COL10\") values ('5','4','\\','\\test',NULL,NULL,NULL,NULL,EMPTY_BLOB(),EMPTY_CLOB());";
|
||||
antlrDmlParser.parse(dml, tables);
|
||||
assertThat(antlrDmlParser.getDmlEntry()).isNotNull();
|
||||
|
||||
LogMinerDmlEntry result = sqlDmlParser.parse(dml, tables, "1");
|
||||
assertThat(result).isNotNull();
|
||||
LogMinerColumnValue value = result.getNewValues().get(2);
|
||||
assertThat(value.getColumnData().toString()).contains("\\");
|
||||
|
||||
dml = "delete from \"" + FULL_TABLE_NAME +
|
||||
"\" where id = 6 and col1 = 2 and col2 = 'te\\xt' and col3 = 'tExt\\' and col4 is null and col5 is null " +
|
||||
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col12 is null";
|
||||
antlrDmlParser.parse(dml, tables);
|
||||
assertThat(antlrDmlParser.getDmlEntry()).isNotNull();
|
||||
|
||||
result = sqlDmlParser.parse(dml, tables, "");
|
||||
assertThat(result).isNotNull();
|
||||
value = result.getOldValues().get(3);
|
||||
assertThat(value.getColumnData().toString()).contains("\\");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldParseStrangeDml() throws Exception {
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
String dml = null;
|
||||
LogMinerDmlEntry result = sqlDmlParser.parse(dml, tables, "");
|
||||
assertThat(result).isNull();
|
||||
dml = "select * from test;null;";
|
||||
result = sqlDmlParser.parse(dml, tables, "");
|
||||
assertThat(result).isNull();
|
||||
dml = "full dummy mess";
|
||||
result = sqlDmlParser.parse(dml, tables, "");
|
||||
assertThat(result).isNull();
|
||||
dml = "delete from non_exiting_table " +
|
||||
" where id = 6 and col1 = 2 and col2 = 'te\\xt' and col3 = 'tExt\\' and col4 is null and col5 is null " +
|
||||
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col12 is null";
|
||||
result = sqlDmlParser.parse(dml, tables, "");
|
||||
assertThat(result).isNull();
|
||||
|
||||
Update update = mock(Update.class);
|
||||
Mockito.when(update.getTables()).thenReturn(new ArrayList<>());
|
||||
dml = "update \"" + FULL_TABLE_NAME + "\" set col1 = 3 " +
|
||||
" where id = 6 and col1 = 2 and col2 = 'te\\xt' and col3 = 'tExt\\' and col4 is null and col5 is null " +
|
||||
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col12 is null and col20 is null";
|
||||
result = sqlDmlParser.parse(dml, tables, "");
|
||||
assertThat(result.getOldValues().size()).isEqualTo(12);
|
||||
assertThat(result.getOldValues().size() == 12).isTrue();
|
||||
|
||||
dml = "update \"" + FULL_TABLE_NAME + "\" set col1 = 3 " +
|
||||
" where id = 6 and col1 = 2 and col2 = 'te\\xt' and col30 = 'tExt\\' and col4 is null and col5 is null " +
|
||||
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col21 is null";
|
||||
result = sqlDmlParser.parse(dml, tables, "");
|
||||
assertThat(result.getNewValues().size() == 12).isTrue();
|
||||
|
||||
dml = "update table1, \"" + FULL_TABLE_NAME + "\" set col1 = 3 " +
|
||||
" where id = 6 and col1 = 2 and col2 = 'te\\xt' and col3 = 'tExt\\' and col4 is null and col5 is null " +
|
||||
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col12 is null and col20 is null";
|
||||
result = sqlDmlParser.parse(dml, tables, "");
|
||||
assertThat(result).isNull();
|
||||
}
|
||||
|
||||
private void verifyUpdate(LogMinerDmlEntry record, boolean checkGeometry, boolean checkOldValues, int oldValuesNumber) {
|
||||
// validate
|
||||
assertThat(record.getCommandType()).isEqualTo(Envelope.Operation.UPDATE);
|
||||
List<LogMinerColumnValue> newValues = record.getNewValues();
|
||||
assertThat(newValues.size()).isEqualTo(12);
|
||||
String concatenatedNames = newValues.stream().map(LogMinerColumnValue::getColumnName).collect(Collectors.joining());
|
||||
assertThat("IDCOL1COL2COL3COL4COL5COL6COL8COL9COL10COL11COL12".equals(concatenatedNames));
|
||||
for (LogMinerColumnValue newValue : newValues) {
|
||||
String columnName = newValue.getColumnName();
|
||||
switch (columnName) {
|
||||
case "COL1":
|
||||
assertThat(newValue.getColumnData()).isEqualTo(BigDecimal.valueOf(9, 0));
|
||||
break;
|
||||
case "COL2":
|
||||
assertThat(newValue.getColumnData()).isEqualTo("diFFerent");
|
||||
break;
|
||||
case "COL3":
|
||||
assertThat(newValue.getColumnData()).isEqualTo("anotheR");
|
||||
break;
|
||||
case "COL4":
|
||||
assertThat(newValue.getColumnData()).isEqualTo("123");
|
||||
break;
|
||||
case "COL6":
|
||||
// todo, which one is expected value format
|
||||
assertThat(newValue.getColumnData()).isEqualTo(5.2F);
|
||||
// assertThat(((Struct)newValue.getColumnData()).get("scale")).isEqualTo(1);
|
||||
// assertThat(((byte[])((Struct)newValue.getColumnData()).get("value"))[0]).isEqualTo((byte) 52);
|
||||
break;
|
||||
case "COL8":
|
||||
assertThat(newValue.getColumnData()).isInstanceOf(Long.class);
|
||||
// todo: DBZ-137 value didn't account for values to be GMT.
|
||||
assertThat(newValue.getColumnData()).isEqualTo(1557800912302000L /* 1557826112302000L */);
|
||||
break;
|
||||
case "COL10":
|
||||
assertThat(newValue.getColumnData()).isInstanceOf(String.class);
|
||||
assertThat(newValue.getColumnData().toString().contains("clob_")).isTrue();
|
||||
break;
|
||||
case "COL11":
|
||||
if (checkGeometry) {
|
||||
assertThat(newValue.getColumnData()).isInstanceOf(String.class);
|
||||
assertThat(newValue.getColumnData().toString().contains("SDO_GEOMETRY")).isTrue();
|
||||
}
|
||||
else {
|
||||
assertThat(newValue.getColumnData()).isNull();
|
||||
}
|
||||
break;
|
||||
case "COL12":
|
||||
assertThat(newValue.getColumnData()).isInstanceOf(Boolean.class);
|
||||
assertThat(newValue.getColumnData()).isEqualTo(true);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
List<LogMinerColumnValue> oldValues = record.getOldValues();
|
||||
if (!checkOldValues) {
|
||||
assertThat(oldValues.size()).isEqualTo(0);
|
||||
}
|
||||
else {
|
||||
assertThat(oldValues.size()).isEqualTo(oldValuesNumber);
|
||||
concatenatedNames = oldValues.stream().map(LogMinerColumnValue::getColumnName).collect(Collectors.joining());
|
||||
assertThat("IDCOL1COL2COL3COL4COL6COL8COL11COL12".equals(concatenatedNames));
|
||||
for (LogMinerColumnValue oldValue : oldValues) {
|
||||
String columnName = oldValue.getColumnName();
|
||||
switch (columnName) {
|
||||
case "COL1":
|
||||
assertThat(oldValue.getColumnData()).isEqualTo(BigDecimal.valueOf(6, 0));
|
||||
break;
|
||||
case "COL2":
|
||||
assertThat(oldValue.getColumnData()).isEqualTo("text");
|
||||
break;
|
||||
case "COL3":
|
||||
assertThat(oldValue.getColumnData()).isEqualTo("text");
|
||||
break;
|
||||
case "COL4":
|
||||
assertThat(oldValue.getColumnData()).isNull();
|
||||
break;
|
||||
case "COL5":
|
||||
assertThat(oldValue.getColumnData()).isNull();
|
||||
break;
|
||||
case "COL6":
|
||||
assertThat(oldValue.getColumnData()).isNull();
|
||||
break;
|
||||
case "COL8":
|
||||
assertThat(oldValue.getColumnData()).isInstanceOf(Long.class);
|
||||
// todo: DBZ-137 value didn't account for values to be GMT.
|
||||
assertThat(oldValue.getColumnData()).isEqualTo(1557800912000000L /* 1557826112000000L */);
|
||||
break;
|
||||
case "COL11":
|
||||
if (checkGeometry) {
|
||||
assertThat(oldValue.getColumnData()).isInstanceOf(String.class);
|
||||
assertThat(oldValue.getColumnData().toString().contains("SDO_GEOMETRY")).isTrue();
|
||||
}
|
||||
else {
|
||||
assertThat(oldValue.getColumnData()).isNull();
|
||||
}
|
||||
break;
|
||||
case "ID":
|
||||
assertThat(oldValue.getColumnData()).isEqualTo(new BigDecimal(5));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyInsert(LogMinerDmlEntry record) {
|
||||
List<LogMinerColumnValue> oldValues = record.getOldValues();
|
||||
assertThat(oldValues.size()).isEqualTo(0);
|
||||
|
||||
assertThat(record.getCommandType()).isEqualTo(Envelope.Operation.CREATE);
|
||||
|
||||
List<LogMinerColumnValue> newValues = record.getNewValues();
|
||||
assertThat(newValues.size()).isEqualTo(12);
|
||||
|
||||
Iterator<LogMinerColumnValue> iterator = newValues.iterator();
|
||||
assertThat(iterator.next().getColumnData()).isEqualTo(new BigDecimal(5));
|
||||
assertThat(iterator.next().getColumnData()).isEqualTo(BigDecimal.valueOf(4, 0));
|
||||
assertThat(iterator.next().getColumnData()).isEqualTo("tExt");
|
||||
assertThat(iterator.next().getColumnData()).isEqualTo("text");
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
// todo handle LOBS
|
||||
// assertThat(iterator.next().getColumnData()).isNull();
|
||||
// assertThat(iterator.next().getColumnData()).isNull();
|
||||
|
||||
}
|
||||
|
||||
private void verifyDelete(LogMinerDmlEntry record, boolean checkOldValues) {
|
||||
assertThat(record.getCommandType()).isEqualTo(Envelope.Operation.DELETE);
|
||||
List<LogMinerColumnValue> newValues = record.getNewValues();
|
||||
assertThat(newValues.size()).isEqualTo(0);
|
||||
|
||||
List<LogMinerColumnValue> oldValues = record.getOldValues();
|
||||
if (!checkOldValues) {
|
||||
assertThat(oldValues.size()).isEqualTo(0);
|
||||
}
|
||||
else {
|
||||
// todo: DBZ-137 should not include COL7 but column is included
|
||||
assertThat(oldValues.size()).isEqualTo(12);
|
||||
String concatenatedColumnNames = oldValues.stream().map(LogMinerColumnValue::getColumnName).collect(Collectors.joining());
|
||||
assertThat("IDCOL1COL2COL3COL4COL5COL6COL8COL9COL10COL11COL12".equals(concatenatedColumnNames));
|
||||
|
||||
Iterator<LogMinerColumnValue> iterator = oldValues.iterator();
|
||||
assertThat(iterator.next().getColumnData()).isEqualTo(new BigDecimal(6));
|
||||
assertThat(iterator.next().getColumnData()).isEqualTo(BigDecimal.valueOf(2, 0));
|
||||
assertThat(iterator.next().getColumnData()).isEqualTo("text");
|
||||
assertThat(iterator.next().getColumnData()).isEqualTo("tExt");
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
assertThat(iterator.next().getColumnData()).isNull();
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,176 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import static org.fest.assertions.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Instant;
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||
import io.debezium.relational.TableId;
|
||||
|
||||
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||
public class RowMapperTest {
|
||||
private ResultSet rs;
|
||||
private TransactionalBufferMetrics metrics;
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
rs = mock(ResultSet.class);
|
||||
metrics = mock(TransactionalBufferMetrics.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testChangeTime() throws SQLException {
|
||||
Mockito.when(rs.getTimestamp(4)).thenReturn(new Timestamp(1000L));
|
||||
Timestamp time = RowMapper.getChangeTime(metrics, rs);
|
||||
assertThat(time.getTime() == 1000L).isTrue();
|
||||
Mockito.when(rs.getTimestamp(4)).thenThrow(SQLException.class);
|
||||
time = RowMapper.getChangeTime(metrics, rs);
|
||||
assertThat(time.getTime() == new Timestamp(Instant.now().getEpochSecond()).getTime()).isTrue();
|
||||
verify(rs, times(2)).getTimestamp(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOperationCode() throws SQLException {
|
||||
Mockito.when(rs.getInt(3)).thenReturn(100);
|
||||
int operation = RowMapper.getOperationCode(metrics, rs);
|
||||
assertThat(operation == 100).isTrue();
|
||||
verify(rs).getInt(3);
|
||||
Mockito.when(rs.getInt(3)).thenThrow(SQLException.class);
|
||||
operation = RowMapper.getOperationCode(metrics, rs);
|
||||
assertThat(operation == 0).isTrue();
|
||||
verify(rs, times(2)).getInt(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTableName() throws SQLException {
|
||||
Mockito.when(rs.getString(7)).thenReturn("table_name");
|
||||
String tableName = RowMapper.getTableName(metrics, rs);
|
||||
assertThat(tableName.equals("table_name")).isTrue();
|
||||
verify(rs).getString(7);
|
||||
Mockito.when(rs.getString(7)).thenThrow(SQLException.class);
|
||||
tableName = RowMapper.getTableName(metrics, rs);
|
||||
assertThat(tableName.equals("")).isTrue();
|
||||
verify(rs, times(2)).getString(7);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSeqOwner() throws SQLException {
|
||||
Mockito.when(rs.getString(8)).thenReturn("owner");
|
||||
String owner = RowMapper.getSegOwner(metrics, rs);
|
||||
assertThat(owner.equals("owner")).isTrue();
|
||||
verify(rs).getString(8);
|
||||
Mockito.when(rs.getString(8)).thenThrow(SQLException.class);
|
||||
owner = RowMapper.getSegOwner(metrics, rs);
|
||||
assertThat(owner.equals("")).isTrue();
|
||||
verify(rs, times(2)).getString(8);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetScn() throws SQLException {
|
||||
Mockito.when(rs.getBigDecimal(1)).thenReturn(new BigDecimal(1));
|
||||
BigDecimal scn = RowMapper.getScn(metrics, rs);
|
||||
assertThat(scn.equals(new BigDecimal(1))).isTrue();
|
||||
verify(rs).getBigDecimal(1);
|
||||
Mockito.when(rs.getBigDecimal(1)).thenThrow(SQLException.class);
|
||||
scn = RowMapper.getScn(metrics, rs);
|
||||
assertThat(scn.equals(new BigDecimal(-1))).isTrue();
|
||||
verify(rs, times(2)).getBigDecimal(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTransactionId() throws SQLException {
|
||||
Mockito.when(rs.getBytes(5)).thenReturn("tr_id".getBytes());
|
||||
String transactionId = RowMapper.getTransactionId(metrics, rs);
|
||||
assertThat(transactionId.equals("74725F6964")).isTrue();
|
||||
verify(rs).getBytes(5);
|
||||
Mockito.when(rs.getBytes(5)).thenThrow(SQLException.class);
|
||||
transactionId = RowMapper.getTransactionId(metrics, rs);
|
||||
assertThat(transactionId.equals("")).isTrue();
|
||||
verify(rs, times(2)).getBytes(5);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlRedo() throws SQLException {
|
||||
Mockito.when(rs.getInt(6)).thenReturn(0);
|
||||
Mockito.when(rs.getString(2)).thenReturn("short_sql");
|
||||
String sql = RowMapper.getSqlRedo(metrics, rs);
|
||||
assertThat(sql.equals("short_sql")).isTrue();
|
||||
verify(rs).getInt(6);
|
||||
verify(rs).getString(2);
|
||||
|
||||
Mockito.when(rs.getInt(6)).thenReturn(1).thenReturn(0);
|
||||
Mockito.when(rs.getString(2)).thenReturn("long").thenReturn("_sql");
|
||||
sql = RowMapper.getSqlRedo(metrics, rs);
|
||||
assertThat(sql.equals("long_sql")).isTrue();
|
||||
verify(rs, times(3)).getInt(6);
|
||||
verify(rs, times(3)).getString(2);
|
||||
|
||||
// test super large DML
|
||||
char[] chars = new char[4000];
|
||||
Arrays.fill(chars, 'a');
|
||||
Mockito.when(rs.getString(2)).thenReturn(new String(chars));
|
||||
Mockito.when(rs.getInt(6)).thenReturn(1);
|
||||
sql = RowMapper.getSqlRedo(metrics, rs);
|
||||
assertThat(sql.length() == 40_000).isTrue();
|
||||
verify(rs, times(13)).getInt(6);
|
||||
verify(rs, times(13)).getString(2);
|
||||
|
||||
Mockito.when(rs.getInt(6)).thenReturn(0);
|
||||
Mockito.when(rs.getString(2)).thenReturn(null);
|
||||
sql = RowMapper.getSqlRedo(metrics, rs);
|
||||
assertThat(sql == null).isTrue();
|
||||
verify(rs, times(13)).getInt(6);
|
||||
verify(rs, times(14)).getString(2);
|
||||
|
||||
Mockito.when(rs.getInt(6)).thenReturn(0);
|
||||
Mockito.when(rs.getString(2)).thenThrow(SQLException.class);
|
||||
sql = RowMapper.getSqlRedo(metrics, rs);
|
||||
assertThat(sql.equals("")).isTrue();
|
||||
verify(rs, times(13)).getInt(6);
|
||||
verify(rs, times(15)).getString(2);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTableId() throws SQLException {
|
||||
Mockito.when(rs.getString(8)).thenReturn("schema");
|
||||
Mockito.when(rs.getString(7)).thenReturn("table");
|
||||
TableId tableId = RowMapper.getTableId("catalog", rs);
|
||||
assertThat(tableId.toString().equals("CATALOG.SCHEMA.TABLE")).isTrue();
|
||||
verify(rs).getString(8);
|
||||
Mockito.when(rs.getString(8)).thenThrow(SQLException.class);
|
||||
|
||||
tableId = null;
|
||||
try {
|
||||
tableId = RowMapper.getTableId("catalog", rs);
|
||||
assertThat(1 == 2).isTrue();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
assertThat(tableId).isNull();
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,69 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import static org.fest.assertions.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||
import io.debezium.connector.oracle.OracleDatabaseSchema;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||
import io.debezium.relational.TableId;
|
||||
|
||||
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||
public class SqlUtilsTest {
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@Test
|
||||
public void testStatements() {
|
||||
String result = SqlUtils.getAddLogFileStatement("ADD", "FILENAME");
|
||||
String expected = "BEGIN sys.dbms_logmnr.add_logfile(LOGFILENAME => 'FILENAME', OPTIONS => ADD);END;";
|
||||
assertThat(expected.equals(result)).isTrue();
|
||||
|
||||
OracleDatabaseSchema schema = mock(OracleDatabaseSchema.class);
|
||||
TableId table1 = new TableId("catalog", "schema", "table1");
|
||||
TableId table2 = new TableId("catalog", "schema", "table2");
|
||||
Set<TableId> tables = new HashSet<>();
|
||||
Mockito.when(schema.tableIds()).thenReturn(tables);
|
||||
result = SqlUtils.queryLogMinerContents("DATABASE", "SCHEMA", schema);
|
||||
expected = "SELECT SCN, SQL_REDO, OPERATION_CODE, TIMESTAMP, XID, CSF, TABLE_NAME, SEG_OWNER, OPERATION, USERNAME " +
|
||||
"FROM V$LOGMNR_CONTENTS WHERE OPERATION_CODE in (1,2,3,5) AND SEG_OWNER = 'DATABASE' AND SCN >= ? AND SCN < ? " +
|
||||
"OR (OPERATION_CODE IN (5,7,34,36) AND USERNAME NOT IN ('SYS','SYSTEM','SCHEMA'))ORDER BY SCN";
|
||||
assertThat(expected.equals(result)).isTrue();
|
||||
|
||||
tables.add(table1);
|
||||
tables.add(table2);
|
||||
result = SqlUtils.queryLogMinerContents("DATABASE", "SCHEMA", schema);
|
||||
expected = "SELECT SCN, SQL_REDO, OPERATION_CODE, TIMESTAMP, XID, CSF, TABLE_NAME, SEG_OWNER, OPERATION, USERNAME " +
|
||||
"FROM V$LOGMNR_CONTENTS WHERE OPERATION_CODE in (1,2,3,5) " +
|
||||
"AND SEG_OWNER = 'DATABASE' AND table_name IN ('table1','table2') AND SEG_NAME IN ('table1','table2') " +
|
||||
"AND SCN >= ? AND SCN < ? OR (OPERATION_CODE IN (5,7,34,36) AND USERNAME NOT IN ('SYS','SYSTEM','SCHEMA'))ORDER BY SCN";
|
||||
assertThat(expected.equals(result)).isTrue();
|
||||
|
||||
result = SqlUtils.getStartLogMinerStatement(10L, 20L, OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG, true);
|
||||
expected = "BEGIN sys.dbms_logmnr.start_logmnr(startScn => '10', endScn => '20', " +
|
||||
"OPTIONS => DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG + DBMS_LOGMNR.CONTINUOUS_MINE + DBMS_LOGMNR.NO_ROWID_IN_STMT);END;";
|
||||
assertThat(expected.equals(result)).isTrue();
|
||||
|
||||
result = SqlUtils.getStartLogMinerStatement(10L, 20L, OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO, false);
|
||||
expected = "BEGIN sys.dbms_logmnr.start_logmnr(startScn => '10', endScn => '20', " +
|
||||
"OPTIONS => DBMS_LOGMNR.DICT_FROM_REDO_LOGS + DBMS_LOGMNR.DDL_DICT_TRACKING + DBMS_LOGMNR.NO_ROWID_IN_STMT);END;";
|
||||
assertThat(expected.equals(result)).isTrue();
|
||||
|
||||
}
|
||||
}
|
@ -0,0 +1,150 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import static org.fest.assertions.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import io.debezium.connector.common.CdcSourceTaskContext;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIs;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||
|
||||
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||
public class TransactionalBufferMetricsTest {
|
||||
|
||||
private TransactionalBufferMetrics metrics;
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
CdcSourceTaskContext taskContext = mock(CdcSourceTaskContext.class);
|
||||
Mockito.when(taskContext.getConnectorName()).thenReturn("connector name");
|
||||
Mockito.when(taskContext.getConnectorType()).thenReturn("connector type");
|
||||
metrics = new TransactionalBufferMetrics(taskContext);
|
||||
}
|
||||
|
||||
@After
|
||||
public void after() {
|
||||
metrics.reset();
|
||||
}
|
||||
|
||||
@Test
|
||||
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Lag calculations fail")
|
||||
public void testLagMetrics() {
|
||||
// no time difference between connector and database
|
||||
long lag = metrics.getLagFromSource();
|
||||
assertThat(lag == 0).isTrue();
|
||||
Instant dbEventTime = Instant.now().minusMillis(2000);
|
||||
metrics.calculateLagMetrics(dbEventTime);
|
||||
lag = metrics.getLagFromSource();
|
||||
assertThat(lag == 2000).isTrue();
|
||||
assertThat(metrics.getMaxLagFromSource() == 2000).isTrue();
|
||||
assertThat(metrics.getMinLagFromSource() == 0).isTrue();
|
||||
assertThat(metrics.getAverageLagFromSource() == 2000).isTrue();
|
||||
|
||||
// not realistic scenario
|
||||
dbEventTime = Instant.now().plusMillis(2000);
|
||||
metrics.calculateLagMetrics(dbEventTime);
|
||||
lag = metrics.getLagFromSource();
|
||||
assertThat(lag == -2000).isTrue();
|
||||
assertThat(metrics.getMaxLagFromSource() == 2000).isTrue();
|
||||
assertThat(metrics.getMinLagFromSource() == -2000).isTrue();
|
||||
assertThat(metrics.getAverageLagFromSource() == 0).isTrue();
|
||||
|
||||
metrics.reset();
|
||||
|
||||
// ##########################
|
||||
// the database time is ahead
|
||||
metrics.setTimeDifference(new AtomicLong(-1000));
|
||||
dbEventTime = Instant.now().minusMillis(2000);
|
||||
metrics.calculateLagMetrics(dbEventTime);
|
||||
lag = metrics.getLagFromSource();
|
||||
assertThat(lag == 3000).isTrue();
|
||||
assertThat(metrics.getMaxLagFromSource() == 3000).isTrue();
|
||||
assertThat(metrics.getMinLagFromSource() == 0).isTrue();
|
||||
assertThat(metrics.getAverageLagFromSource() == 3000).isTrue();
|
||||
|
||||
dbEventTime = Instant.now().minusMillis(3000);
|
||||
metrics.calculateLagMetrics(dbEventTime);
|
||||
lag = metrics.getLagFromSource();
|
||||
assertThat(lag == 4000).isTrue();
|
||||
assertThat(metrics.getMaxLagFromSource() == 4000).isTrue();
|
||||
assertThat(metrics.getMinLagFromSource() == 0).isTrue();
|
||||
assertThat(metrics.getAverageLagFromSource() == 3500).isTrue();
|
||||
|
||||
metrics.reset();
|
||||
|
||||
// ##########################
|
||||
// the database time is behind
|
||||
metrics.setTimeDifference(new AtomicLong(1000));
|
||||
dbEventTime = Instant.now().minusMillis(2000);
|
||||
metrics.calculateLagMetrics(dbEventTime);
|
||||
lag = metrics.getLagFromSource();
|
||||
assertThat(lag == 1000).isTrue();
|
||||
assertThat(metrics.getMaxLagFromSource() == 1000).isTrue();
|
||||
assertThat(metrics.getMinLagFromSource() == 0).isTrue();
|
||||
assertThat(metrics.getAverageLagFromSource() == 1000).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOtherMetrics() {
|
||||
metrics.incrementScnFreezeCounter();
|
||||
assertThat(metrics.getScnFreezeCounter() == 1).isTrue();
|
||||
|
||||
metrics.incrementErrorCounter();
|
||||
assertThat(metrics.getErrorCounter() == 1).isTrue();
|
||||
|
||||
metrics.incrementWarningCounter();
|
||||
assertThat(metrics.getWarningCounter() == 1).isTrue();
|
||||
|
||||
metrics.incrementCommittedDmlCounter(5_000);
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
metrics.incrementCapturedDmlCounter();
|
||||
metrics.incrementCommittedTransactions();
|
||||
}
|
||||
assertThat(metrics.getCapturedDmlCount() == 1000).isTrue();
|
||||
assertThat(metrics.getCapturedDmlThroughput() > 10_000).isTrue();
|
||||
assertThat(metrics.getNumberOfCommittedTransactions() == 1000).isTrue();
|
||||
assertThat(metrics.getCommitThroughput() >= 1_000).isTrue();
|
||||
|
||||
metrics.incrementRolledBackTransactions();
|
||||
assertThat(metrics.getNumberOfRolledBackTransactions() == 1).isTrue();
|
||||
|
||||
metrics.setActiveTransactions(5);
|
||||
assertThat(metrics.getNumberOfActiveTransactions() == 5).isTrue();
|
||||
|
||||
metrics.addRolledBackTransactionId("rolledback id");
|
||||
assertThat(metrics.getNumberOfRolledBackTransactions() == 1).isTrue();
|
||||
assertThat(metrics.getRolledBackTransactionIds().contains("rolledback id")).isTrue();
|
||||
|
||||
metrics.addAbandonedTransactionId("abandoned id");
|
||||
assertThat(metrics.getAbandonedTransactionIds().size() == 1).isTrue();
|
||||
assertThat(metrics.getAbandonedTransactionIds().contains("abandoned id")).isTrue();
|
||||
|
||||
metrics.setOldestScn(10L);
|
||||
assertThat(metrics.getOldestScn() == 10L).isTrue();
|
||||
|
||||
metrics.setCommittedScn(10L);
|
||||
assertThat(metrics.getCommittedScn() == 10L).isTrue();
|
||||
|
||||
assertThat(metrics.toString().contains("capturedDmlCounter=1000")).isTrue();
|
||||
|
||||
}
|
||||
}
|
@ -0,0 +1,361 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import static io.debezium.config.CommonConnectorConfig.DEFAULT_MAX_BATCH_SIZE;
|
||||
import static io.debezium.config.CommonConnectorConfig.DEFAULT_MAX_QUEUE_SIZE;
|
||||
import static junit.framework.TestCase.assertNotSame;
|
||||
import static junit.framework.TestCase.assertSame;
|
||||
import static junit.framework.TestCase.assertTrue;
|
||||
import static org.fest.assertions.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
|
||||
import io.debezium.config.Configuration;
|
||||
import io.debezium.connector.base.ChangeEventQueue;
|
||||
import io.debezium.connector.oracle.OracleConnector;
|
||||
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||
import io.debezium.connector.oracle.xstream.LcrPosition;
|
||||
import io.debezium.pipeline.DataChangeEvent;
|
||||
import io.debezium.pipeline.ErrorHandler;
|
||||
import io.debezium.pipeline.txmetadata.TransactionContext;
|
||||
|
||||
/**
|
||||
* @author Andrey Pustovetov
|
||||
*/
|
||||
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||
public class TransactionalBufferTest {
|
||||
|
||||
private static final String SERVER_NAME = "serverX";
|
||||
private static final String TRANSACTION_ID = "transaction";
|
||||
private static final String OTHER_TRANSACTION_ID = "other_transaction";
|
||||
private static final String SQL_ONE = "update table";
|
||||
private static final String SQL_TWO = "insert into table";
|
||||
private static final String MESSAGE = "OK";
|
||||
private static final BigDecimal SCN = BigDecimal.ONE;
|
||||
private static final BigDecimal OTHER_SCN = BigDecimal.TEN;
|
||||
private static final BigDecimal LARGEST_SCN = BigDecimal.valueOf(100L);
|
||||
private static final Timestamp TIMESTAMP = new Timestamp(System.currentTimeMillis());
|
||||
private static final Configuration config = new Configuration() {
|
||||
@Override
|
||||
public Set<String> keys() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getString(String key) {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
private static final OracleConnectorConfig connectorConfig = new OracleConnectorConfig(config);
|
||||
private static OracleOffsetContext offsetContext;
|
||||
|
||||
private ErrorHandler errorHandler;
|
||||
private TransactionalBuffer transactionalBuffer;
|
||||
private TransactionalBufferMetrics metrics;
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
ChangeEventQueue<DataChangeEvent> queue = new ChangeEventQueue.Builder<DataChangeEvent>()
|
||||
.pollInterval(Duration.of(DEFAULT_MAX_QUEUE_SIZE, ChronoUnit.MILLIS))
|
||||
.maxBatchSize(DEFAULT_MAX_BATCH_SIZE)
|
||||
.maxQueueSize(DEFAULT_MAX_QUEUE_SIZE)
|
||||
.build();
|
||||
errorHandler = new ErrorHandler(OracleConnector.class, SERVER_NAME, queue);
|
||||
metrics = mock(TransactionalBufferMetrics.class);
|
||||
transactionalBuffer = new TransactionalBuffer(SERVER_NAME, errorHandler, metrics,
|
||||
DEFAULT_MAX_QUEUE_SIZE);
|
||||
}
|
||||
|
||||
@After
|
||||
public void after() throws InterruptedException {
|
||||
transactionalBuffer.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIsEmpty() {
|
||||
assertThat(transactionalBuffer.isEmpty()).isEqualTo(true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIsNotEmptyWhenTransactionIsRegistered() {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
assertThat(transactionalBuffer.isEmpty()).isEqualTo(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIsNotEmptyWhenTransactionIsCommitting() {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> Thread.sleep(1000));
|
||||
offsetContext = new OracleOffsetContext(connectorConfig, SCN.longValue(), SCN.longValue(), (LcrPosition) null, false, true, new TransactionContext());
|
||||
transactionalBuffer.commit(TRANSACTION_ID, SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||
assertThat(transactionalBuffer.isEmpty()).isEqualTo(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIsEmptyWhenTransactionIsCommitted() throws InterruptedException {
|
||||
CountDownLatch commitLatch = new CountDownLatch(1);
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> commitLatch.countDown());
|
||||
offsetContext = new OracleOffsetContext(connectorConfig, SCN.longValue(), SCN.longValue(), (LcrPosition) null, false, true, new TransactionContext());
|
||||
transactionalBuffer.commit(TRANSACTION_ID, SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||
commitLatch.await();
|
||||
Thread.sleep(1000);
|
||||
assertThat(transactionalBuffer.isEmpty()).isEqualTo(true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIsEmptyWhenTransactionIsRolledBack() {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||
assertThat(transactionalBuffer.isEmpty()).isEqualTo(true);
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(SCN);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNonEmptyFirstTransactionIsRolledBack() {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "insert", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||
assertThat(transactionalBuffer.isEmpty()).isEqualTo(false);
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||
assertThat(transactionalBuffer.getRolledBackTransactionIds().contains(TRANSACTION_ID)).isTrue();
|
||||
assertThat(transactionalBuffer.getRolledBackTransactionIds().contains(OTHER_TRANSACTION_ID)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNonEmptySecondTransactionIsRolledBack() {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.rollback(OTHER_TRANSACTION_ID, "");
|
||||
assertThat(transactionalBuffer.isEmpty()).isEqualTo(false);
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||
assertThat(transactionalBuffer.getRolledBackTransactionIds().contains(TRANSACTION_ID)).isFalse();
|
||||
assertThat(transactionalBuffer.getRolledBackTransactionIds().contains(OTHER_TRANSACTION_ID)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCalculateScnWhenTransactionIsCommitted() throws InterruptedException {
|
||||
CountDownLatch commitLatch = new CountDownLatch(1);
|
||||
AtomicReference<BigDecimal> smallestScnContainer = new AtomicReference<>();
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
smallestScnContainer.set(smallestScn);
|
||||
commitLatch.countDown();
|
||||
});
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(SCN); // before commit
|
||||
offsetContext = new OracleOffsetContext(connectorConfig, SCN.longValue(), SCN.longValue(), null, false, true, new TransactionContext());
|
||||
transactionalBuffer.commit(TRANSACTION_ID, SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||
commitLatch.await();
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(SCN); // after commit
|
||||
|
||||
assertThat(smallestScnContainer.get()).isNull();
|
||||
|
||||
assertThat(transactionalBuffer.getRolledBackTransactionIds().isEmpty()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCalculateScnWhenFirstTransactionIsCommitted() throws InterruptedException {
|
||||
CountDownLatch commitLatch = new CountDownLatch(1);
|
||||
AtomicReference<BigDecimal> smallestScnContainer = new AtomicReference<>();
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
smallestScnContainer.set(smallestScn);
|
||||
commitLatch.countDown();
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN); // before commit
|
||||
offsetContext = new OracleOffsetContext(connectorConfig, SCN.longValue(), SCN.longValue(), null, false, true, new TransactionContext());
|
||||
transactionalBuffer.commit(TRANSACTION_ID, SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||
commitLatch.await();
|
||||
// after commit, it stays the same because OTHER_TRANSACTION_ID is not committed yet
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||
|
||||
assertThat(smallestScnContainer.get()).isEqualTo(OTHER_SCN);
|
||||
assertThat(transactionalBuffer.getRolledBackTransactionIds().isEmpty()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCalculateScnWhenSecondTransactionIsCommitted() throws InterruptedException {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
CountDownLatch commitLatch = new CountDownLatch(1);
|
||||
AtomicReference<BigDecimal> smallestScnContainer = new AtomicReference<>();
|
||||
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
smallestScnContainer.set(smallestScn);
|
||||
commitLatch.countDown();
|
||||
});
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN); // before commit
|
||||
offsetContext = new OracleOffsetContext(connectorConfig, OTHER_SCN.longValue(), OTHER_SCN.longValue(), null, false, true, new TransactionContext());
|
||||
transactionalBuffer.commit(OTHER_TRANSACTION_ID, OTHER_SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||
commitLatch.await();
|
||||
assertThat(smallestScnContainer.get()).isEqualTo(SCN);
|
||||
// after committing OTHER_TRANSACTION_ID
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||
assertThat(transactionalBuffer.getRolledBackTransactionIds().isEmpty()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testResetLargestScn() {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN); // before commit
|
||||
offsetContext = new OracleOffsetContext(connectorConfig, OTHER_SCN.longValue(), OTHER_SCN.longValue(), null, false, true, new TransactionContext());
|
||||
transactionalBuffer.commit(OTHER_TRANSACTION_ID, OTHER_SCN, offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN); // after commit
|
||||
|
||||
transactionalBuffer.resetLargestScn(null);
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(BigDecimal.ZERO);
|
||||
transactionalBuffer.resetLargestScn(OTHER_SCN.longValue());
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAbandoningOneTransaction() {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.abandonLongTransactions(SCN.longValue());
|
||||
assertThat(transactionalBuffer.isEmpty()).isEqualTo(true);
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(BigDecimal.ZERO);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAbandoningTransactionHavingAnotherOne() {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.abandonLongTransactions(SCN.longValue());
|
||||
assertThat(transactionalBuffer.isEmpty()).isEqualTo(false);
|
||||
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTransactionDump() {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), SQL_ONE, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), SQL_ONE, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), SQL_TWO, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
assertThat(transactionalBuffer.toString()).contains(SQL_ONE);
|
||||
assertThat(transactionalBuffer.toString()).contains(SQL_TWO);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDuplicatedRedoSql() {
|
||||
|
||||
assertThat(transactionalBuffer.getLargestScn().equals(BigDecimal.ZERO));
|
||||
|
||||
final String insertIntoATable = "insert into a table";
|
||||
final String anotherInsertIntoATable = "another insert into a table";
|
||||
final String duplicatedInsertIntoATable = "duplicated insert into a table";
|
||||
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), insertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, OTHER_SCN, Instant.now(), anotherInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
assertThat(transactionalBuffer.getLargestScn().equals(OTHER_SCN));
|
||||
assertThat(transactionalBuffer.toString().contains(insertIntoATable));
|
||||
assertThat(transactionalBuffer.toString().contains(anotherInsertIntoATable));
|
||||
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||
|
||||
// duplications are OK in different transactions
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), duplicatedInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, SCN, Instant.now(), duplicatedInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
assertThat(transactionalBuffer.toString().indexOf(duplicatedInsertIntoATable) != transactionalBuffer.toString().lastIndexOf(duplicatedInsertIntoATable));
|
||||
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||
transactionalBuffer.rollback(OTHER_TRANSACTION_ID, "");
|
||||
|
||||
// duplications are NOT OK in a transactions for different SCNs if they are sequential
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), duplicatedInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, OTHER_SCN, Instant.now(), duplicatedInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
assertThat(transactionalBuffer.toString().indexOf(duplicatedInsertIntoATable) == transactionalBuffer.toString().lastIndexOf(duplicatedInsertIntoATable));
|
||||
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||
|
||||
// duplications are OK in a transactions for different SCNs if they are NOT sequential
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), duplicatedInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, OTHER_SCN, Instant.now(), insertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, LARGEST_SCN, Instant.now(), duplicatedInsertIntoATable,
|
||||
(timestamp, smallestScn, commitScn, counter) -> {
|
||||
});
|
||||
assertThat(transactionalBuffer.toString().indexOf(duplicatedInsertIntoATable) != transactionalBuffer.toString().lastIndexOf(duplicatedInsertIntoATable));
|
||||
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCommitQueueOverflowProcessedOnCaller() throws InterruptedException {
|
||||
Thread mainThread = Thread.currentThread();
|
||||
int commitQueueCapacity = 10;
|
||||
transactionalBuffer = new TransactionalBuffer(SERVER_NAME, errorHandler, metrics,
|
||||
commitQueueCapacity);
|
||||
int transactionToCommitCount = commitQueueCapacity + 1;
|
||||
CountDownLatch countDownLatch = new CountDownLatch(transactionToCommitCount + 1);
|
||||
for (int i = 0; i <= commitQueueCapacity; i++) {
|
||||
commitTransaction((timestamp, smallestScn, commitScn, counter) -> {
|
||||
assertNotSame(mainThread, Thread.currentThread());
|
||||
TimeUnit.MILLISECONDS.sleep(100);
|
||||
countDownLatch.countDown();
|
||||
});
|
||||
}
|
||||
// Commit one more over the capacity. This should process in the test thread, applying backpressure
|
||||
// to the caller
|
||||
commitTransaction((timestamp, smallestScn, commitScn, counter) -> {
|
||||
assertSame(mainThread, Thread.currentThread());
|
||||
countDownLatch.countDown();
|
||||
});
|
||||
|
||||
TimeUnit.SECONDS.sleep(2);
|
||||
|
||||
// Commit one more over the capacity. After delay, the executor had time to recover and empty its queue
|
||||
// This should go back to processing in the executor thread
|
||||
commitTransaction((timestamp, smallestScn, commitScn, counter) -> {
|
||||
assertNotSame(mainThread, Thread.currentThread());
|
||||
countDownLatch.countDown();
|
||||
});
|
||||
|
||||
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
private void commitTransaction(TransactionalBuffer.CommitCallback commitCallback) {
|
||||
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", commitCallback);
|
||||
offsetContext = new OracleOffsetContext(connectorConfig, SCN.longValue(), SCN.longValue(), null, false, true, new TransactionContext());
|
||||
transactionalBuffer.commit(TRANSACTION_ID, SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||
}
|
||||
}
|
@ -0,0 +1,94 @@
|
||||
/*
|
||||
* Copyright Debezium Authors.
|
||||
*
|
||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package io.debezium.connector.oracle.logminer;
|
||||
|
||||
import static org.fest.assertions.Assertions.assertThat;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.Timestamp;
|
||||
import java.sql.Types;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestRule;
|
||||
|
||||
import io.debezium.connector.oracle.antlr.OracleDdlParser;
|
||||
import io.debezium.connector.oracle.jsqlparser.SimpleDmlParser;
|
||||
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueImpl;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntryImpl;
|
||||
import io.debezium.data.Envelope;
|
||||
import io.debezium.relational.Tables;
|
||||
import io.debezium.util.IoUtil;
|
||||
|
||||
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||
public class ValueHolderTest {
|
||||
private static final String TABLE_NAME = "TEST";
|
||||
private static final String CATALOG_NAME = "CATALOG";
|
||||
private static final String SCHEMA_NAME = "DEBEZIUM";
|
||||
private OracleDdlParser ddlParser;
|
||||
private SimpleDmlParser sqlDmlParser;
|
||||
private Tables tables;
|
||||
private static final String FULL_TABLE_NAME = SCHEMA_NAME + "\".\"" + TABLE_NAME;
|
||||
|
||||
@Rule
|
||||
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
OracleChangeRecordValueConverter converters = new OracleChangeRecordValueConverter(null);
|
||||
ddlParser = new OracleDdlParser(true, CATALOG_NAME, SCHEMA_NAME);
|
||||
sqlDmlParser = new SimpleDmlParser(CATALOG_NAME, SCHEMA_NAME, converters);
|
||||
tables = new Tables();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testValueHolders() throws Exception {
|
||||
LogMinerColumnValue column1 = new LogMinerColumnValueImpl("COLUMN1", Types.NUMERIC);
|
||||
assertThat(column1.equals(column1)).isTrue();
|
||||
assertThat(column1.equals(null)).isFalse();
|
||||
assertThat(new LogMinerColumnValueWrapper(column1).isProcessed()).isFalse();
|
||||
|
||||
column1.setColumnData(new BigDecimal(5));
|
||||
LogMinerColumnValue column2 = new LogMinerColumnValueImpl("COLUMN2", Types.VARCHAR);
|
||||
column2.setColumnData("Text");
|
||||
List<LogMinerColumnValue> newValues = new ArrayList<>();
|
||||
newValues.add(column1);
|
||||
newValues.add(column2);
|
||||
LogMinerDmlEntryImpl dmlEntryExpected = new LogMinerDmlEntryImpl(Envelope.Operation.CREATE, newValues, Collections.emptyList());
|
||||
dmlEntryExpected.setTransactionId("transaction_id");
|
||||
dmlEntryExpected.setObjectName(TABLE_NAME);
|
||||
dmlEntryExpected.setObjectOwner(SCHEMA_NAME);
|
||||
dmlEntryExpected.setScn(BigDecimal.ONE);
|
||||
dmlEntryExpected.setSourceTime(new Timestamp(1000));
|
||||
|
||||
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_small_table.sql", null, getClass(), null, null));
|
||||
ddlParser.parse(createStatement, tables);
|
||||
|
||||
String dml = "insert into \"" + FULL_TABLE_NAME + "\" (\"column1\",\"column2\") values ('5','Text');";
|
||||
LogMinerDmlEntry dmlEntryParsed = sqlDmlParser.parse(dml, tables, "1");
|
||||
|
||||
assertThat(dmlEntryParsed.equals(dmlEntryExpected)).isTrue();
|
||||
assertThat(dmlEntryExpected.getCommandType() == Envelope.Operation.CREATE).isTrue();
|
||||
assertThat(dmlEntryExpected.getScn().equals(BigDecimal.ONE)).isTrue();
|
||||
assertThat(dmlEntryExpected.getSourceTime().equals(new Timestamp(1000))).isTrue();
|
||||
assertThat(dmlEntryExpected.getTransactionId().equals("transaction_id")).isTrue();
|
||||
assertThat(dmlEntryExpected.getObjectOwner().equals(SCHEMA_NAME)).isTrue();
|
||||
assertThat(dmlEntryExpected.getObjectName().equals(TABLE_NAME)).isTrue();
|
||||
|
||||
assertThat(dmlEntryExpected.equals(null)).isFalse();
|
||||
assertThat(dmlEntryExpected.equals(dmlEntryExpected)).isTrue();
|
||||
}
|
||||
}
|
@ -26,6 +26,7 @@ public class TestHelper {
|
||||
public static final Path DB_HISTORY_PATH = Testing.Files.createTestingPath("file-db-history-connect.txt").toAbsolutePath();
|
||||
|
||||
public static final String CONNECTOR_USER = "c##xstrm";
|
||||
public static final String CONNECTOR_USER_LOGMINER = "c##dbzuser";
|
||||
|
||||
public static final String CONNECTOR_NAME = "oracle";
|
||||
|
||||
@ -46,13 +47,36 @@ public class TestHelper {
|
||||
*/
|
||||
public static final String TYPE_SCALE_PARAMETER_KEY = "__debezium.source.column.scale";
|
||||
|
||||
// todo: unify logminer and xstream user accounts
|
||||
public static final String CONNECTOR_USER_PASS = "xs";
|
||||
public static final String CONNECTOR_USER_PASS_LOGMINER = "dbz";
|
||||
public static final String HOST = "localhost";
|
||||
public static final String SCHEMA_USER = "debezium";
|
||||
public static final String SCHEMA_PASS = "dbz";
|
||||
public static final String DATABASE = "ORCLPDB1";
|
||||
public static final String DATABASE_CDB = "ORCLCDB";
|
||||
|
||||
public static String getConnectorUserName() {
|
||||
if (TestHelper.adapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
|
||||
return CONNECTOR_USER_LOGMINER;
|
||||
}
|
||||
return CONNECTOR_USER;
|
||||
}
|
||||
|
||||
public static String getConnectorUserPassword() {
|
||||
if (TestHelper.adapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
|
||||
return CONNECTOR_USER_PASS_LOGMINER;
|
||||
}
|
||||
return CONNECTOR_USER_PASS;
|
||||
}
|
||||
|
||||
private static JdbcConfiguration defaultJdbcConfig() {
|
||||
return JdbcConfiguration.copy(Configuration.fromSystemProperties("database."))
|
||||
.withDefault(JdbcConfiguration.HOSTNAME, "localhost")
|
||||
.withDefault(JdbcConfiguration.HOSTNAME, HOST)
|
||||
.withDefault(JdbcConfiguration.PORT, 1521)
|
||||
.withDefault(JdbcConfiguration.USER, CONNECTOR_USER)
|
||||
.withDefault(JdbcConfiguration.PASSWORD, "xs")
|
||||
.withDefault(JdbcConfiguration.DATABASE, "ORCLCDB")
|
||||
.withDefault(JdbcConfiguration.USER, getConnectorUserName())
|
||||
.withDefault(JdbcConfiguration.PASSWORD, getConnectorUserPassword())
|
||||
.withDefault(JdbcConfiguration.DATABASE, DATABASE_CDB)
|
||||
.build();
|
||||
}
|
||||
|
||||
@ -71,6 +95,7 @@ public static Configuration.Builder defaultConfig() {
|
||||
.with(OracleConnectorConfig.PDB_NAME, "ORCLPDB1")
|
||||
.with(OracleConnectorConfig.XSTREAM_SERVER_NAME, "dbzxout")
|
||||
.with(OracleConnectorConfig.DATABASE_HISTORY, FileDatabaseHistory.class)
|
||||
.with(OracleConnectorConfig.SCHEMA_NAME, SCHEMA_USER)
|
||||
.with(FileDatabaseHistory.FILE_PATH, DB_HISTORY_PATH)
|
||||
.with(OracleConnectorConfig.INCLUDE_SCHEMA_CHANGES, false);
|
||||
}
|
||||
@ -79,7 +104,7 @@ public static OracleConnection defaultConnection() {
|
||||
Configuration config = defaultConfig().build();
|
||||
Configuration jdbcConfig = config.subset("database.", true);
|
||||
|
||||
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());
|
||||
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory(), TestHelper.class::getClassLoader);
|
||||
|
||||
String pdbName = new OracleConnectorConfig(config).getPdbName();
|
||||
|
||||
@ -90,16 +115,29 @@ public static OracleConnection defaultConnection() {
|
||||
return jdbcConnection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Database level connection.
|
||||
* this is PDB level connector with LogMiner adapter
|
||||
* @return OracleConnection
|
||||
*/
|
||||
public static OracleConnection logMinerPdbConnection() {
|
||||
Configuration jdbcConfig = testJdbcConfig().edit()
|
||||
.with(OracleConnectorConfig.CONNECTOR_ADAPTER, "LogMiner")
|
||||
.with(OracleConnectorConfig.DRIVER_TYPE, "thin")
|
||||
.build();
|
||||
return new OracleConnection(jdbcConfig, new OracleConnectionFactory(), TestHelper.class::getClassLoader);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a JDBC configuration for the test data schema and user (NOT the XStream user).
|
||||
*/
|
||||
private static JdbcConfiguration testJdbcConfig() {
|
||||
return JdbcConfiguration.copy(Configuration.fromSystemProperties("database."))
|
||||
.withDefault(JdbcConfiguration.HOSTNAME, "localhost")
|
||||
.withDefault(JdbcConfiguration.HOSTNAME, HOST)
|
||||
.withDefault(JdbcConfiguration.PORT, 1521)
|
||||
.withDefault(JdbcConfiguration.USER, "debezium")
|
||||
.withDefault(JdbcConfiguration.PASSWORD, "dbz")
|
||||
.withDefault(JdbcConfiguration.DATABASE, "ORCLPDB1")
|
||||
.withDefault(JdbcConfiguration.USER, SCHEMA_USER)
|
||||
.withDefault(JdbcConfiguration.PASSWORD, SCHEMA_PASS)
|
||||
.withDefault(JdbcConfiguration.DATABASE, DATABASE)
|
||||
.build();
|
||||
}
|
||||
|
||||
@ -108,11 +146,11 @@ private static JdbcConfiguration testJdbcConfig() {
|
||||
*/
|
||||
private static JdbcConfiguration adminJdbcConfig() {
|
||||
return JdbcConfiguration.copy(Configuration.fromSystemProperties("database.admin."))
|
||||
.withDefault(JdbcConfiguration.HOSTNAME, "localhost")
|
||||
.withDefault(JdbcConfiguration.HOSTNAME, HOST)
|
||||
.withDefault(JdbcConfiguration.PORT, 1521)
|
||||
.withDefault(JdbcConfiguration.USER, "sys as sysdba")
|
||||
.withDefault(JdbcConfiguration.PASSWORD, "top_secret")
|
||||
.withDefault(JdbcConfiguration.DATABASE, "ORCLPDB1")
|
||||
.withDefault(JdbcConfiguration.DATABASE, DATABASE)
|
||||
.build();
|
||||
}
|
||||
|
||||
@ -140,7 +178,7 @@ public static OracleConnection testConnection() {
|
||||
Configuration config = testConfig().build();
|
||||
Configuration jdbcConfig = config.subset("database.", true);
|
||||
|
||||
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());
|
||||
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory(), TestHelper.class::getClassLoader);
|
||||
try {
|
||||
jdbcConnection.setAutoCommit(false);
|
||||
}
|
||||
@ -161,7 +199,7 @@ public static OracleConnection adminConnection() {
|
||||
Configuration config = adminConfig().build();
|
||||
Configuration jdbcConfig = config.subset("database.", true);
|
||||
|
||||
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());
|
||||
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory(), TestHelper.class::getClassLoader);
|
||||
try {
|
||||
jdbcConnection.setAutoCommit(false);
|
||||
}
|
||||
@ -192,4 +230,9 @@ public static void dropTable(OracleConnection connection, String table) {
|
||||
public static int defaultMessageConsumerPollTimeout() {
|
||||
return 120;
|
||||
}
|
||||
|
||||
public static OracleConnectorConfig.ConnectorAdapter adapter() {
|
||||
final String s = System.getProperty(OracleConnectorConfig.CONNECTOR_ADAPTER.name());
|
||||
return (s == null || s.length() == 0) ? OracleConnectorConfig.ConnectorAdapter.XSTREAM : OracleConnectorConfig.ConnectorAdapter.parse(s);
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,4 @@
|
||||
CREATE TABLE TEST
|
||||
(COLUMN1 NUMBER(19) NOT NULL,
|
||||
COLUMN2 VARCHAR2(255)
|
||||
PRIMARY KEY (COLUMN1));
|
@ -1,13 +1,16 @@
|
||||
create table DEBEZIUM
|
||||
create table TEST
|
||||
(id number(19) not null,
|
||||
col1 number(4,2),
|
||||
col1 numeric(4,2),
|
||||
col2 varchar2(255) default 'debezium' not null ,
|
||||
col3 nvarchar2(255) not null,
|
||||
col4 char(4),
|
||||
col5 nchar(4),
|
||||
col6 float(126),
|
||||
col7 date,
|
||||
-- todo: dbz-137 this field was removed, re-add it?
|
||||
-- col7 date,
|
||||
col8 timestamp,
|
||||
col9 blob,
|
||||
col10 clob,
|
||||
col11 "mdsys"."sdo_geometry",
|
||||
col12 number(1,0),
|
||||
primary key (id));
|
||||
|
@ -13,3 +13,5 @@ log4j.logger.io.debezium.embedded.EmbeddedEngine$EmbeddedConfig=WARN
|
||||
#log4j.logger.io.debezium.embedded.EmbeddedEngine=DEBUG
|
||||
log4j.logger.io.debezium.core=DEBUG
|
||||
log4j.logger.io.debezium.connector.oracle=DEBUG
|
||||
log4j.logger.io.debezium.connector.oracle.logminer.LogMinerQueryResultProcessor=TRACE
|
||||
log4j.logger.io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSource=TRACE
|
Loading…
Reference in New Issue
Block a user