DBZ-137 log miner oracle contribution
This commit is contained in:
parent
7c2cc444bb
commit
d344ca48f8
@ -74,6 +74,12 @@ Then the Oracle connector can be built like so:
|
|||||||
|
|
||||||
$ mvn clean install -pl debezium-connector-oracle -am -Poracle -Dinstantclient.dir=/path/to/instant-client-dir
|
$ mvn clean install -pl debezium-connector-oracle -am -Poracle -Dinstantclient.dir=/path/to/instant-client-dir
|
||||||
|
|
||||||
|
#### Oracle Log Miner
|
||||||
|
|
||||||
|
If the connector is to be built and tested using the Oracle Log Miner implementation, it can be built like so:
|
||||||
|
|
||||||
|
$ mvn clean install -pl debezium-connector-oracle -am -Poracle,logminer -Dinstantclient.dir=/home/to/instant-client-dir
|
||||||
|
|
||||||
#### For Oracle 11g
|
#### For Oracle 11g
|
||||||
|
|
||||||
To run Debezium Oracle connector with Oracle 11g, add these additional parameters. If running with Oracle 12c+, leave these parameters to default.
|
To run Debezium Oracle connector with Oracle 11g, add these additional parameters. If running with Oracle 12c+, leave these parameters to default.
|
||||||
|
@ -41,6 +41,11 @@
|
|||||||
<groupId>org.antlr</groupId>
|
<groupId>org.antlr</groupId>
|
||||||
<artifactId>antlr4-runtime</artifactId>
|
<artifactId>antlr4-runtime</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.github.jsqlparser</groupId>
|
||||||
|
<artifactId>jsqlparser</artifactId>
|
||||||
|
<version>2.1</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<!-- Testing -->
|
<!-- Testing -->
|
||||||
<dependency>
|
<dependency>
|
||||||
@ -85,6 +90,12 @@
|
|||||||
<artifactId>awaitility</artifactId>
|
<artifactId>awaitility</artifactId>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.mockito</groupId>
|
||||||
|
<artifactId>mockito-core</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<!-- Required by VerifyRecord -->
|
<!-- Required by VerifyRecord -->
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>io.confluent</groupId>
|
<groupId>io.confluent</groupId>
|
||||||
@ -93,6 +104,10 @@
|
|||||||
</dependency>
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<adapter.name>xstream</adapter.name>
|
||||||
|
</properties>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
<resources>
|
<resources>
|
||||||
<!-- Apply the properties set in the POM to the resource files -->
|
<!-- Apply the properties set in the POM to the resource files -->
|
||||||
@ -139,6 +154,9 @@
|
|||||||
<enableAssertions>true</enableAssertions>
|
<enableAssertions>true</enableAssertions>
|
||||||
<forkCount>1</forkCount>
|
<forkCount>1</forkCount>
|
||||||
<argLine>-Djava.library.path=${instantclient.dir} -Ddebezium.embedded.shutdown.pause.before.interrupt.ms=20000</argLine>
|
<argLine>-Djava.library.path=${instantclient.dir} -Ddebezium.embedded.shutdown.pause.before.interrupt.ms=20000</argLine>
|
||||||
|
<systemPropertyVariables>
|
||||||
|
<database.connection.adapter>${adapter.name}</database.connection.adapter>
|
||||||
|
</systemPropertyVariables>
|
||||||
</configuration>
|
</configuration>
|
||||||
</plugin>
|
</plugin>
|
||||||
</plugins>
|
</plugins>
|
||||||
@ -186,5 +204,11 @@
|
|||||||
</plugins>
|
</plugins>
|
||||||
</build>
|
</build>
|
||||||
</profile>
|
</profile>
|
||||||
|
<profile>
|
||||||
|
<id>logminer</id>
|
||||||
|
<properties>
|
||||||
|
<adapter.name>logminer</adapter.name>
|
||||||
|
</properties>
|
||||||
|
</profile>
|
||||||
</profiles>
|
</profiles>
|
||||||
</project>
|
</project>
|
||||||
|
@ -0,0 +1,42 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle;
|
||||||
|
|
||||||
|
import io.debezium.data.Envelope.Operation;
|
||||||
|
import io.debezium.pipeline.spi.OffsetContext;
|
||||||
|
import io.debezium.relational.RelationalChangeRecordEmitter;
|
||||||
|
import io.debezium.relational.Table;
|
||||||
|
import io.debezium.util.Clock;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Base class to emit change data based on a single entry event.
|
||||||
|
*/
|
||||||
|
public abstract class BaseChangeRecordEmitter<T> extends RelationalChangeRecordEmitter {
|
||||||
|
|
||||||
|
protected final Table table;
|
||||||
|
|
||||||
|
protected BaseChangeRecordEmitter(OffsetContext offset, Table table, Clock clock) {
|
||||||
|
super(offset, clock);
|
||||||
|
this.table = table;
|
||||||
|
}
|
||||||
|
|
||||||
|
abstract protected Operation getOperation();
|
||||||
|
|
||||||
|
abstract protected String getColumnName(T columnValue);
|
||||||
|
|
||||||
|
abstract protected Object getColumnData(T columnValue);
|
||||||
|
|
||||||
|
protected Object[] getColumnValues(T[] columnValues) {
|
||||||
|
Object[] values = new Object[table.columns().size()];
|
||||||
|
|
||||||
|
for (T columnValue : columnValues) {
|
||||||
|
int index = table.columnWithName(getColumnName(columnValue)).position() - 1;
|
||||||
|
values[index] = getColumnData(columnValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
}
|
@ -18,25 +18,31 @@
|
|||||||
import io.debezium.schema.SchemaChangeEvent;
|
import io.debezium.schema.SchemaChangeEvent;
|
||||||
import io.debezium.schema.SchemaChangeEvent.SchemaChangeEventType;
|
import io.debezium.schema.SchemaChangeEvent.SchemaChangeEventType;
|
||||||
|
|
||||||
import oracle.streams.DDLLCR;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@link SchemaChangeEventEmitter} implementation based on Oracle.
|
* {@link SchemaChangeEventEmitter} implementation based on Oracle.
|
||||||
*
|
*
|
||||||
* @author Gunnar Morling
|
* @author Gunnar Morling
|
||||||
*/
|
*/
|
||||||
public class OracleSchemaChangeEventEmitter implements SchemaChangeEventEmitter {
|
public class BaseOracleSchemaChangeEventEmitter implements SchemaChangeEventEmitter {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(OracleSchemaChangeEventEmitter.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(BaseOracleSchemaChangeEventEmitter.class);
|
||||||
|
|
||||||
private final OracleOffsetContext offsetContext;
|
private final OracleOffsetContext offsetContext;
|
||||||
private final TableId tableId;
|
private final TableId tableId;
|
||||||
private final DDLLCR ddlLcr;
|
private String sourceDatabaseName;
|
||||||
|
private String objectOwner;
|
||||||
|
private String ddlText;
|
||||||
|
private String commandType;
|
||||||
|
|
||||||
public OracleSchemaChangeEventEmitter(OracleOffsetContext offsetContext, TableId tableId, DDLLCR ddlLcr) {
|
public BaseOracleSchemaChangeEventEmitter(OracleOffsetContext offsetContext, TableId tableId,
|
||||||
|
String sourceDatabaseName, String objectOwner, String ddlText,
|
||||||
|
String commandType) {
|
||||||
this.offsetContext = offsetContext;
|
this.offsetContext = offsetContext;
|
||||||
this.tableId = tableId;
|
this.tableId = tableId;
|
||||||
this.ddlLcr = ddlLcr;
|
this.sourceDatabaseName = sourceDatabaseName;
|
||||||
|
this.objectOwner = objectOwner;
|
||||||
|
this.ddlText = ddlText;
|
||||||
|
this.commandType = commandType;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -49,13 +55,13 @@ public void emitSchemaChangeEvent(Receiver receiver) throws InterruptedException
|
|||||||
Tables tables = new Tables();
|
Tables tables = new Tables();
|
||||||
|
|
||||||
OracleDdlParser parser = new OracleDdlParser();
|
OracleDdlParser parser = new OracleDdlParser();
|
||||||
parser.setCurrentDatabase(ddlLcr.getSourceDatabaseName());
|
parser.setCurrentDatabase(sourceDatabaseName);
|
||||||
parser.setCurrentSchema(ddlLcr.getObjectOwner());
|
parser.setCurrentSchema(objectOwner);
|
||||||
parser.parse(ddlLcr.getDDLText(), tables);
|
parser.parse(ddlText, tables);
|
||||||
|
|
||||||
Set<TableId> changedTableIds = tables.drainChanges();
|
Set<TableId> changedTableIds = tables.drainChanges();
|
||||||
if (changedTableIds.isEmpty()) {
|
if (changedTableIds.isEmpty()) {
|
||||||
throw new IllegalArgumentException("Couldn't parse DDL statement " + ddlLcr.getDDLText());
|
throw new IllegalArgumentException("Couldn't parse DDL statement " + ddlText);
|
||||||
}
|
}
|
||||||
|
|
||||||
Table table = tables.forTable(tableId);
|
Table table = tables.forTable(tableId);
|
||||||
@ -64,16 +70,16 @@ public void emitSchemaChangeEvent(Receiver receiver) throws InterruptedException
|
|||||||
offsetContext.getPartition(),
|
offsetContext.getPartition(),
|
||||||
offsetContext.getOffset(),
|
offsetContext.getOffset(),
|
||||||
offsetContext.getSourceInfo(),
|
offsetContext.getSourceInfo(),
|
||||||
ddlLcr.getSourceDatabaseName(),
|
sourceDatabaseName,
|
||||||
ddlLcr.getObjectOwner(),
|
objectOwner,
|
||||||
ddlLcr.getDDLText(),
|
ddlText,
|
||||||
table,
|
table,
|
||||||
eventType,
|
eventType,
|
||||||
false));
|
false));
|
||||||
}
|
}
|
||||||
|
|
||||||
private SchemaChangeEventType getSchemaChangeEventType() {
|
private SchemaChangeEventType getSchemaChangeEventType() {
|
||||||
switch (ddlLcr.getCommandType()) {
|
switch (commandType) {
|
||||||
case "CREATE TABLE":
|
case "CREATE TABLE":
|
||||||
return SchemaChangeEventType.CREATE;
|
return SchemaChangeEventType.CREATE;
|
||||||
case "ALTER TABLE":
|
case "ALTER TABLE":
|
||||||
@ -83,7 +89,7 @@ private SchemaChangeEventType getSchemaChangeEventType() {
|
|||||||
LOGGER.warn("DROP TABLE not yet implemented");
|
LOGGER.warn("DROP TABLE not yet implemented");
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
LOGGER.debug("Ignoring DDL event of type {}", ddlLcr.getCommandType());
|
LOGGER.debug("Ignoring DDL event of type {}", commandType);
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
@ -5,6 +5,9 @@
|
|||||||
*/
|
*/
|
||||||
package io.debezium.connector.oracle;
|
package io.debezium.connector.oracle;
|
||||||
|
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
|
import io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSource;
|
||||||
|
import io.debezium.connector.oracle.xstream.XstreamStreamingChangeEventSource;
|
||||||
import io.debezium.pipeline.ErrorHandler;
|
import io.debezium.pipeline.ErrorHandler;
|
||||||
import io.debezium.pipeline.EventDispatcher;
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
import io.debezium.pipeline.source.spi.ChangeEventSourceFactory;
|
import io.debezium.pipeline.source.spi.ChangeEventSourceFactory;
|
||||||
@ -23,15 +26,20 @@ public class OracleChangeEventSourceFactory implements ChangeEventSourceFactory
|
|||||||
private final EventDispatcher<TableId> dispatcher;
|
private final EventDispatcher<TableId> dispatcher;
|
||||||
private final Clock clock;
|
private final Clock clock;
|
||||||
private final OracleDatabaseSchema schema;
|
private final OracleDatabaseSchema schema;
|
||||||
|
private final Configuration config;
|
||||||
|
private final OracleTaskContext taskContext;
|
||||||
|
|
||||||
public OracleChangeEventSourceFactory(OracleConnectorConfig configuration, OracleConnection jdbcConnection,
|
public OracleChangeEventSourceFactory(OracleConnectorConfig configuration, OracleConnection jdbcConnection,
|
||||||
ErrorHandler errorHandler, EventDispatcher<TableId> dispatcher, Clock clock, OracleDatabaseSchema schema) {
|
ErrorHandler errorHandler, EventDispatcher<TableId> dispatcher, Clock clock, OracleDatabaseSchema schema,
|
||||||
|
Configuration config, OracleTaskContext taskContext) {
|
||||||
this.configuration = configuration;
|
this.configuration = configuration;
|
||||||
this.jdbcConnection = jdbcConnection;
|
this.jdbcConnection = jdbcConnection;
|
||||||
this.errorHandler = errorHandler;
|
this.errorHandler = errorHandler;
|
||||||
this.dispatcher = dispatcher;
|
this.dispatcher = dispatcher;
|
||||||
this.clock = clock;
|
this.clock = clock;
|
||||||
this.schema = schema;
|
this.schema = schema;
|
||||||
|
this.config = config;
|
||||||
|
this.taskContext = taskContext;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -42,13 +50,25 @@ public SnapshotChangeEventSource getSnapshotChangeEventSource(OffsetContext offs
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public StreamingChangeEventSource getStreamingChangeEventSource(OffsetContext offsetContext) {
|
public StreamingChangeEventSource getStreamingChangeEventSource(OffsetContext offsetContext) {
|
||||||
return new OracleStreamingChangeEventSource(
|
OracleConnectorConfig.ConnectorAdapter adapter = configuration.getAdapter();
|
||||||
|
if (adapter == OracleConnectorConfig.ConnectorAdapter.XSTREAM) {
|
||||||
|
return new XstreamStreamingChangeEventSource(
|
||||||
|
configuration,
|
||||||
|
(OracleOffsetContext) offsetContext,
|
||||||
|
jdbcConnection,
|
||||||
|
dispatcher,
|
||||||
|
errorHandler,
|
||||||
|
clock,
|
||||||
|
schema);
|
||||||
|
}
|
||||||
|
return new LogMinerStreamingChangeEventSource(
|
||||||
configuration,
|
configuration,
|
||||||
(OracleOffsetContext) offsetContext,
|
(OracleOffsetContext) offsetContext,
|
||||||
jdbcConnection,
|
jdbcConnection,
|
||||||
dispatcher,
|
dispatcher,
|
||||||
errorHandler,
|
errorHandler,
|
||||||
clock,
|
clock,
|
||||||
schema);
|
schema,
|
||||||
|
taskContext);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -5,12 +5,22 @@
|
|||||||
*/
|
*/
|
||||||
package io.debezium.connector.oracle;
|
package io.debezium.connector.oracle;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.DatabaseMetaData;
|
||||||
|
import java.sql.PreparedStatement;
|
||||||
|
import java.sql.ResultSet;
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.sql.Statement;
|
import java.sql.Statement;
|
||||||
import java.sql.Types;
|
import java.sql.Types;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
import java.util.function.Supplier;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
@ -19,6 +29,7 @@
|
|||||||
import io.debezium.config.Configuration;
|
import io.debezium.config.Configuration;
|
||||||
import io.debezium.jdbc.JdbcConnection;
|
import io.debezium.jdbc.JdbcConnection;
|
||||||
import io.debezium.relational.Column;
|
import io.debezium.relational.Column;
|
||||||
|
import io.debezium.relational.ColumnEditor;
|
||||||
import io.debezium.relational.TableEditor;
|
import io.debezium.relational.TableEditor;
|
||||||
import io.debezium.relational.TableId;
|
import io.debezium.relational.TableId;
|
||||||
import io.debezium.relational.Tables;
|
import io.debezium.relational.Tables;
|
||||||
@ -36,8 +47,8 @@ public class OracleConnection extends JdbcConnection {
|
|||||||
*/
|
*/
|
||||||
private static final int ORACLE_UNSET_SCALE = -127;
|
private static final int ORACLE_UNSET_SCALE = -127;
|
||||||
|
|
||||||
public OracleConnection(Configuration config, ConnectionFactory connectionFactory) {
|
public OracleConnection(Configuration config, ConnectionFactory connectionFactory, Supplier<ClassLoader> classLoaderSupplier) {
|
||||||
super(config, connectionFactory);
|
super(config, connectionFactory, classLoaderSupplier);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setSessionToPdb(String pdbName) {
|
public void setSessionToPdb(String pdbName) {
|
||||||
@ -96,6 +107,89 @@ public Set<TableId> readTableNames(String databaseCatalog, String schemaNamePatt
|
|||||||
.collect(Collectors.toSet());
|
.collect(Collectors.toSet());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected Set<TableId> getAllTableIds(String catalogName, String schemaNamePattern, boolean isView) throws SQLException {
|
||||||
|
|
||||||
|
String query = "select table_name, owner from all_tables where table_name NOT LIKE 'MDRT_%' AND table_name not LIKE 'MDXT_%' " +
|
||||||
|
" and owner like '%" + schemaNamePattern.toUpperCase() + "%'";
|
||||||
|
if (isView) {
|
||||||
|
query = "select view_name, owner from all_views where owner like '%" + schemaNamePattern.toUpperCase() + "%'";
|
||||||
|
}
|
||||||
|
Set<TableId> tableIds = new HashSet<>();
|
||||||
|
|
||||||
|
try (PreparedStatement statement = connection().prepareStatement(query);
|
||||||
|
ResultSet result = statement.executeQuery();) {
|
||||||
|
while (result.next()) {
|
||||||
|
String tableName = result.getString(1);
|
||||||
|
final String schemaName = result.getString(2);
|
||||||
|
TableId tableId = new TableId(catalogName, schemaName, tableName);
|
||||||
|
tableIds.add(tableId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
LOGGER.trace("TableIds are: {}", tableIds);
|
||||||
|
}
|
||||||
|
return tableIds;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override // todo replace metadata with a query
|
||||||
|
protected Optional<ColumnEditor> readTableColumn(ResultSet columnMetadata, TableId tableId, ColumnNameFilter columnFilter) throws SQLException {
|
||||||
|
return super.readTableColumn(columnMetadata, tableId, columnFilter);
|
||||||
|
}
|
||||||
|
|
||||||
|
// todo replace metadata with something like this
|
||||||
|
private ResultSet getTableColumnsInfo(String schemaNamePattern, String tableName) throws SQLException {
|
||||||
|
String columnQuery = "select column_name, data_type, data_length, data_precision, data_scale, default_length, density, char_length from " +
|
||||||
|
"all_tab_columns where owner like '" + schemaNamePattern + "' and table_name='" + tableName + "'";
|
||||||
|
|
||||||
|
PreparedStatement statement = connection().prepareStatement(columnQuery);
|
||||||
|
return statement.executeQuery();
|
||||||
|
}
|
||||||
|
|
||||||
|
// this is much faster, we will use it until full replacement of the metadata usage TODO
|
||||||
|
public void readSchemaForCapturedTables(Tables tables, String databaseCatalog, String schemaNamePattern,
|
||||||
|
ColumnNameFilter columnFilter, boolean removeTablesNotFoundInJdbc, Set<TableId> capturedTables)
|
||||||
|
throws SQLException {
|
||||||
|
|
||||||
|
Set<TableId> tableIdsBefore = new HashSet<>(tables.tableIds());
|
||||||
|
|
||||||
|
DatabaseMetaData metadata = connection().getMetaData();
|
||||||
|
Map<TableId, List<Column>> columnsByTable = new HashMap<>();
|
||||||
|
|
||||||
|
for (TableId tableId : capturedTables) {
|
||||||
|
try (ResultSet columnMetadata = metadata.getColumns(databaseCatalog, schemaNamePattern, tableId.table(), null)) {
|
||||||
|
while (columnMetadata.next()) {
|
||||||
|
// add all whitelisted columns
|
||||||
|
readTableColumn(columnMetadata, tableId, columnFilter).ifPresent(column -> {
|
||||||
|
columnsByTable.computeIfAbsent(tableId, t -> new ArrayList<>())
|
||||||
|
.add(column.create());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read the metadata for the primary keys ...
|
||||||
|
for (Map.Entry<TableId, List<Column>> tableEntry : columnsByTable.entrySet()) {
|
||||||
|
// First get the primary key information, which must be done for *each* table ...
|
||||||
|
List<String> pkColumnNames = readPrimaryKeyNames(metadata, tableEntry.getKey());
|
||||||
|
|
||||||
|
// Then define the table ...
|
||||||
|
List<Column> columns = tableEntry.getValue();
|
||||||
|
Collections.sort(columns);
|
||||||
|
tables.overwriteTable(tableEntry.getKey(), columns, pkColumnNames, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (removeTablesNotFoundInJdbc) {
|
||||||
|
// Remove any definitions for tables that were not found in the database metadata ...
|
||||||
|
tableIdsBefore.removeAll(columnsByTable.keySet());
|
||||||
|
tableIdsBefore.forEach(tables::removeTable);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (TableId tableId : capturedTables) {
|
||||||
|
overrideOracleSpecificColumnTypes(tables, tableId, tableId);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void readSchema(Tables tables, String databaseCatalog, String schemaNamePattern, TableFilter tableFilter,
|
public void readSchema(Tables tables, String databaseCatalog, String schemaNamePattern, TableFilter tableFilter,
|
||||||
ColumnNameFilter columnFilter, boolean removeTablesNotFoundInJdbc)
|
ColumnNameFilter columnFilter, boolean removeTablesNotFoundInJdbc)
|
||||||
@ -111,36 +205,59 @@ public void readSchema(Tables tables, String databaseCatalog, String schemaNameP
|
|||||||
TableId tableIdWithCatalog = new TableId(databaseCatalog, tableId.schema(), tableId.table());
|
TableId tableIdWithCatalog = new TableId(databaseCatalog, tableId.schema(), tableId.table());
|
||||||
|
|
||||||
if (tableFilter.isIncluded(tableIdWithCatalog)) {
|
if (tableFilter.isIncluded(tableIdWithCatalog)) {
|
||||||
TableEditor editor = tables.editTable(tableId);
|
overrideOracleSpecificColumnTypes(tables, tableId, tableIdWithCatalog);
|
||||||
editor.tableId(tableIdWithCatalog);
|
|
||||||
|
|
||||||
List<String> columnNames = new ArrayList<>(editor.columnNames());
|
|
||||||
for (String columnName : columnNames) {
|
|
||||||
Column column = editor.columnWithName(columnName);
|
|
||||||
if (column.jdbcType() == Types.TIMESTAMP) {
|
|
||||||
editor.addColumn(
|
|
||||||
column.edit()
|
|
||||||
.length(column.scale().orElse(Column.UNSET_INT_VALUE))
|
|
||||||
.scale(null)
|
|
||||||
.create());
|
|
||||||
}
|
|
||||||
// NUMBER columns without scale value have it set to -127 instead of null;
|
|
||||||
// let's rectify that
|
|
||||||
else if (column.jdbcType() == OracleTypes.NUMBER) {
|
|
||||||
column.scale()
|
|
||||||
.filter(s -> s == ORACLE_UNSET_SCALE)
|
|
||||||
.ifPresent(s -> {
|
|
||||||
editor.addColumn(
|
|
||||||
column.edit()
|
|
||||||
.scale(null)
|
|
||||||
.create());
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tables.overwriteTable(editor.create());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tables.removeTable(tableId);
|
tables.removeTable(tableId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void overrideOracleSpecificColumnTypes(Tables tables, TableId tableId, TableId tableIdWithCatalog) {
|
||||||
|
TableEditor editor = tables.editTable(tableId);
|
||||||
|
editor.tableId(tableIdWithCatalog);
|
||||||
|
|
||||||
|
List<String> columnNames = new ArrayList<>(editor.columnNames());
|
||||||
|
for (String columnName : columnNames) {
|
||||||
|
Column column = editor.columnWithName(columnName);
|
||||||
|
if (column.jdbcType() == Types.TIMESTAMP) {
|
||||||
|
editor.addColumn(
|
||||||
|
column.edit()
|
||||||
|
.length(column.scale().orElse(Column.UNSET_INT_VALUE))
|
||||||
|
.scale(null)
|
||||||
|
.create());
|
||||||
|
}
|
||||||
|
// NUMBER columns without scale value have it set to -127 instead of null;
|
||||||
|
// let's rectify that
|
||||||
|
else if (column.jdbcType() == OracleTypes.NUMBER) {
|
||||||
|
column.scale()
|
||||||
|
.filter(s -> s == ORACLE_UNSET_SCALE)
|
||||||
|
.ifPresent(s -> {
|
||||||
|
editor.addColumn(
|
||||||
|
column.edit()
|
||||||
|
.scale(null)
|
||||||
|
.create());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tables.overwriteTable(editor.create());
|
||||||
|
}
|
||||||
|
|
||||||
|
public OracleConnection executeLegacy(String... sqlStatements) throws SQLException {
|
||||||
|
return executeLegacy(statement -> {
|
||||||
|
for (String sqlStatement : sqlStatements) {
|
||||||
|
if (sqlStatement != null) {
|
||||||
|
statement.execute(sqlStatement);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public OracleConnection executeLegacy(Operations operations) throws SQLException {
|
||||||
|
Connection conn = connection();
|
||||||
|
try (Statement statement = conn.createStatement()) {
|
||||||
|
operations.apply(statement);
|
||||||
|
commit();
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -22,7 +22,9 @@ public Connection connect(JdbcConfiguration config) throws SQLException {
|
|||||||
String user = config.getUser();
|
String user = config.getUser();
|
||||||
String password = config.getPassword();
|
String password = config.getPassword();
|
||||||
|
|
||||||
return DriverManager.getConnection(
|
String driverType = config.getString(OracleConnectorConfig.DRIVER_TYPE);
|
||||||
"jdbc:oracle:oci:@" + hostName + ":" + port + "/" + database, user, password);
|
final String url = "jdbc:oracle:" + driverType + ":@" + hostName + ":" + port + "/" + database;
|
||||||
|
|
||||||
|
return DriverManager.getConnection(url, user, password);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -5,6 +5,8 @@
|
|||||||
*/
|
*/
|
||||||
package io.debezium.connector.oracle;
|
package io.debezium.connector.oracle;
|
||||||
|
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
|
||||||
import org.apache.kafka.common.config.ConfigDef;
|
import org.apache.kafka.common.config.ConfigDef;
|
||||||
import org.apache.kafka.common.config.ConfigDef.Importance;
|
import org.apache.kafka.common.config.ConfigDef.Importance;
|
||||||
import org.apache.kafka.common.config.ConfigDef.Type;
|
import org.apache.kafka.common.config.ConfigDef.Type;
|
||||||
@ -17,12 +19,17 @@
|
|||||||
import io.debezium.config.Field.ValidationOutput;
|
import io.debezium.config.Field.ValidationOutput;
|
||||||
import io.debezium.connector.AbstractSourceInfo;
|
import io.debezium.connector.AbstractSourceInfo;
|
||||||
import io.debezium.connector.SourceInfoStructMaker;
|
import io.debezium.connector.SourceInfoStructMaker;
|
||||||
|
import io.debezium.connector.oracle.xstream.LcrPosition;
|
||||||
|
import io.debezium.connector.oracle.xstream.OracleVersion;
|
||||||
import io.debezium.document.Document;
|
import io.debezium.document.Document;
|
||||||
|
import io.debezium.function.Predicates;
|
||||||
import io.debezium.heartbeat.Heartbeat;
|
import io.debezium.heartbeat.Heartbeat;
|
||||||
import io.debezium.jdbc.JdbcConfiguration;
|
import io.debezium.jdbc.JdbcConfiguration;
|
||||||
|
import io.debezium.relational.ColumnId;
|
||||||
import io.debezium.relational.HistorizedRelationalDatabaseConnectorConfig;
|
import io.debezium.relational.HistorizedRelationalDatabaseConnectorConfig;
|
||||||
import io.debezium.relational.RelationalDatabaseConnectorConfig;
|
import io.debezium.relational.RelationalDatabaseConnectorConfig;
|
||||||
import io.debezium.relational.TableId;
|
import io.debezium.relational.TableId;
|
||||||
|
import io.debezium.relational.Tables;
|
||||||
import io.debezium.relational.Tables.TableFilter;
|
import io.debezium.relational.Tables.TableFilter;
|
||||||
import io.debezium.relational.history.HistoryRecordComparator;
|
import io.debezium.relational.history.HistoryRecordComparator;
|
||||||
import io.debezium.relational.history.KafkaDatabaseHistory;
|
import io.debezium.relational.history.KafkaDatabaseHistory;
|
||||||
@ -90,6 +97,13 @@ public class OracleConnectorConfig extends HistorizedRelationalDatabaseConnector
|
|||||||
.withDescription("Name of the pluggable database when working with a multi-tenant set-up. "
|
.withDescription("Name of the pluggable database when working with a multi-tenant set-up. "
|
||||||
+ "The CDB name must be given via " + DATABASE_NAME.name() + " in this case.");
|
+ "The CDB name must be given via " + DATABASE_NAME.name() + " in this case.");
|
||||||
|
|
||||||
|
public static final Field SCHEMA_NAME = Field.create(DATABASE_CONFIG_PREFIX + "schema")
|
||||||
|
.withDisplayName("Schema name")
|
||||||
|
.withType(Type.STRING)
|
||||||
|
.withWidth(Width.MEDIUM)
|
||||||
|
.withImportance(Importance.HIGH)
|
||||||
|
.withDescription("Name of the connection user to the database ");
|
||||||
|
|
||||||
public static final Field XSTREAM_SERVER_NAME = Field.create(DATABASE_CONFIG_PREFIX + "out.server.name")
|
public static final Field XSTREAM_SERVER_NAME = Field.create(DATABASE_CONFIG_PREFIX + "out.server.name")
|
||||||
.withDisplayName("XStream out server name")
|
.withDisplayName("XStream out server name")
|
||||||
.withType(Type.STRING)
|
.withType(Type.STRING)
|
||||||
@ -125,6 +139,55 @@ public class OracleConnectorConfig extends HistorizedRelationalDatabaseConnector
|
|||||||
public static final Field SERVER_NAME = RelationalDatabaseConnectorConfig.SERVER_NAME
|
public static final Field SERVER_NAME = RelationalDatabaseConnectorConfig.SERVER_NAME
|
||||||
.withValidation(CommonConnectorConfig::validateServerNameIsDifferentFromHistoryTopicName);
|
.withValidation(CommonConnectorConfig::validateServerNameIsDifferentFromHistoryTopicName);
|
||||||
|
|
||||||
|
public static final Field CONNECTOR_ADAPTER = Field.create(DATABASE_CONFIG_PREFIX + "connection.adapter")
|
||||||
|
.withDisplayName("Connector adapter")
|
||||||
|
.withEnum(ConnectorAdapter.class, ConnectorAdapter.XSTREAM)
|
||||||
|
.withWidth(Width.MEDIUM)
|
||||||
|
.withImportance(Importance.HIGH)
|
||||||
|
.withDescription("There are two adapters: XStream and LogMiner.");
|
||||||
|
|
||||||
|
public static final Field SNAPSHOT_SKIP_LOCKS = Field.create("snapshot.skip.locks")
|
||||||
|
.withDisplayName("Should schema be locked as we build the schema snapshot?")
|
||||||
|
.withType(Type.BOOLEAN)
|
||||||
|
.withWidth(Width.SHORT)
|
||||||
|
.withImportance(Importance.LOW)
|
||||||
|
.withDefault(false)
|
||||||
|
.withValidation(Field::isBoolean)
|
||||||
|
.withDescription(
|
||||||
|
"If true, locks all tables to be captured as we build the schema snapshot. This will prevent from any concurrent schema changes being applied to them.");
|
||||||
|
|
||||||
|
public static final Field LOG_MINING_STRATEGY = Field.create("log.mining.strategy")
|
||||||
|
.withDisplayName("Log Mining Strategy")
|
||||||
|
.withEnum(LogMiningStrategy.class, LogMiningStrategy.CATALOG_IN_REDO)
|
||||||
|
.withWidth(Width.MEDIUM)
|
||||||
|
.withImportance(Importance.HIGH)
|
||||||
|
.withDescription("There are strategies: Online catalog with faster mining but no captured DDL. Another - with data dictionary loaded into REDO LOG files");
|
||||||
|
|
||||||
|
// this option could be true up to Oracle 18c version. Starting from Oracle 19c this option cannot be true todo should we do it?
|
||||||
|
public static final Field CONTINUOUS_MINE = Field.create("log.mining.continuous.mine")
|
||||||
|
.withDisplayName("Should log mining session configured with CONTINUOUS_MINE setting?")
|
||||||
|
.withType(Type.BOOLEAN)
|
||||||
|
.withWidth(Width.SHORT)
|
||||||
|
.withImportance(Importance.LOW)
|
||||||
|
.withDefault(false)
|
||||||
|
.withValidation(Field::isBoolean)
|
||||||
|
.withDescription("If true, CONTINUOUS_MINE option will be added to the log mining session. This will manage log files switches seamlessly.");
|
||||||
|
|
||||||
|
public static final Field SNAPSHOT_ENHANCEMENT_TOKEN = Field.create("snapshot.enhance.predicate.scn")
|
||||||
|
.withDisplayName("A string to replace on snapshot predicate enhancement")
|
||||||
|
.withType(Type.STRING)
|
||||||
|
.withWidth(Width.MEDIUM)
|
||||||
|
.withImportance(Importance.HIGH)
|
||||||
|
.withDescription("A token to replace on snapshot predicate template");
|
||||||
|
|
||||||
|
public static final Field DRIVER_TYPE = Field.create("database.driver.type")
|
||||||
|
.withDisplayName("oci for xStream or thin for LogMiner")
|
||||||
|
.withType(Type.STRING)
|
||||||
|
.withWidth(Width.SHORT)
|
||||||
|
.withImportance(Importance.HIGH)
|
||||||
|
.withDefault("oci")
|
||||||
|
.withDescription("A token to use in connection factories");
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The set of {@link Field}s defined as part of this configuration.
|
* The set of {@link Field}s defined as part of this configuration.
|
||||||
*/
|
*/
|
||||||
@ -149,11 +212,18 @@ public class OracleConnectorConfig extends HistorizedRelationalDatabaseConnector
|
|||||||
CommonConnectorConfig.MAX_BATCH_SIZE,
|
CommonConnectorConfig.MAX_BATCH_SIZE,
|
||||||
CommonConnectorConfig.MAX_QUEUE_SIZE,
|
CommonConnectorConfig.MAX_QUEUE_SIZE,
|
||||||
CommonConnectorConfig.SNAPSHOT_DELAY_MS,
|
CommonConnectorConfig.SNAPSHOT_DELAY_MS,
|
||||||
|
CommonConnectorConfig.SNAPSHOT_FETCH_SIZE,
|
||||||
CommonConnectorConfig.PROVIDE_TRANSACTION_METADATA,
|
CommonConnectorConfig.PROVIDE_TRANSACTION_METADATA,
|
||||||
Heartbeat.HEARTBEAT_INTERVAL,
|
Heartbeat.HEARTBEAT_INTERVAL,
|
||||||
Heartbeat.HEARTBEAT_TOPICS_PREFIX,
|
Heartbeat.HEARTBEAT_TOPICS_PREFIX,
|
||||||
TABLENAME_CASE_INSENSITIVE,
|
TABLENAME_CASE_INSENSITIVE,
|
||||||
ORACLE_VERSION,
|
ORACLE_VERSION,
|
||||||
|
SNAPSHOT_SKIP_LOCKS,
|
||||||
|
SCHEMA_NAME,
|
||||||
|
CONNECTOR_ADAPTER,
|
||||||
|
LOG_MINING_STRATEGY,
|
||||||
|
SNAPSHOT_ENHANCEMENT_TOKEN,
|
||||||
|
DRIVER_TYPE,
|
||||||
CommonConnectorConfig.EVENT_PROCESSING_FAILURE_HANDLING_MODE);
|
CommonConnectorConfig.EVENT_PROCESSING_FAILURE_HANDLING_MODE);
|
||||||
|
|
||||||
private final String databaseName;
|
private final String databaseName;
|
||||||
@ -163,23 +233,46 @@ public class OracleConnectorConfig extends HistorizedRelationalDatabaseConnector
|
|||||||
|
|
||||||
private final boolean tablenameCaseInsensitive;
|
private final boolean tablenameCaseInsensitive;
|
||||||
private final OracleVersion oracleVersion;
|
private final OracleVersion oracleVersion;
|
||||||
|
private final String schemaName;
|
||||||
|
private final Tables.ColumnNameFilter columnFilter;
|
||||||
|
|
||||||
public OracleConnectorConfig(Configuration config) {
|
public OracleConnectorConfig(Configuration config) {
|
||||||
super(OracleConnector.class, config, config.getString(SERVER_NAME), new SystemTablesPredicate(), x -> x.schema() + "." + x.table(), true);
|
super(OracleConnector.class, config, config.getString(SERVER_NAME), new SystemTablesPredicate(), x -> x.schema() + "." + x.table(), true);
|
||||||
|
|
||||||
this.databaseName = config.getString(DATABASE_NAME);
|
this.databaseName = setUpperCase(config.getString(DATABASE_NAME));
|
||||||
this.pdbName = config.getString(PDB_NAME);
|
this.pdbName = setUpperCase(config.getString(PDB_NAME));
|
||||||
this.xoutServerName = config.getString(XSTREAM_SERVER_NAME);
|
this.xoutServerName = config.getString(XSTREAM_SERVER_NAME);
|
||||||
this.snapshotMode = SnapshotMode.parse(config.getString(SNAPSHOT_MODE));
|
this.snapshotMode = SnapshotMode.parse(config.getString(SNAPSHOT_MODE));
|
||||||
this.tablenameCaseInsensitive = config.getBoolean(TABLENAME_CASE_INSENSITIVE);
|
this.tablenameCaseInsensitive = config.getBoolean(TABLENAME_CASE_INSENSITIVE);
|
||||||
this.oracleVersion = OracleVersion.parse(config.getString(ORACLE_VERSION));
|
this.oracleVersion = OracleVersion.parse(config.getString(ORACLE_VERSION));
|
||||||
|
this.schemaName = setUpperCase(config.getString(SCHEMA_NAME));
|
||||||
|
String blacklistedColumns = setUpperCase(config.getString(RelationalDatabaseConnectorConfig.COLUMN_BLACKLIST));
|
||||||
|
this.columnFilter = getColumnNameFilter(blacklistedColumns);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String setUpperCase(String property) {
|
||||||
|
property = property == null ? null : property.toUpperCase();
|
||||||
|
return property;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Tables.ColumnNameFilter getColumnNameFilter(String excludedColumnPatterns) {
|
||||||
|
return new Tables.ColumnNameFilter() {
|
||||||
|
|
||||||
|
Predicate<ColumnId> delegate = Predicates.excludes(excludedColumnPatterns, ColumnId::toString);
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean matches(String catalogName, String schemaName, String tableName, String columnName) {
|
||||||
|
// ignore database name and schema name, we are supposed to capture from one database and one schema
|
||||||
|
return delegate.test(new ColumnId(new TableId(null, null, tableName), columnName));
|
||||||
|
}
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ConfigDef configDef() {
|
public static ConfigDef configDef() {
|
||||||
ConfigDef config = new ConfigDef();
|
ConfigDef config = new ConfigDef();
|
||||||
|
|
||||||
Field.group(config, "Oracle", HOSTNAME, PORT, USER, PASSWORD, SERVER_NAME, DATABASE_NAME, PDB_NAME,
|
Field.group(config, "Oracle", HOSTNAME, PORT, USER, PASSWORD, SERVER_NAME, DATABASE_NAME, PDB_NAME,
|
||||||
XSTREAM_SERVER_NAME, SNAPSHOT_MODE);
|
XSTREAM_SERVER_NAME, SNAPSHOT_MODE, CONNECTOR_ADAPTER, LOG_MINING_STRATEGY);
|
||||||
Field.group(config, "History Storage", KafkaDatabaseHistory.BOOTSTRAP_SERVERS,
|
Field.group(config, "History Storage", KafkaDatabaseHistory.BOOTSTRAP_SERVERS,
|
||||||
KafkaDatabaseHistory.TOPIC, KafkaDatabaseHistory.RECOVERY_POLL_ATTEMPTS,
|
KafkaDatabaseHistory.TOPIC, KafkaDatabaseHistory.RECOVERY_POLL_ATTEMPTS,
|
||||||
KafkaDatabaseHistory.RECOVERY_POLL_INTERVAL_MS, HistorizedRelationalDatabaseConnectorConfig.DATABASE_HISTORY);
|
KafkaDatabaseHistory.RECOVERY_POLL_INTERVAL_MS, HistorizedRelationalDatabaseConnectorConfig.DATABASE_HISTORY);
|
||||||
@ -188,12 +281,14 @@ public static ConfigDef configDef() {
|
|||||||
RelationalDatabaseConnectorConfig.TABLE_BLACKLIST,
|
RelationalDatabaseConnectorConfig.TABLE_BLACKLIST,
|
||||||
RelationalDatabaseConnectorConfig.TABLE_EXCLUDE_LIST,
|
RelationalDatabaseConnectorConfig.TABLE_EXCLUDE_LIST,
|
||||||
RelationalDatabaseConnectorConfig.MSG_KEY_COLUMNS,
|
RelationalDatabaseConnectorConfig.MSG_KEY_COLUMNS,
|
||||||
|
RelationalDatabaseConnectorConfig.COLUMN_BLACKLIST,
|
||||||
RelationalDatabaseConnectorConfig.TABLE_IGNORE_BUILTIN,
|
RelationalDatabaseConnectorConfig.TABLE_IGNORE_BUILTIN,
|
||||||
CommonConnectorConfig.PROVIDE_TRANSACTION_METADATA,
|
CommonConnectorConfig.PROVIDE_TRANSACTION_METADATA,
|
||||||
Heartbeat.HEARTBEAT_INTERVAL, Heartbeat.HEARTBEAT_TOPICS_PREFIX,
|
Heartbeat.HEARTBEAT_INTERVAL, Heartbeat.HEARTBEAT_TOPICS_PREFIX,
|
||||||
CommonConnectorConfig.EVENT_PROCESSING_FAILURE_HANDLING_MODE);
|
CommonConnectorConfig.EVENT_PROCESSING_FAILURE_HANDLING_MODE);
|
||||||
Field.group(config, "Connector", CommonConnectorConfig.POLL_INTERVAL_MS, CommonConnectorConfig.MAX_BATCH_SIZE,
|
Field.group(config, "Connector", CommonConnectorConfig.POLL_INTERVAL_MS, CommonConnectorConfig.MAX_BATCH_SIZE,
|
||||||
CommonConnectorConfig.MAX_QUEUE_SIZE, CommonConnectorConfig.SNAPSHOT_DELAY_MS);
|
CommonConnectorConfig.MAX_QUEUE_SIZE, CommonConnectorConfig.SNAPSHOT_DELAY_MS, CommonConnectorConfig.SNAPSHOT_FETCH_SIZE,
|
||||||
|
SNAPSHOT_SKIP_LOCKS, SNAPSHOT_ENHANCEMENT_TOKEN, DRIVER_TYPE);
|
||||||
|
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
@ -222,19 +317,36 @@ public OracleVersion getOracleVersion() {
|
|||||||
return oracleVersion;
|
return oracleVersion;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getSchemaName() {
|
||||||
|
return schemaName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Tables.ColumnNameFilter getColumnFilter() {
|
||||||
|
return columnFilter;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected HistoryRecordComparator getHistoryRecordComparator() {
|
protected HistoryRecordComparator getHistoryRecordComparator() {
|
||||||
return new HistoryRecordComparator() {
|
return new HistoryRecordComparator() {
|
||||||
@Override
|
@Override
|
||||||
protected boolean isPositionAtOrBefore(Document recorded, Document desired) {
|
protected boolean isPositionAtOrBefore(Document recorded, Document desired) {
|
||||||
final LcrPosition recordedPosition = LcrPosition.valueOf(recorded.getString(SourceInfo.LCR_POSITION_KEY));
|
Long recordedScn;
|
||||||
final LcrPosition desiredPosition = LcrPosition.valueOf(desired.getString(SourceInfo.LCR_POSITION_KEY));
|
Long desiredScn;
|
||||||
final Long recordedScn = recordedPosition != null ? recordedPosition.getScn() : recorded.getLong(SourceInfo.SCN_KEY);
|
if (getAdapter() == OracleConnectorConfig.ConnectorAdapter.XSTREAM) {
|
||||||
final Long desiredScn = desiredPosition != null ? desiredPosition.getScn() : desired.getLong(SourceInfo.SCN_KEY);
|
final LcrPosition recordedPosition = LcrPosition.valueOf(recorded.getString(SourceInfo.LCR_POSITION_KEY));
|
||||||
|
final LcrPosition desiredPosition = LcrPosition.valueOf(desired.getString(SourceInfo.LCR_POSITION_KEY));
|
||||||
|
recordedScn = recordedPosition != null ? recordedPosition.getScn() : recorded.getLong(SourceInfo.SCN_KEY);
|
||||||
|
desiredScn = desiredPosition != null ? desiredPosition.getScn() : desired.getLong(SourceInfo.SCN_KEY);
|
||||||
|
return (recordedPosition != null && desiredPosition != null)
|
||||||
|
? recordedPosition.compareTo(desiredPosition) < 1
|
||||||
|
: recordedScn.compareTo(desiredScn) < 1;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
recordedScn = recorded.getLong(SourceInfo.SCN_KEY);
|
||||||
|
desiredScn = desired.getLong(SourceInfo.SCN_KEY);
|
||||||
|
return recordedScn.compareTo(desiredScn) < 1;
|
||||||
|
}
|
||||||
|
|
||||||
return (recordedPosition != null && desiredPosition != null)
|
|
||||||
? recordedPosition.compareTo(desiredPosition) < 1
|
|
||||||
: recordedScn.compareTo(desiredScn) < 1;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@ -375,6 +487,117 @@ public static SnapshotMode parse(String value, String defaultValue) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public enum ConnectorAdapter implements EnumeratedValue {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is based on XStream API.
|
||||||
|
*/
|
||||||
|
XSTREAM("XStream"),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is based on LogMiner utility.
|
||||||
|
*/
|
||||||
|
LOG_MINER("LogMiner");
|
||||||
|
|
||||||
|
private final String value;
|
||||||
|
|
||||||
|
ConnectorAdapter(String value) {
|
||||||
|
this.value = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getValue() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine if the supplied value is one of the predefined options.
|
||||||
|
*
|
||||||
|
* @param value the configuration property value; may not be null
|
||||||
|
* @return the matching option, or null if no match is found
|
||||||
|
*/
|
||||||
|
public static ConnectorAdapter parse(String value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
value = value.trim();
|
||||||
|
for (ConnectorAdapter adapter : ConnectorAdapter.values()) {
|
||||||
|
if (adapter.getValue().equalsIgnoreCase(value)) {
|
||||||
|
return adapter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ConnectorAdapter parse(String value, String defaultValue) {
|
||||||
|
ConnectorAdapter mode = parse(value);
|
||||||
|
|
||||||
|
if (mode == null && defaultValue != null) {
|
||||||
|
mode = parse(defaultValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
return mode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum LogMiningStrategy implements EnumeratedValue {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This strategy uses Log Miner with data dictionary in online catalog.
|
||||||
|
* This option will not capture DDL , but acts fast on REDO LOG switch events
|
||||||
|
* This option does not use CONTINUOUS_MINE option
|
||||||
|
*/
|
||||||
|
ONLINE_CATALOG("online_catalog"),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This strategy uses Log Miner with data dictionary in REDO LOG files.
|
||||||
|
* This option will capture DDL, but will develop some lag on REDO LOG switch event and will eventually catch up
|
||||||
|
* This option does not use CONTINUOUS_MINE option
|
||||||
|
* This is default value
|
||||||
|
*/
|
||||||
|
CATALOG_IN_REDO("redo_log_catalog");
|
||||||
|
|
||||||
|
private final String value;
|
||||||
|
|
||||||
|
LogMiningStrategy(String value) {
|
||||||
|
this.value = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getValue() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine if the supplied value is one of the predefined options.
|
||||||
|
*
|
||||||
|
* @param value the configuration property value; may not be null
|
||||||
|
* @return the matching option, or null if no match is found
|
||||||
|
*/
|
||||||
|
public static LogMiningStrategy parse(String value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
value = value.trim();
|
||||||
|
for (LogMiningStrategy adapter : LogMiningStrategy.values()) {
|
||||||
|
if (adapter.getValue().equalsIgnoreCase(value)) {
|
||||||
|
return adapter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static LogMiningStrategy parse(String value, String defaultValue) {
|
||||||
|
LogMiningStrategy mode = parse(value);
|
||||||
|
|
||||||
|
if (mode == null && defaultValue != null) {
|
||||||
|
mode = parse(defaultValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
return mode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A {@link TableFilter} that excludes all Oracle system tables.
|
* A {@link TableFilter} that excludes all Oracle system tables.
|
||||||
*
|
*
|
||||||
@ -415,6 +638,38 @@ public String getContextName() {
|
|||||||
return Module.contextName();
|
return Module.contextName();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return true if lock to be obtained before building the schema
|
||||||
|
*/
|
||||||
|
public boolean skipSnapshotLock() {
|
||||||
|
return getConfig().getBoolean(SNAPSHOT_SKIP_LOCKS);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return connection adapter
|
||||||
|
*/
|
||||||
|
public ConnectorAdapter getAdapter() {
|
||||||
|
return ConnectorAdapter.parse(getConfig().getString(CONNECTOR_ADAPTER));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return Log Mining strategy
|
||||||
|
*/
|
||||||
|
public LogMiningStrategy getLogMiningStrategy() {
|
||||||
|
return LogMiningStrategy.parse(getConfig().getString(LOG_MINING_STRATEGY));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return String token to replace
|
||||||
|
*/
|
||||||
|
public String getTokenToReplaceInSnapshotPredicate() {
|
||||||
|
return getConfig().getString(SNAPSHOT_ENHANCEMENT_TOKEN);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isContinuousMining() {
|
||||||
|
return getConfig().getBoolean(CONTINUOUS_MINE);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validate the time.precision.mode configuration.
|
* Validate the time.precision.mode configuration.
|
||||||
*
|
*
|
||||||
|
@ -51,11 +51,15 @@ public ChangeEventSourceCoordinator start(Configuration config) {
|
|||||||
SchemaNameAdjuster schemaNameAdjuster = SchemaNameAdjuster.create(LOGGER);
|
SchemaNameAdjuster schemaNameAdjuster = SchemaNameAdjuster.create(LOGGER);
|
||||||
|
|
||||||
Configuration jdbcConfig = config.subset("database.", true);
|
Configuration jdbcConfig = config.subset("database.", true);
|
||||||
jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());
|
jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory(), () -> getClass().getClassLoader());
|
||||||
this.schema = new OracleDatabaseSchema(connectorConfig, schemaNameAdjuster, topicSelector, jdbcConnection);
|
this.schema = new OracleDatabaseSchema(connectorConfig, schemaNameAdjuster, topicSelector, jdbcConnection);
|
||||||
this.schema.initializeStorage();
|
this.schema.initializeStorage();
|
||||||
|
|
||||||
OffsetContext previousOffset = getPreviousOffset(new OracleOffsetContext.Loader(connectorConfig));
|
String adapterString = config.getString("connection.adapter");
|
||||||
|
adapterString = adapterString == null ? config.getString(OracleConnectorConfig.CONNECTOR_ADAPTER) : adapterString;
|
||||||
|
OracleConnectorConfig.ConnectorAdapter adapter = OracleConnectorConfig.ConnectorAdapter.parse(adapterString);
|
||||||
|
OffsetContext previousOffset = getPreviousOffset(new OracleOffsetContext.Loader(connectorConfig, adapter));
|
||||||
|
|
||||||
if (previousOffset != null) {
|
if (previousOffset != null) {
|
||||||
schema.recover(previousOffset);
|
schema.recover(previousOffset);
|
||||||
}
|
}
|
||||||
@ -91,7 +95,7 @@ public ChangeEventSourceCoordinator start(Configuration config) {
|
|||||||
errorHandler,
|
errorHandler,
|
||||||
OracleConnector.class,
|
OracleConnector.class,
|
||||||
connectorConfig,
|
connectorConfig,
|
||||||
new OracleChangeEventSourceFactory(connectorConfig, jdbcConnection, errorHandler, dispatcher, clock, schema),
|
new OracleChangeEventSourceFactory(connectorConfig, jdbcConnection, errorHandler, dispatcher, clock, schema, jdbcConfig, taskContext),
|
||||||
new DefaultChangeEventSourceMetricsFactory(),
|
new DefaultChangeEventSourceMetricsFactory(),
|
||||||
dispatcher,
|
dispatcher,
|
||||||
schema);
|
schema);
|
||||||
|
@ -13,6 +13,7 @@
|
|||||||
import io.debezium.relational.Table;
|
import io.debezium.relational.Table;
|
||||||
import io.debezium.relational.TableId;
|
import io.debezium.relational.TableId;
|
||||||
import io.debezium.relational.TableSchemaBuilder;
|
import io.debezium.relational.TableSchemaBuilder;
|
||||||
|
import io.debezium.relational.Tables;
|
||||||
import io.debezium.relational.ddl.DdlParser;
|
import io.debezium.relational.ddl.DdlParser;
|
||||||
import io.debezium.relational.history.TableChanges;
|
import io.debezium.relational.history.TableChanges;
|
||||||
import io.debezium.schema.SchemaChangeEvent;
|
import io.debezium.schema.SchemaChangeEvent;
|
||||||
@ -42,6 +43,10 @@ public OracleDatabaseSchema(OracleConnectorConfig connectorConfig, SchemaNameAdj
|
|||||||
connectorConfig.getKeyMapper());
|
connectorConfig.getKeyMapper());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Tables getTables() {
|
||||||
|
return tables();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected DdlParser getDdlParser() {
|
protected DdlParser getDdlParser() {
|
||||||
return new OracleDdlParser();
|
return new OracleDdlParser();
|
||||||
|
@ -14,6 +14,7 @@
|
|||||||
import org.apache.kafka.connect.data.Struct;
|
import org.apache.kafka.connect.data.Struct;
|
||||||
|
|
||||||
import io.debezium.connector.SnapshotRecord;
|
import io.debezium.connector.SnapshotRecord;
|
||||||
|
import io.debezium.connector.oracle.xstream.LcrPosition;
|
||||||
import io.debezium.pipeline.spi.OffsetContext;
|
import io.debezium.pipeline.spi.OffsetContext;
|
||||||
import io.debezium.pipeline.txmetadata.TransactionContext;
|
import io.debezium.pipeline.txmetadata.TransactionContext;
|
||||||
import io.debezium.relational.TableId;
|
import io.debezium.relational.TableId;
|
||||||
@ -35,6 +36,12 @@ public class OracleOffsetContext implements OffsetContext {
|
|||||||
*/
|
*/
|
||||||
private boolean snapshotCompleted;
|
private boolean snapshotCompleted;
|
||||||
|
|
||||||
|
public OracleOffsetContext(OracleConnectorConfig connectorConfig, long scn, Long commitScn, LcrPosition lcrPosition,
|
||||||
|
boolean snapshot, boolean snapshotCompleted, TransactionContext transactionContext) {
|
||||||
|
this(connectorConfig, scn, lcrPosition, snapshot, snapshotCompleted, transactionContext);
|
||||||
|
sourceInfo.setCommitScn(commitScn);
|
||||||
|
}
|
||||||
|
|
||||||
private OracleOffsetContext(OracleConnectorConfig connectorConfig, long scn, LcrPosition lcrPosition,
|
private OracleOffsetContext(OracleConnectorConfig connectorConfig, long scn, LcrPosition lcrPosition,
|
||||||
boolean snapshot, boolean snapshotCompleted, TransactionContext transactionContext) {
|
boolean snapshot, boolean snapshotCompleted, TransactionContext transactionContext) {
|
||||||
partition = Collections.singletonMap(SERVER_PARTITION_KEY, connectorConfig.getLogicalName());
|
partition = Collections.singletonMap(SERVER_PARTITION_KEY, connectorConfig.getLogicalName());
|
||||||
@ -126,6 +133,7 @@ public static Builder create() {
|
|||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
offset.put(SourceInfo.SCN_KEY, sourceInfo.getScn());
|
offset.put(SourceInfo.SCN_KEY, sourceInfo.getScn());
|
||||||
|
offset.put(SourceInfo.COMMIT_SCN_KEY, sourceInfo.getCommitScn());
|
||||||
}
|
}
|
||||||
return transactionContext.store(offset);
|
return transactionContext.store(offset);
|
||||||
}
|
}
|
||||||
@ -145,10 +153,18 @@ public void setScn(long scn) {
|
|||||||
sourceInfo.setScn(scn);
|
sourceInfo.setScn(scn);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setCommitScn(Long commitScn) {
|
||||||
|
sourceInfo.setCommitScn(commitScn);
|
||||||
|
}
|
||||||
|
|
||||||
public long getScn() {
|
public long getScn() {
|
||||||
return sourceInfo.getScn();
|
return sourceInfo.getScn();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Long getCommitScn() {
|
||||||
|
return sourceInfo.getCommitScn();
|
||||||
|
}
|
||||||
|
|
||||||
public void setLcrPosition(LcrPosition lcrPosition) {
|
public void setLcrPosition(LcrPosition lcrPosition) {
|
||||||
sourceInfo.setLcrPosition(lcrPosition);
|
sourceInfo.setLcrPosition(lcrPosition);
|
||||||
}
|
}
|
||||||
@ -223,9 +239,12 @@ public TransactionContext getTransactionContext() {
|
|||||||
public static class Loader implements OffsetContext.Loader {
|
public static class Loader implements OffsetContext.Loader {
|
||||||
|
|
||||||
private final OracleConnectorConfig connectorConfig;
|
private final OracleConnectorConfig connectorConfig;
|
||||||
|
private final OracleConnectorConfig.ConnectorAdapter adapter;
|
||||||
|
|
||||||
public Loader(OracleConnectorConfig connectorConfig) {
|
// todo resolve adapter from the config rather than passing it
|
||||||
|
public Loader(OracleConnectorConfig connectorConfig, OracleConnectorConfig.ConnectorAdapter adapter) {
|
||||||
this.connectorConfig = connectorConfig;
|
this.connectorConfig = connectorConfig;
|
||||||
|
this.adapter = adapter;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -235,12 +254,20 @@ public Loader(OracleConnectorConfig connectorConfig) {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public OffsetContext load(Map<String, ?> offset) {
|
public OffsetContext load(Map<String, ?> offset) {
|
||||||
LcrPosition lcrPosition = LcrPosition.valueOf((String) offset.get(SourceInfo.LCR_POSITION_KEY));
|
|
||||||
Long scn = lcrPosition != null ? lcrPosition.getScn() : (Long) offset.get(SourceInfo.SCN_KEY);
|
|
||||||
boolean snapshot = Boolean.TRUE.equals(offset.get(SourceInfo.SNAPSHOT_KEY));
|
boolean snapshot = Boolean.TRUE.equals(offset.get(SourceInfo.SNAPSHOT_KEY));
|
||||||
boolean snapshotCompleted = Boolean.TRUE.equals(offset.get(SNAPSHOT_COMPLETED_KEY));
|
boolean snapshotCompleted = Boolean.TRUE.equals(offset.get(SNAPSHOT_COMPLETED_KEY));
|
||||||
|
Long scn;
|
||||||
|
if (adapter == OracleConnectorConfig.ConnectorAdapter.LOG_MINER) {
|
||||||
|
scn = (Long) offset.get(SourceInfo.SCN_KEY);
|
||||||
|
Long commitScn = (Long) offset.get(SourceInfo.COMMIT_SCN_KEY);
|
||||||
|
return new OracleOffsetContext(connectorConfig, scn, commitScn, null, snapshot, snapshotCompleted, TransactionContext.load(offset));
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
LcrPosition lcrPosition = LcrPosition.valueOf((String) offset.get(SourceInfo.LCR_POSITION_KEY));
|
||||||
|
scn = lcrPosition != null ? lcrPosition.getScn() : (Long) offset.get(SourceInfo.SCN_KEY);
|
||||||
|
return new OracleOffsetContext(connectorConfig, scn, lcrPosition, snapshot, snapshotCompleted, TransactionContext.load(offset));
|
||||||
|
}
|
||||||
|
|
||||||
return new OracleOffsetContext(connectorConfig, scn, lcrPosition, snapshot, snapshotCompleted, TransactionContext.load(offset));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -10,6 +10,9 @@
|
|||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.sql.Savepoint;
|
import java.sql.Savepoint;
|
||||||
import java.sql.Statement;
|
import java.sql.Statement;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
@ -17,6 +20,7 @@
|
|||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.logminer.LogMinerHelper;
|
||||||
import io.debezium.pipeline.EventDispatcher;
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
import io.debezium.pipeline.source.spi.SnapshotProgressListener;
|
import io.debezium.pipeline.source.spi.SnapshotProgressListener;
|
||||||
import io.debezium.pipeline.source.spi.StreamingChangeEventSource;
|
import io.debezium.pipeline.source.spi.StreamingChangeEventSource;
|
||||||
@ -54,6 +58,7 @@ public OracleSnapshotChangeEventSource(OracleConnectorConfig connectorConfig, Or
|
|||||||
protected SnapshottingTask getSnapshottingTask(OffsetContext previousOffset) {
|
protected SnapshottingTask getSnapshottingTask(OffsetContext previousOffset) {
|
||||||
boolean snapshotSchema = true;
|
boolean snapshotSchema = true;
|
||||||
boolean snapshotData = true;
|
boolean snapshotData = true;
|
||||||
|
boolean skipSnapsotLock = false;
|
||||||
|
|
||||||
// found a previous offset and the earlier snapshot has completed
|
// found a previous offset and the earlier snapshot has completed
|
||||||
if (previousOffset != null && !previousOffset.isSnapshotRunning()) {
|
if (previousOffset != null && !previousOffset.isSnapshotRunning()) {
|
||||||
@ -62,9 +67,10 @@ protected SnapshottingTask getSnapshottingTask(OffsetContext previousOffset) {
|
|||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
snapshotData = connectorConfig.getSnapshotMode().includeData();
|
snapshotData = connectorConfig.getSnapshotMode().includeData();
|
||||||
|
skipSnapsotLock = connectorConfig.skipSnapshotLock();
|
||||||
}
|
}
|
||||||
|
|
||||||
return new SnapshottingTask(snapshotSchema, snapshotData);
|
return new SnapshottingTask(snapshotSchema, snapshotData, skipSnapsotLock);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -79,7 +85,9 @@ protected SnapshotContext prepare(ChangeEventSourceContext context) throws Excep
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Set<TableId> getAllTableIds(RelationalSnapshotContext ctx) throws Exception {
|
protected Set<TableId> getAllTableIds(RelationalSnapshotContext ctx) throws Exception {
|
||||||
return jdbcConnection.readTableNames(ctx.catalogName, null, null, new String[]{ "TABLE" });
|
return jdbcConnection.getAllTableIds(ctx.catalogName, connectorConfig.getSchemaName(), false);
|
||||||
|
// this very slow approach(commented out), it took 30 minutes on an instance with 600 tables
|
||||||
|
// return jdbcConnection.readTableNames(ctx.catalogName, null, null, new String[] {"TABLE"} );
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -126,6 +134,10 @@ protected void determineSnapshotOffset(RelationalSnapshotContext ctx) throws Exc
|
|||||||
}
|
}
|
||||||
|
|
||||||
private long getCurrentScn(SnapshotContext ctx) throws SQLException {
|
private long getCurrentScn(SnapshotContext ctx) throws SQLException {
|
||||||
|
if (connectorConfig.getAdapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
|
||||||
|
return LogMinerHelper.getCurrentScn(jdbcConnection.connection());
|
||||||
|
}
|
||||||
|
|
||||||
try (Statement statement = jdbcConnection.connection().createStatement();
|
try (Statement statement = jdbcConnection.connection().createStatement();
|
||||||
ResultSet rs = statement.executeQuery("select CURRENT_SCN from V$DATABASE")) {
|
ResultSet rs = statement.executeQuery("select CURRENT_SCN from V$DATABASE")) {
|
||||||
|
|
||||||
@ -195,6 +207,19 @@ protected void readTableStructure(ChangeEventSourceContext sourceContext, Relati
|
|||||||
throw new InterruptedException("Interrupted while reading structure of schema " + schema);
|
throw new InterruptedException("Interrupted while reading structure of schema " + schema);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// todo: DBZ-137 the new readSchemaForCapturedTables seems to cause failures.
|
||||||
|
// For now, reverted to the default readSchema implementation as the intended goal
|
||||||
|
// with the new implementation was to be faster, not change behavior.
|
||||||
|
// if (connectorConfig.getAdapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
|
||||||
|
// jdbcConnection.readSchemaForCapturedTables(
|
||||||
|
// snapshotContext.tables,
|
||||||
|
// snapshotContext.catalogName,
|
||||||
|
// schema,
|
||||||
|
// connectorConfig.getColumnFilter(),
|
||||||
|
// false,
|
||||||
|
// snapshotContext.capturedTables);
|
||||||
|
// }
|
||||||
|
// else {
|
||||||
jdbcConnection.readSchema(
|
jdbcConnection.readSchema(
|
||||||
snapshotContext.tables,
|
snapshotContext.tables,
|
||||||
snapshotContext.catalogName,
|
snapshotContext.catalogName,
|
||||||
@ -202,9 +227,22 @@ protected void readTableStructure(ChangeEventSourceContext sourceContext, Relati
|
|||||||
connectorConfig.getTableFilters().dataCollectionFilter(),
|
connectorConfig.getTableFilters().dataCollectionFilter(),
|
||||||
null,
|
null,
|
||||||
false);
|
false);
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String enhanceOverriddenSelect(RelationalSnapshotContext snapshotContext, String overriddenSelect, TableId tableId) {
|
||||||
|
String columnString = buildSelectColumns(connectorConfig.getConfig().getString(connectorConfig.COLUMN_BLACKLIST), snapshotContext.tables.forTable(tableId));
|
||||||
|
overriddenSelect = overriddenSelect.replaceFirst("\\*", columnString);
|
||||||
|
long snapshotOffset = (Long) snapshotContext.offset.getOffset().get("scn");
|
||||||
|
String token = connectorConfig.getTokenToReplaceInSnapshotPredicate();
|
||||||
|
if (token != null) {
|
||||||
|
return overriddenSelect.replaceAll(token, " AS OF SCN " + snapshotOffset);
|
||||||
|
}
|
||||||
|
return overriddenSelect;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected SchemaChangeEvent getCreateTableEvent(RelationalSnapshotContext snapshotContext, Table table) throws SQLException {
|
protected SchemaChangeEvent getCreateTableEvent(RelationalSnapshotContext snapshotContext, Table table) throws SQLException {
|
||||||
try (Statement statement = jdbcConnection.connection().createStatement();
|
try (Statement statement = jdbcConnection.connection().createStatement();
|
||||||
@ -231,11 +269,44 @@ protected SchemaChangeEvent getCreateTableEvent(RelationalSnapshotContext snapsh
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Optional<String> getSnapshotSelect(SnapshotContext snapshotContext, TableId tableId) {
|
protected Optional<String> getSnapshotSelect(RelationalSnapshotContext snapshotContext, TableId tableId) {
|
||||||
|
String columnString = buildSelectColumns(connectorConfig.getConfig().getString(connectorConfig.COLUMN_BLACKLIST), snapshotContext.tables.forTable(tableId));
|
||||||
|
|
||||||
long snapshotOffset = (Long) snapshotContext.offset.getOffset().get("scn");
|
long snapshotOffset = (Long) snapshotContext.offset.getOffset().get("scn");
|
||||||
return Optional.of("SELECT * FROM " + quote(tableId) + " AS OF SCN " + snapshotOffset);
|
return Optional.of("SELECT * FROM " + quote(tableId) + " AS OF SCN " + snapshotOffset);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to build "whitelisted" column list
|
||||||
|
* @param blackListColumnStr comma separated columns blacklist
|
||||||
|
* @param table the table
|
||||||
|
* @return column list for select
|
||||||
|
*/
|
||||||
|
public static String buildSelectColumns(String blackListColumnStr, Table table) {
|
||||||
|
String columnsToSelect = "*";
|
||||||
|
if (blackListColumnStr != null && blackListColumnStr.trim().length() > 0
|
||||||
|
&& blackListColumnStr.toUpperCase().contains(table.id().table())) {
|
||||||
|
String allTableColumns = table.retrieveColumnNames().stream()
|
||||||
|
.map(columnName -> {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
if (!columnName.contains(table.id().table())) {
|
||||||
|
sb.append(table.id().table()).append(".").append(columnName);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
sb.append(columnName);
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}).collect(Collectors.joining(","));
|
||||||
|
// todo this is an unnecessary code, fix unit test, then remove it
|
||||||
|
String catalog = table.id().catalog();
|
||||||
|
List<String> blackList = new ArrayList<>(Arrays.asList(blackListColumnStr.trim().toUpperCase().replaceAll(catalog + ".", "").split(",")));
|
||||||
|
List<String> allColumns = new ArrayList<>(Arrays.asList(allTableColumns.toUpperCase().split(",")));
|
||||||
|
allColumns.removeAll(blackList);
|
||||||
|
columnsToSelect = String.join(",", allColumns);
|
||||||
|
}
|
||||||
|
return columnsToSelect;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void complete(SnapshotContext snapshotContext) {
|
protected void complete(SnapshotContext snapshotContext) {
|
||||||
if (connectorConfig.getPdbName() != null) {
|
if (connectorConfig.getPdbName() != null) {
|
||||||
|
@ -23,6 +23,7 @@ public OracleSourceInfoStructMaker(String connector, String version, CommonConne
|
|||||||
.field(SourceInfo.TABLE_NAME_KEY, Schema.STRING_SCHEMA)
|
.field(SourceInfo.TABLE_NAME_KEY, Schema.STRING_SCHEMA)
|
||||||
.field(SourceInfo.TXID_KEY, Schema.OPTIONAL_STRING_SCHEMA)
|
.field(SourceInfo.TXID_KEY, Schema.OPTIONAL_STRING_SCHEMA)
|
||||||
.field(SourceInfo.SCN_KEY, Schema.OPTIONAL_INT64_SCHEMA)
|
.field(SourceInfo.SCN_KEY, Schema.OPTIONAL_INT64_SCHEMA)
|
||||||
|
.field(SourceInfo.COMMIT_SCN_KEY, Schema.OPTIONAL_INT64_SCHEMA)
|
||||||
.field(SourceInfo.LCR_POSITION_KEY, Schema.OPTIONAL_STRING_SCHEMA)
|
.field(SourceInfo.LCR_POSITION_KEY, Schema.OPTIONAL_STRING_SCHEMA)
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
@ -43,6 +44,9 @@ public Struct struct(SourceInfo sourceInfo) {
|
|||||||
if (sourceInfo.getLcrPosition() != null) {
|
if (sourceInfo.getLcrPosition() != null) {
|
||||||
ret.put(SourceInfo.LCR_POSITION_KEY, sourceInfo.getLcrPosition().toString());
|
ret.put(SourceInfo.LCR_POSITION_KEY, sourceInfo.getLcrPosition().toString());
|
||||||
}
|
}
|
||||||
|
if (sourceInfo.getCommitScn() != null) {
|
||||||
|
ret.put(SourceInfo.COMMIT_SCN_KEY, sourceInfo.getCommitScn());
|
||||||
|
}
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -18,6 +18,7 @@
|
|||||||
import org.apache.kafka.connect.data.Field;
|
import org.apache.kafka.connect.data.Field;
|
||||||
import org.apache.kafka.connect.data.SchemaBuilder;
|
import org.apache.kafka.connect.data.SchemaBuilder;
|
||||||
|
|
||||||
|
import io.debezium.config.CommonConnectorConfig.BinaryHandlingMode;
|
||||||
import io.debezium.data.SpecialValueDecimal;
|
import io.debezium.data.SpecialValueDecimal;
|
||||||
import io.debezium.data.VariableScaleDecimal;
|
import io.debezium.data.VariableScaleDecimal;
|
||||||
import io.debezium.jdbc.JdbcValueConverters;
|
import io.debezium.jdbc.JdbcValueConverters;
|
||||||
@ -25,6 +26,7 @@
|
|||||||
import io.debezium.jdbc.TemporalPrecisionMode;
|
import io.debezium.jdbc.TemporalPrecisionMode;
|
||||||
import io.debezium.relational.Column;
|
import io.debezium.relational.Column;
|
||||||
import io.debezium.relational.ValueConverter;
|
import io.debezium.relational.ValueConverter;
|
||||||
|
import io.debezium.time.Date;
|
||||||
import io.debezium.time.MicroDuration;
|
import io.debezium.time.MicroDuration;
|
||||||
import io.debezium.time.ZonedTimestamp;
|
import io.debezium.time.ZonedTimestamp;
|
||||||
import io.debezium.util.NumberConversions;
|
import io.debezium.util.NumberConversions;
|
||||||
@ -33,7 +35,9 @@
|
|||||||
import oracle.jdbc.OracleTypes;
|
import oracle.jdbc.OracleTypes;
|
||||||
import oracle.sql.BINARY_DOUBLE;
|
import oracle.sql.BINARY_DOUBLE;
|
||||||
import oracle.sql.BINARY_FLOAT;
|
import oracle.sql.BINARY_FLOAT;
|
||||||
|
import oracle.sql.BLOB;
|
||||||
import oracle.sql.CHAR;
|
import oracle.sql.CHAR;
|
||||||
|
import oracle.sql.CLOB;
|
||||||
import oracle.sql.DATE;
|
import oracle.sql.DATE;
|
||||||
import oracle.sql.INTERVALDS;
|
import oracle.sql.INTERVALDS;
|
||||||
import oracle.sql.INTERVALYM;
|
import oracle.sql.INTERVALYM;
|
||||||
@ -78,8 +82,13 @@ public SchemaBuilder schemaBuilder(Column column) {
|
|||||||
case OracleTypes.INTERVALYM:
|
case OracleTypes.INTERVALYM:
|
||||||
case OracleTypes.INTERVALDS:
|
case OracleTypes.INTERVALDS:
|
||||||
return MicroDuration.builder();
|
return MicroDuration.builder();
|
||||||
default:
|
case Types.STRUCT:
|
||||||
return super.schemaBuilder(column);
|
return SchemaBuilder.string();
|
||||||
|
default: {
|
||||||
|
SchemaBuilder builder = super.schemaBuilder(column);
|
||||||
|
logger.info("JdbcValueConverters returned '{}' for column '{}'", builder.getClass().getName(), column.name());
|
||||||
|
return builder;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -92,12 +101,17 @@ private SchemaBuilder getNumericSchema(Column column) {
|
|||||||
if (scale <= 0) {
|
if (scale <= 0) {
|
||||||
int width = column.length() - scale;
|
int width = column.length() - scale;
|
||||||
|
|
||||||
|
if (scale == 0 && column.length() == 1) {
|
||||||
|
return SchemaBuilder.bool();
|
||||||
|
}
|
||||||
if (width < 3) {
|
if (width < 3) {
|
||||||
return SchemaBuilder.int8();
|
return SchemaBuilder.int8();
|
||||||
}
|
}
|
||||||
else if (width < 5) {
|
else if (width < 5) {
|
||||||
return SchemaBuilder.int16();
|
return SchemaBuilder.int16();
|
||||||
}
|
}
|
||||||
|
// todo: DBZ-137 this was changed and caused issues with datatype tests, reverted for now.
|
||||||
|
// else if (width < 10 || (width == 10 && scale == 0)) {
|
||||||
else if (width < 10) {
|
else if (width < 10) {
|
||||||
return SchemaBuilder.int32();
|
return SchemaBuilder.int32();
|
||||||
}
|
}
|
||||||
@ -128,7 +142,11 @@ public ValueConverter converter(Column column, Field fieldDefn) {
|
|||||||
case Types.VARCHAR:
|
case Types.VARCHAR:
|
||||||
case Types.NCHAR:
|
case Types.NCHAR:
|
||||||
case Types.NVARCHAR:
|
case Types.NVARCHAR:
|
||||||
|
case Types.STRUCT:
|
||||||
|
case Types.CLOB:
|
||||||
return data -> convertString(column, fieldDefn, data);
|
return data -> convertString(column, fieldDefn, data);
|
||||||
|
case Types.BLOB:
|
||||||
|
return data -> convertBinary(column, fieldDefn, data, binaryMode);
|
||||||
case OracleTypes.BINARY_FLOAT:
|
case OracleTypes.BINARY_FLOAT:
|
||||||
return data -> convertFloat(column, fieldDefn, data);
|
return data -> convertFloat(column, fieldDefn, data);
|
||||||
case OracleTypes.BINARY_DOUBLE:
|
case OracleTypes.BINARY_DOUBLE:
|
||||||
@ -136,6 +154,8 @@ public ValueConverter converter(Column column, Field fieldDefn) {
|
|||||||
case Types.NUMERIC:
|
case Types.NUMERIC:
|
||||||
return getNumericConverter(column, fieldDefn);
|
return getNumericConverter(column, fieldDefn);
|
||||||
case Types.FLOAT:
|
case Types.FLOAT:
|
||||||
|
// todo: DBZ-137 is there a reason why floats need to be converted to doubles rather than var-scales?
|
||||||
|
// return data -> convertDouble(column, fieldDefn, data);
|
||||||
return data -> convertVariableScale(column, fieldDefn, data);
|
return data -> convertVariableScale(column, fieldDefn, data);
|
||||||
case OracleTypes.TIMESTAMPTZ:
|
case OracleTypes.TIMESTAMPTZ:
|
||||||
case OracleTypes.TIMESTAMPLTZ:
|
case OracleTypes.TIMESTAMPLTZ:
|
||||||
@ -149,11 +169,23 @@ public ValueConverter converter(Column column, Field fieldDefn) {
|
|||||||
return super.converter(column, fieldDefn);
|
return super.converter(column, fieldDefn);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Object getFloatConverter(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof BigDecimal) {
|
||||||
|
return ((BigDecimal) data).floatValue();
|
||||||
|
}
|
||||||
|
return convertVariableScale(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
private ValueConverter getNumericConverter(Column column, Field fieldDefn) {
|
private ValueConverter getNumericConverter(Column column, Field fieldDefn) {
|
||||||
if (column.scale().isPresent()) {
|
if (column.scale().isPresent()) {
|
||||||
Integer scale = column.scale().get();
|
Integer scale = column.scale().get();
|
||||||
|
|
||||||
if (scale <= 0) {
|
if (scale <= 0) {
|
||||||
|
// Boolean represtented as Number(1,0)
|
||||||
|
if (scale == 0 && column.length() == 1) {
|
||||||
|
return data -> convertBoolean(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
int width = column.length() - scale;
|
int width = column.length() - scale;
|
||||||
|
|
||||||
if (width < 3) {
|
if (width < 3) {
|
||||||
@ -162,6 +194,8 @@ private ValueConverter getNumericConverter(Column column, Field fieldDefn) {
|
|||||||
else if (width < 5) {
|
else if (width < 5) {
|
||||||
return data -> convertNumericAsSmallInt(column, fieldDefn, data);
|
return data -> convertNumericAsSmallInt(column, fieldDefn, data);
|
||||||
}
|
}
|
||||||
|
// todo: DBZ-137 this was changed and caused issues with datatype tests, reverted for now.
|
||||||
|
// else if (width < 10 || (width == 10 && scale == 0)) {
|
||||||
else if (width < 10) {
|
else if (width < 10) {
|
||||||
return data -> convertNumericAsInteger(column, fieldDefn, data);
|
return data -> convertNumericAsInteger(column, fieldDefn, data);
|
||||||
}
|
}
|
||||||
@ -183,10 +217,21 @@ protected Object convertString(Column column, Field fieldDefn, Object data) {
|
|||||||
if (data instanceof CHAR) {
|
if (data instanceof CHAR) {
|
||||||
return ((CHAR) data).stringValue();
|
return ((CHAR) data).stringValue();
|
||||||
}
|
}
|
||||||
|
if (data instanceof CLOB) {
|
||||||
|
return ((CLOB) data).toString();
|
||||||
|
}
|
||||||
|
|
||||||
return super.convertString(column, fieldDefn, data);
|
return super.convertString(column, fieldDefn, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertBinary(Column column, Field fieldDefn, Object data, BinaryHandlingMode mode) {
|
||||||
|
if (data instanceof BLOB) {
|
||||||
|
return ((BLOB) data).getBytes();
|
||||||
|
}
|
||||||
|
return super.convertBinary(column, fieldDefn, data, mode);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Object convertInteger(Column column, Field fieldDefn, Object data) {
|
protected Object convertInteger(Column column, Field fieldDefn, Object data) {
|
||||||
if (data instanceof NUMBER) {
|
if (data instanceof NUMBER) {
|
||||||
@ -312,6 +357,24 @@ protected Object convertNumericAsBigInteger(Column column, Field fieldDefn, Obje
|
|||||||
return super.convertBigInt(column, fieldDefn, data);
|
return super.convertBigInt(column, fieldDefn, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Converts a value object for an expected JDBC type of {@link Types#BOOLEAN}.
|
||||||
|
*
|
||||||
|
* @param column the column definition describing the {@code data} value; never null
|
||||||
|
* @param fieldDefn the field definition; never null
|
||||||
|
* @param data the data object to be converted into a {@link Date Kafka Connect date} type; never null
|
||||||
|
* @return the converted value, or null if the conversion could not be made and the column allows nulls
|
||||||
|
* @throws IllegalArgumentException if the value could not be converted but the column does not allow nulls
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
protected Object convertBoolean(Column column, Field fieldDefn, Object data) {
|
||||||
|
|
||||||
|
if (data instanceof BigDecimal) {
|
||||||
|
return ((BigDecimal) data).byteValue() == 0 ? Boolean.FALSE : Boolean.TRUE;
|
||||||
|
}
|
||||||
|
return super.convertBoolean(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Object convertTinyInt(Column column, Field fieldDefn, Object data) {
|
protected Object convertTinyInt(Column column, Field fieldDefn, Object data) {
|
||||||
return convertValue(column, fieldDefn, data, BYTE_FALSE, (r) -> {
|
return convertValue(column, fieldDefn, data, BYTE_FALSE, (r) -> {
|
||||||
@ -379,6 +442,9 @@ else if (data instanceof TIMESTAMPLTZ) {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Object convertTimestampToEpochMicros(Column column, Field fieldDefn, Object data) {
|
protected Object convertTimestampToEpochMicros(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof Long) {
|
||||||
|
return data;
|
||||||
|
}
|
||||||
return super.convertTimestampToEpochMicros(column, fieldDefn, fromOracleTimeClasses(column, data));
|
return super.convertTimestampToEpochMicros(column, fieldDefn, fromOracleTimeClasses(column, data));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -9,6 +9,7 @@
|
|||||||
|
|
||||||
import io.debezium.annotation.NotThreadSafe;
|
import io.debezium.annotation.NotThreadSafe;
|
||||||
import io.debezium.connector.common.BaseSourceInfo;
|
import io.debezium.connector.common.BaseSourceInfo;
|
||||||
|
import io.debezium.connector.oracle.xstream.LcrPosition;
|
||||||
import io.debezium.relational.TableId;
|
import io.debezium.relational.TableId;
|
||||||
|
|
||||||
@NotThreadSafe
|
@NotThreadSafe
|
||||||
@ -16,10 +17,12 @@ public class SourceInfo extends BaseSourceInfo {
|
|||||||
|
|
||||||
public static final String TXID_KEY = "txId";
|
public static final String TXID_KEY = "txId";
|
||||||
public static final String SCN_KEY = "scn";
|
public static final String SCN_KEY = "scn";
|
||||||
|
public static final String COMMIT_SCN_KEY = "commit_scn";
|
||||||
public static final String LCR_POSITION_KEY = "lcr_position";
|
public static final String LCR_POSITION_KEY = "lcr_position";
|
||||||
public static final String SNAPSHOT_KEY = "snapshot";
|
public static final String SNAPSHOT_KEY = "snapshot";
|
||||||
|
|
||||||
private long scn;
|
private long scn;
|
||||||
|
private Long commitScn;
|
||||||
private LcrPosition lcrPosition;
|
private LcrPosition lcrPosition;
|
||||||
private String transactionId;
|
private String transactionId;
|
||||||
private Instant sourceTime;
|
private Instant sourceTime;
|
||||||
@ -33,10 +36,18 @@ public long getScn() {
|
|||||||
return scn;
|
return scn;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Long getCommitScn() {
|
||||||
|
return commitScn;
|
||||||
|
}
|
||||||
|
|
||||||
public void setScn(long scn) {
|
public void setScn(long scn) {
|
||||||
this.scn = scn;
|
this.scn = scn;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setCommitScn(Long commitScn) {
|
||||||
|
this.commitScn = commitScn;
|
||||||
|
}
|
||||||
|
|
||||||
public LcrPosition getLcrPosition() {
|
public LcrPosition getLcrPosition() {
|
||||||
return lcrPosition;
|
return lcrPosition;
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,95 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.antlr;
|
||||||
|
|
||||||
|
import org.antlr.v4.runtime.CharStream;
|
||||||
|
import org.antlr.v4.runtime.CommonTokenStream;
|
||||||
|
import org.antlr.v4.runtime.tree.ParseTree;
|
||||||
|
|
||||||
|
import io.debezium.antlr.AntlrDdlParser;
|
||||||
|
import io.debezium.antlr.AntlrDdlParserListener;
|
||||||
|
import io.debezium.antlr.DataTypeResolver;
|
||||||
|
import io.debezium.connector.oracle.antlr.listener.OracleDmlParserListener;
|
||||||
|
import io.debezium.connector.oracle.logminer.OracleChangeRecordValueConverter;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlLexer;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||||
|
import io.debezium.relational.SystemVariables;
|
||||||
|
import io.debezium.relational.Tables;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is the main Oracle Antlr DML parser
|
||||||
|
*/
|
||||||
|
public class OracleDmlParser extends AntlrDdlParser<PlSqlLexer, PlSqlParser> {
|
||||||
|
|
||||||
|
private LogMinerDmlEntry dmlEntry;
|
||||||
|
protected String catalogName;
|
||||||
|
protected String schemaName;
|
||||||
|
private OracleChangeRecordValueConverter converter;
|
||||||
|
|
||||||
|
public OracleDmlParser(boolean throwErrorsFromTreeWalk, final String catalogName, final String schemaName, OracleChangeRecordValueConverter converter) {
|
||||||
|
super(throwErrorsFromTreeWalk);
|
||||||
|
this.catalogName = catalogName;
|
||||||
|
this.schemaName = schemaName;
|
||||||
|
this.converter = converter;
|
||||||
|
}
|
||||||
|
|
||||||
|
public LogMinerDmlEntry getDmlEntry() {
|
||||||
|
return dmlEntry;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDmlEntry(LogMinerDmlEntry dml) {
|
||||||
|
this.dmlEntry = dml;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void parse(String dmlContent, Tables databaseTables) {
|
||||||
|
if (!dmlContent.endsWith(";")) {
|
||||||
|
dmlContent = dmlContent + ";";
|
||||||
|
}
|
||||||
|
// DML content is case sensitive
|
||||||
|
super.parse(dmlContent, databaseTables);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ParseTree parseTree(PlSqlParser parser) {
|
||||||
|
return parser.unit_statement();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AntlrDdlParserListener createParseTreeWalkerListener() {
|
||||||
|
return new OracleDmlParserListener(catalogName, schemaName, this);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected PlSqlLexer createNewLexerInstance(CharStream charStreams) {
|
||||||
|
return new PlSqlLexer(charStreams);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected PlSqlParser createNewParserInstance(CommonTokenStream commonTokenStream) {
|
||||||
|
return new PlSqlParser(commonTokenStream);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected boolean isGrammarInUpperCase() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected DataTypeResolver initializeDataTypeResolver() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected SystemVariables createNewSystemVariablesInstance() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OracleChangeRecordValueConverter getConverters() {
|
||||||
|
return converter;
|
||||||
|
}
|
||||||
|
}
|
@ -6,6 +6,8 @@
|
|||||||
package io.debezium.connector.oracle.antlr.listener;
|
package io.debezium.connector.oracle.antlr.listener;
|
||||||
|
|
||||||
import static io.debezium.antlr.AntlrDdlParser.getText;
|
import static io.debezium.antlr.AntlrDdlParser.getText;
|
||||||
|
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getColumnName;
|
||||||
|
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getTableName;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -0,0 +1,68 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.antlr.listener;
|
||||||
|
|
||||||
|
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getTableName;
|
||||||
|
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||||
|
import io.debezium.connector.oracle.logminer.OracleChangeRecordValueConverter;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueImpl;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParserBaseListener;
|
||||||
|
import io.debezium.relational.Column;
|
||||||
|
import io.debezium.relational.Table;
|
||||||
|
import io.debezium.text.ParsingException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class contains common methods for DML parser listeners
|
||||||
|
*/
|
||||||
|
abstract class BaseDmlParserListener<T> extends PlSqlParserBaseListener {
|
||||||
|
|
||||||
|
protected String catalogName;
|
||||||
|
protected String schemaName;
|
||||||
|
protected Table table;
|
||||||
|
final OracleChangeRecordValueConverter converter;
|
||||||
|
String alias;
|
||||||
|
|
||||||
|
protected OracleDmlParser parser;
|
||||||
|
|
||||||
|
Map<T, LogMinerColumnValueWrapper> newColumnValues = new LinkedHashMap<>();
|
||||||
|
Map<T, LogMinerColumnValueWrapper> oldColumnValues = new LinkedHashMap<>();
|
||||||
|
|
||||||
|
BaseDmlParserListener(String catalogName, String schemaName, OracleDmlParser parser) {
|
||||||
|
this.parser = parser;
|
||||||
|
this.catalogName = catalogName;
|
||||||
|
this.schemaName = schemaName;
|
||||||
|
this.converter = parser.getConverters();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Defines the key of the Map of LogMinerColumnValueWrapper. It could be String or Integer
|
||||||
|
abstract protected T getKey(Column column, int index);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method prepares all column value placeholders, based on the table metadata
|
||||||
|
* @param ctx DML table expression context
|
||||||
|
*/
|
||||||
|
void init(PlSqlParser.Dml_table_expression_clauseContext ctx) {
|
||||||
|
String tableName = getTableName(ctx.tableview_name());
|
||||||
|
table = parser.databaseTables().forTable(catalogName, schemaName, tableName);
|
||||||
|
if (table == null) {
|
||||||
|
throw new ParsingException(null, "Trying to parse a table, which does not exist.");
|
||||||
|
}
|
||||||
|
for (int i = 0; i < table.columns().size(); i++) {
|
||||||
|
Column column = table.columns().get(i);
|
||||||
|
int type = column.jdbcType();
|
||||||
|
T key = getKey(column, i);
|
||||||
|
String name = ParserUtils.stripeQuotes(column.name().toUpperCase());
|
||||||
|
newColumnValues.put(key, new LogMinerColumnValueWrapper(new LogMinerColumnValueImpl(name, type)));
|
||||||
|
oldColumnValues.put(key, new LogMinerColumnValueWrapper(new LogMinerColumnValueImpl(name, type)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,73 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.antlr.listener;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||||
|
import io.debezium.relational.Column;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class parses recursively logical expression tree for DELETE and UPDATE statements
|
||||||
|
*/
|
||||||
|
abstract class BaseDmlStringParserListener extends BaseDmlParserListener<String> {
|
||||||
|
|
||||||
|
boolean isUpdate;
|
||||||
|
|
||||||
|
BaseDmlStringParserListener(String catalogName, String schemaName, OracleDmlParser parser) {
|
||||||
|
super(catalogName, schemaName, parser);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void enterTable_alias(PlSqlParser.Table_aliasContext ctx) {
|
||||||
|
alias = ctx.getText().toUpperCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Logical expressions are trees and (column name, value) pairs are nested in this tree.
|
||||||
|
* This methods extracts those pairs and store them in List<LogMinerColumnValue> oldValues
|
||||||
|
* This method is used by VALUES parsers of update and delete statements.
|
||||||
|
* @param logicalExpression expression tree
|
||||||
|
*/
|
||||||
|
void parseRecursively(PlSqlParser.Logical_expressionContext logicalExpression) {
|
||||||
|
|
||||||
|
int count = logicalExpression.logical_expression().size();
|
||||||
|
if (count == 0) {
|
||||||
|
|
||||||
|
String nullValue = logicalExpression.getStop().getText();
|
||||||
|
|
||||||
|
String expression = logicalExpression.getText();
|
||||||
|
String columnName = "";
|
||||||
|
String value = "";
|
||||||
|
if (expression.contains("=")) {
|
||||||
|
columnName = expression.substring(0, expression.indexOf("=")).toUpperCase();
|
||||||
|
value = expression.substring(expression.indexOf("=") + 1);
|
||||||
|
}
|
||||||
|
if ("null".equalsIgnoreCase(nullValue)) {
|
||||||
|
columnName = expression.substring(0, expression.toUpperCase().indexOf("ISNULL")).toUpperCase();
|
||||||
|
value = nullValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
columnName = ParserUtils.stripeAlias(columnName, alias);
|
||||||
|
columnName = ParserUtils.stripeQuotes(columnName);
|
||||||
|
|
||||||
|
Column column = table.columnWithName(columnName);
|
||||||
|
Object stripedValue = ParserUtils.removeApostrophes(value);
|
||||||
|
|
||||||
|
LogMinerColumnValueWrapper logMinerColumnValueWrapper = oldColumnValues.get(columnName);
|
||||||
|
if (logMinerColumnValueWrapper != null) { // todo this used to happen for ROWID pseudo column. Test if this is not a problem after NO_ROWID_IN_STMT option
|
||||||
|
Object valueObject = ParserUtils.convertValueToSchemaType(column, stripedValue, converter);
|
||||||
|
logMinerColumnValueWrapper.setProcessed(true);
|
||||||
|
logMinerColumnValueWrapper.getColumnValue().setColumnData(valueObject);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
parseRecursively(logicalExpression.logical_expression(i));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -57,11 +57,19 @@ public void enterPrimary_key_clause(PlSqlParser.Primary_key_clauseContext ctx) {
|
|||||||
|
|
||||||
// todo use dataTypeResolver instead
|
// todo use dataTypeResolver instead
|
||||||
private void resolveColumnDataType(PlSqlParser.Column_definitionContext ctx) {
|
private void resolveColumnDataType(PlSqlParser.Column_definitionContext ctx) {
|
||||||
PlSqlParser.Precision_partContext precisionPart = ctx.datatype().precision_part();
|
|
||||||
|
|
||||||
columnEditor.name(getColumnName(ctx.column_name()));
|
columnEditor.name(getColumnName(ctx.column_name()));
|
||||||
|
|
||||||
if (ctx.datatype().native_datatype_element() != null) {
|
PlSqlParser.Precision_partContext precisionPart = null;
|
||||||
|
if (ctx.datatype() != null) {
|
||||||
|
precisionPart = ctx.datatype().precision_part();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx.datatype() == null) {
|
||||||
|
if (ctx.type_name() != null && "\"MDSYS\".\"SDO_GEOMETRY\"".equalsIgnoreCase(ctx.type_name().getText())) {
|
||||||
|
columnEditor.jdbcType(Types.STRUCT).type("MDSYS.SDO_GEOMETRY");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (ctx.datatype().native_datatype_element() != null) {
|
||||||
if (ctx.datatype().native_datatype_element().INT() != null
|
if (ctx.datatype().native_datatype_element().INT() != null
|
||||||
|| ctx.datatype().native_datatype_element().INTEGER() != null
|
|| ctx.datatype().native_datatype_element().INTEGER() != null
|
||||||
|| ctx.datatype().native_datatype_element().SMALLINT() != null
|
|| ctx.datatype().native_datatype_element().SMALLINT() != null
|
||||||
|
@ -5,6 +5,9 @@
|
|||||||
*/
|
*/
|
||||||
package io.debezium.connector.oracle.antlr.listener;
|
package io.debezium.connector.oracle.antlr.listener;
|
||||||
|
|
||||||
|
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getColumnName;
|
||||||
|
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getTableName;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@ -49,19 +52,21 @@ public void enterCreate_table(PlSqlParser.Create_tableContext ctx) {
|
|||||||
public void exitCreate_table(PlSqlParser.Create_tableContext ctx) {
|
public void exitCreate_table(PlSqlParser.Create_tableContext ctx) {
|
||||||
Table table = getTable();
|
Table table = getTable();
|
||||||
assert table != null;
|
assert table != null;
|
||||||
|
|
||||||
parser.runIfNotNull(() -> {
|
parser.runIfNotNull(() -> {
|
||||||
listeners.remove(columnDefinitionParserListener);
|
listeners.remove(columnDefinitionParserListener);
|
||||||
columnDefinitionParserListener = null;
|
columnDefinitionParserListener = null;
|
||||||
parser.databaseTables().overwriteTable(table);
|
parser.databaseTables().overwriteTable(table);
|
||||||
// parser.signalCreateTable(tableEditor.tableId(), ctx); todo ?
|
// parser.signalCreateTable(tableEditor.tableId(), ctx); todo ?
|
||||||
}, tableEditor, table);
|
}, tableEditor, table);
|
||||||
|
|
||||||
super.exitCreate_table(ctx);
|
super.exitCreate_table(ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void enterColumn_definition(PlSqlParser.Column_definitionContext ctx) {
|
public void enterColumn_definition(PlSqlParser.Column_definitionContext ctx) {
|
||||||
parser.runIfNotNull(() -> {
|
parser.runIfNotNull(() -> {
|
||||||
String columnName = getColumnName(ctx.column_name());
|
String columnName = ParserUtils.stripeQuotes(getColumnName(ctx.column_name()));
|
||||||
ColumnEditor columnEditor = Column.editor().name(columnName);
|
ColumnEditor columnEditor = Column.editor().name(columnName);
|
||||||
if (columnDefinitionParserListener == null) {
|
if (columnDefinitionParserListener == null) {
|
||||||
columnDefinitionParserListener = new ColumnDefinitionParserListener(tableEditor, columnEditor, parser.dataTypeResolver());
|
columnDefinitionParserListener = new ColumnDefinitionParserListener(tableEditor, columnEditor, parser.dataTypeResolver());
|
||||||
@ -87,7 +92,7 @@ public void exitColumn_definition(PlSqlParser.Column_definitionContext ctx) {
|
|||||||
public void exitOut_of_line_constraint(PlSqlParser.Out_of_line_constraintContext ctx) {
|
public void exitOut_of_line_constraint(PlSqlParser.Out_of_line_constraintContext ctx) {
|
||||||
if (ctx.PRIMARY() != null) {
|
if (ctx.PRIMARY() != null) {
|
||||||
List<String> pkColumnNames = ctx.column_name().stream()
|
List<String> pkColumnNames = ctx.column_name().stream()
|
||||||
.map(this::getColumnName)
|
.map(ParserUtils::getColumnName)
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
tableEditor.setPrimaryKeyNames(pkColumnNames);
|
tableEditor.setPrimaryKeyNames(pkColumnNames);
|
||||||
|
@ -0,0 +1,67 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.antlr.listener;
|
||||||
|
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntryImpl;
|
||||||
|
import io.debezium.data.Envelope;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||||
|
import io.debezium.relational.Column;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class parses delete statements.
|
||||||
|
* LogMiner instruments all the values in WHERE cause regardless of original statement.
|
||||||
|
* In other words if the original statement is
|
||||||
|
* delete from debezium where col1 = 2 and if there are 2 records to delete,
|
||||||
|
* LogMiner will contain following two statements:
|
||||||
|
*
|
||||||
|
* delete from "DEBEZIUM" where "ID" = 6 and "COL1" = 2 and "COL2" = 'text' and "COL3" = 'text' and "COL4" IS NULL and "COL5" IS NULL and "COL6" IS NULL and "COL7" IS NULL and "COL8" IS NULL
|
||||||
|
* delete from "DEBEZIUM" where "ID" = 7 and "COL1" = 2 and "COL2" = 'text' and "COL3" = 'text' and "COL4" IS NULL and "COL5" IS NULL and "COL6" IS NULL and "COL7" IS NULL and "COL8" IS NULL
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class DeleteParserListener extends BaseDmlStringParserListener {
|
||||||
|
|
||||||
|
DeleteParserListener(final String catalogName, final String schemaName, final OracleDmlParser parser) {
|
||||||
|
super(catalogName, schemaName, parser);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String getKey(Column column, int index) {
|
||||||
|
return column.name();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void enterDelete_statement(PlSqlParser.Delete_statementContext ctx) {
|
||||||
|
init(ctx.general_table_ref().dml_table_expression_clause());
|
||||||
|
newColumnValues.clear();
|
||||||
|
PlSqlParser.Table_aliasContext tableAlias = ctx.general_table_ref().table_alias();
|
||||||
|
alias = tableAlias == null ? "" : tableAlias.getText().toUpperCase();
|
||||||
|
PlSqlParser.Where_clauseContext where = ctx.where_clause();
|
||||||
|
if (where != null) {
|
||||||
|
parseRecursively(ctx.where_clause().expression().logical_expression());
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
oldColumnValues.clear();
|
||||||
|
}
|
||||||
|
super.enterDelete_statement(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void exitDelete_statement(PlSqlParser.Delete_statementContext ctx) {
|
||||||
|
List<LogMinerColumnValue> actualOldValues = oldColumnValues.values()
|
||||||
|
.stream().map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||||
|
LogMinerDmlEntry newRecord = new LogMinerDmlEntryImpl(Envelope.Operation.DELETE, Collections.emptyList(), actualOldValues);
|
||||||
|
parser.setDmlEntry(newRecord);
|
||||||
|
super.exitDelete_statement(ctx);
|
||||||
|
}
|
||||||
|
}
|
@ -5,14 +5,18 @@
|
|||||||
*/
|
*/
|
||||||
package io.debezium.connector.oracle.antlr.listener;
|
package io.debezium.connector.oracle.antlr.listener;
|
||||||
|
|
||||||
|
import static io.debezium.connector.oracle.antlr.listener.ParserUtils.getTableName;
|
||||||
|
|
||||||
import io.debezium.connector.oracle.antlr.OracleDdlParser;
|
import io.debezium.connector.oracle.antlr.OracleDdlParser;
|
||||||
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParserBaseListener;
|
||||||
import io.debezium.relational.TableId;
|
import io.debezium.relational.TableId;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class is parsing Oracle drop table statements.
|
* This class is parsing Oracle drop table statements.
|
||||||
*/
|
*/
|
||||||
public class DropTableParserListener extends BaseParserListener {
|
public class DropTableParserListener extends PlSqlParserBaseListener {
|
||||||
|
|
||||||
private String catalogName;
|
private String catalogName;
|
||||||
private String schemaName;
|
private String schemaName;
|
||||||
private OracleDdlParser parser;
|
private OracleDdlParser parser;
|
||||||
|
@ -0,0 +1,87 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.antlr.listener;
|
||||||
|
|
||||||
|
import static io.debezium.antlr.AntlrDdlParser.getText;
|
||||||
|
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntryImpl;
|
||||||
|
import io.debezium.data.Envelope;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||||
|
import io.debezium.relational.Column;
|
||||||
|
import io.debezium.text.ParsingException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class parses Oracle INSERT statements.
|
||||||
|
* if the original tested query was: insert into DEBEZIUM (id,col3) values (2, 'some text')
|
||||||
|
*
|
||||||
|
* LogMiner will supply:
|
||||||
|
*
|
||||||
|
* insert into "DEBEZIUM"("ID","COL1","COL2","COL3","COL4","COL5","COL6","COL7","COL8","COL9","COL10")
|
||||||
|
* values (2,NULL,'debezium','some text',NULL,NULL,NULL,NULL,NULL,EMPTY_BLOB(),EMPTY_CLOB())
|
||||||
|
* update "DEBEZIUM" set "COL9" = NULL, "COL10" = NULL where "ID" = 2 and "COL1" IS NULL and "COL2" = 'debezium'
|
||||||
|
* and "COL3" = 'some text' and "COL4" IS NULL and "COL5" IS NULL and "COL6" IS NULL
|
||||||
|
* and "COL7" IS NULL and "COL8" IS NULL
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class InsertParserListener extends BaseDmlParserListener<Integer> {
|
||||||
|
|
||||||
|
InsertParserListener(String catalogName, String schemaName, OracleDmlParser parser) {
|
||||||
|
super(catalogName, schemaName, parser);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Integer getKey(Column column, int index) {
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void enterInsert_statement(PlSqlParser.Insert_statementContext ctx) {
|
||||||
|
init(ctx.single_table_insert().insert_into_clause().general_table_ref().dml_table_expression_clause());
|
||||||
|
oldColumnValues.clear();
|
||||||
|
super.enterInsert_statement(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void enterValues_clause(PlSqlParser.Values_clauseContext ctx) {
|
||||||
|
if (table == null) {
|
||||||
|
throw new ParsingException(null, "Trying to parse a statement for a table which does not exist. " +
|
||||||
|
"Statement: " + getText(ctx));
|
||||||
|
}
|
||||||
|
|
||||||
|
List<PlSqlParser.ExpressionContext> values = ctx.expressions().expression();
|
||||||
|
for (int i = 0; i < values.size(); i++) {
|
||||||
|
PlSqlParser.ExpressionContext value = values.get(i);
|
||||||
|
LogMinerColumnValueWrapper columnObject = newColumnValues.get(i);
|
||||||
|
|
||||||
|
String columnName = columnObject.getColumnValue().getColumnName();
|
||||||
|
Column column = table.columnWithName(columnName);
|
||||||
|
|
||||||
|
String valueText = value.logical_expression().getText();
|
||||||
|
valueText = ParserUtils.removeApostrophes(valueText);
|
||||||
|
Object valueObject = ParserUtils.convertValueToSchemaType(column, valueText, converter);
|
||||||
|
|
||||||
|
columnObject.getColumnValue().setColumnData(valueObject);
|
||||||
|
}
|
||||||
|
super.enterValues_clause(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void exitSingle_table_insert(PlSqlParser.Single_table_insertContext ctx) {
|
||||||
|
List<LogMinerColumnValue> actualNewValues = newColumnValues.values()
|
||||||
|
.stream().map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||||
|
LogMinerDmlEntry newRecord = new LogMinerDmlEntryImpl(Envelope.Operation.CREATE, actualNewValues, Collections.emptyList());
|
||||||
|
parser.setDmlEntry(newRecord);
|
||||||
|
super.exitSingle_table_insert(ctx);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,54 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.antlr.listener;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.concurrent.CopyOnWriteArrayList;
|
||||||
|
|
||||||
|
import org.antlr.v4.runtime.ParserRuleContext;
|
||||||
|
import org.antlr.v4.runtime.tree.ParseTreeListener;
|
||||||
|
|
||||||
|
import io.debezium.antlr.AntlrDdlParserListener;
|
||||||
|
import io.debezium.antlr.ProxyParseTreeListenerUtil;
|
||||||
|
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParserBaseListener;
|
||||||
|
import io.debezium.text.ParsingException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class is Oracle main DML parser listener class.
|
||||||
|
* It instantiates supported listeners, walks listeners through every parsing rule and collects parsing exceptions.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class OracleDmlParserListener extends PlSqlParserBaseListener implements AntlrDdlParserListener {
|
||||||
|
|
||||||
|
private final List<ParseTreeListener> listeners = new CopyOnWriteArrayList<>();
|
||||||
|
private final Collection<ParsingException> errors = new ArrayList<>();
|
||||||
|
|
||||||
|
public OracleDmlParserListener(final String catalogName, final String schemaName,
|
||||||
|
final OracleDmlParser parser) {
|
||||||
|
listeners.add(new InsertParserListener(catalogName, schemaName, parser));
|
||||||
|
listeners.add(new UpdateParserListener(catalogName, schemaName, parser));
|
||||||
|
listeners.add(new DeleteParserListener(catalogName, schemaName, parser));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Collection<ParsingException> getErrors() {
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void enterEveryRule(ParserRuleContext ctx) {
|
||||||
|
ProxyParseTreeListenerUtil.delegateEnterRule(ctx, listeners, errors);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void exitEveryRule(ParserRuleContext ctx) {
|
||||||
|
ProxyParseTreeListenerUtil.delegateExitRule(ctx, listeners, errors);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,149 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.antlr.listener;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.kafka.connect.data.Field;
|
||||||
|
import org.apache.kafka.connect.data.Schema;
|
||||||
|
import org.apache.kafka.connect.data.SchemaBuilder;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.logminer.OracleChangeRecordValueConverter;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||||
|
import io.debezium.relational.Column;
|
||||||
|
import io.debezium.relational.Table;
|
||||||
|
import io.debezium.relational.ValueConverter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class contains a few methods for parser listeners
|
||||||
|
*/
|
||||||
|
public class ParserUtils {
|
||||||
|
|
||||||
|
private ParserUtils() {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method returns pure column name without quotes
|
||||||
|
* @param ctx column name context
|
||||||
|
* @return column name
|
||||||
|
*/
|
||||||
|
public static String getColumnName(final PlSqlParser.Column_nameContext ctx) {
|
||||||
|
return stripeQuotes(ctx.identifier().id_expression().getText());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* stripes double quotes that surrounds a string
|
||||||
|
* @param text text
|
||||||
|
* @return text without surrounding double quotes
|
||||||
|
*/
|
||||||
|
public static String stripeQuotes(String text) {
|
||||||
|
if (text != null && text.indexOf("\"") == 0 && text.lastIndexOf("\"") == text.length() - 1) {
|
||||||
|
return text.substring(1, text.length() - 1);
|
||||||
|
}
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* this method stripes table alias and dot give string
|
||||||
|
* @param text string with possible alias
|
||||||
|
* @param alias table alias
|
||||||
|
* @return striped string
|
||||||
|
*/
|
||||||
|
public static String stripeAlias(String text, String alias) {
|
||||||
|
int index = text.indexOf(alias + ".");
|
||||||
|
if (index >= 0) {
|
||||||
|
return text.substring(alias.length() + 1);
|
||||||
|
}
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize new column values with old column values.
|
||||||
|
* It does not override new values which were processed already in where clause parsing
|
||||||
|
* @param newColumnValues values to set or insert
|
||||||
|
* @param oldColumnValues values in WHERE clause
|
||||||
|
* @param table Debezium Table object
|
||||||
|
*/
|
||||||
|
public static void cloneOldToNewColumnValues(Map<String, LogMinerColumnValueWrapper> newColumnValues, Map<String, LogMinerColumnValueWrapper> oldColumnValues,
|
||||||
|
Table table) {
|
||||||
|
for (Column column : table.columns()) {
|
||||||
|
final LogMinerColumnValueWrapper newColumnValue = newColumnValues.get(column.name());
|
||||||
|
if (!newColumnValue.isProcessed()) {
|
||||||
|
final LogMinerColumnValueWrapper oldColumnValue = oldColumnValues.get(column.name());
|
||||||
|
newColumnValue.setProcessed(true);
|
||||||
|
newColumnValue.getColumnValue().setColumnData(oldColumnValue.getColumnValue().getColumnData());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This converts the given value to the appropriate object. The conversion is based on the column definition
|
||||||
|
*
|
||||||
|
* @param column column Object
|
||||||
|
* @param value value object
|
||||||
|
* @param converters given converter
|
||||||
|
* @return object as the result of this conversion. It could be null if converter cannot build the schema
|
||||||
|
* or if converter or value are null
|
||||||
|
*/
|
||||||
|
public static Object convertValueToSchemaType(Column column, Object value, OracleChangeRecordValueConverter converters) {
|
||||||
|
if (converters != null && value != null) {
|
||||||
|
final SchemaBuilder schemaBuilder = converters.schemaBuilder(column);
|
||||||
|
if (schemaBuilder == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
final Schema schema = schemaBuilder.build();
|
||||||
|
final Field field = new Field(column.name(), 1, schema);
|
||||||
|
final ValueConverter valueConverter = converters.converter(column, field);
|
||||||
|
|
||||||
|
return valueConverter.convert(value);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* In some cases values of the parsed expression are enclosed in apostrophes.
|
||||||
|
* Even null values are surrounded by single apostrophes. This method removes them.
|
||||||
|
*
|
||||||
|
* @param text supplied value which might be enclosed by apostrophes.
|
||||||
|
* @return clean String or null in case if test = "null" or = "NULL"
|
||||||
|
*/
|
||||||
|
public static String removeApostrophes(String text) {
|
||||||
|
if (text != null && text.indexOf("'") == 0 && text.lastIndexOf("'") == text.length() - 1) {
|
||||||
|
return text.substring(1, text.length() - 1);
|
||||||
|
}
|
||||||
|
if ("null".equalsIgnoreCase(text)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* this is to handle cases when a record contains escape character(s)
|
||||||
|
* @param text before parsing we replaced it with double escape, now revert it back
|
||||||
|
* @return string with double slashes
|
||||||
|
*/
|
||||||
|
public static String replaceDoubleBackSlashes(String text) {
|
||||||
|
if (text != null && text.contains("\\\\")) {
|
||||||
|
return text.replaceAll("\\\\\\\\", "\\\\");
|
||||||
|
}
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Obtains the table name
|
||||||
|
* @param tableview_name table view context
|
||||||
|
* @return table name
|
||||||
|
*/
|
||||||
|
static String getTableName(final PlSqlParser.Tableview_nameContext tableview_name) {
|
||||||
|
if (tableview_name.id_expression() != null) {
|
||||||
|
return stripeQuotes(tableview_name.id_expression().getText());
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
return stripeQuotes(tableview_name.identifier().id_expression().getText());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,104 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.antlr.listener;
|
||||||
|
|
||||||
|
import static io.debezium.antlr.AntlrDdlParser.getText;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntryImpl;
|
||||||
|
import io.debezium.data.Envelope;
|
||||||
|
import io.debezium.ddl.parser.oracle.generated.PlSqlParser;
|
||||||
|
import io.debezium.relational.Column;
|
||||||
|
import io.debezium.text.ParsingException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class parses UPDATE statements.
|
||||||
|
* For the original query:
|
||||||
|
* update debezium set test = '7' where test1 = '6' (let's assume we have 3 records with such value)
|
||||||
|
*
|
||||||
|
* logMiner with supply:
|
||||||
|
*
|
||||||
|
* update "debezium" set "TEST" = '7' where "DUMMY" = '1' and "TEST" = '2' and "TEST1" = '6' and "TEST2" = '1'
|
||||||
|
* update "debezium" set "TEST" = '7' where "DUMMY" = '2' and "TEST" = '2' and "TEST1" = '6' and "TEST2" = '1'
|
||||||
|
* update "debezium" set "TEST" = '7' where "DUMMY" = '3' and "TEST" = '2' and "TEST1" = '6' and "TEST2" = '1'
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class UpdateParserListener extends BaseDmlStringParserListener {
|
||||||
|
|
||||||
|
UpdateParserListener(String catalogName, String schemaName, OracleDmlParser parser) {
|
||||||
|
super(catalogName, schemaName, parser);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String getKey(Column column, int index) {
|
||||||
|
return column.name();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void enterUpdate_statement(PlSqlParser.Update_statementContext ctx) {
|
||||||
|
init(ctx.general_table_ref().dml_table_expression_clause());
|
||||||
|
isUpdate = true;
|
||||||
|
super.enterUpdate_statement(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* this method could be invoked by delete, insert or update statements, but we should act on update only
|
||||||
|
* @param ctx where clause context
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void enterWhere_clause(PlSqlParser.Where_clauseContext ctx) {
|
||||||
|
if (isUpdate) {
|
||||||
|
parseRecursively(ctx.expression().logical_expression());
|
||||||
|
ParserUtils.cloneOldToNewColumnValues(newColumnValues, oldColumnValues, table);
|
||||||
|
}
|
||||||
|
isUpdate = false;
|
||||||
|
super.enterWhere_clause(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void enterColumn_based_update_set_clause(PlSqlParser.Column_based_update_set_clauseContext ctx) {
|
||||||
|
if (table == null) {
|
||||||
|
throw new ParsingException(null, "Trying to parse a statement for a table which does not exist. " +
|
||||||
|
"Statement: " + getText(ctx));
|
||||||
|
}
|
||||||
|
String columnName = ctx.column_name().getText().toUpperCase();
|
||||||
|
String stripedName = ParserUtils.stripeAlias(columnName, alias);
|
||||||
|
stripedName = ParserUtils.stripeQuotes(stripedName);
|
||||||
|
String value = ctx.getText().substring(columnName.length() + 1);
|
||||||
|
String nullValue = ctx.expression().getStop().getText();
|
||||||
|
if ("null".equalsIgnoreCase(nullValue)) {
|
||||||
|
value = nullValue;
|
||||||
|
}
|
||||||
|
Object stripedValue = ParserUtils.removeApostrophes(value);
|
||||||
|
|
||||||
|
Column column = table.columnWithName(stripedName);
|
||||||
|
Object valueObject = ParserUtils.convertValueToSchemaType(column, stripedValue, converter);
|
||||||
|
|
||||||
|
LogMinerColumnValueWrapper logMinerColumnValueWrapper = newColumnValues.get(stripedName);
|
||||||
|
logMinerColumnValueWrapper.setProcessed(true);
|
||||||
|
logMinerColumnValueWrapper.getColumnValue().setColumnData(valueObject);
|
||||||
|
|
||||||
|
super.enterColumn_based_update_set_clause(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void exitUpdate_statement(PlSqlParser.Update_statementContext ctx) {
|
||||||
|
List<LogMinerColumnValue> actualNewValues = newColumnValues.values().stream()
|
||||||
|
.filter(LogMinerColumnValueWrapper::isProcessed).map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||||
|
List<LogMinerColumnValue> actualOldValues = oldColumnValues.values().stream()
|
||||||
|
.filter(LogMinerColumnValueWrapper::isProcessed).map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||||
|
LogMinerDmlEntry newRecord = new LogMinerDmlEntryImpl(Envelope.Operation.UPDATE, actualNewValues, actualOldValues);
|
||||||
|
parser.setDmlEntry(newRecord);
|
||||||
|
super.exitUpdate_statement(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,287 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.jsqlparser;
|
||||||
|
|
||||||
|
import java.io.StringReader;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.antlr.listener.ParserUtils;
|
||||||
|
import io.debezium.connector.oracle.logminer.OracleChangeRecordValueConverter;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueImpl;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntryImpl;
|
||||||
|
import io.debezium.data.Envelope;
|
||||||
|
import io.debezium.relational.Column;
|
||||||
|
import io.debezium.relational.Table;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
import io.debezium.relational.Tables;
|
||||||
|
import io.debezium.text.ParsingException;
|
||||||
|
|
||||||
|
import net.sf.jsqlparser.JSQLParserException;
|
||||||
|
import net.sf.jsqlparser.expression.Alias;
|
||||||
|
import net.sf.jsqlparser.expression.Expression;
|
||||||
|
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
|
||||||
|
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
|
||||||
|
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
|
||||||
|
import net.sf.jsqlparser.expression.operators.relational.IsNullExpression;
|
||||||
|
import net.sf.jsqlparser.expression.operators.relational.ItemsList;
|
||||||
|
import net.sf.jsqlparser.expression.operators.relational.ItemsListVisitorAdapter;
|
||||||
|
import net.sf.jsqlparser.parser.CCJSqlParserManager;
|
||||||
|
import net.sf.jsqlparser.statement.Statement;
|
||||||
|
import net.sf.jsqlparser.statement.delete.Delete;
|
||||||
|
import net.sf.jsqlparser.statement.insert.Insert;
|
||||||
|
import net.sf.jsqlparser.statement.update.Update;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class does parsing of simple DML: insert, update, delete.
|
||||||
|
* Log Miner supplies very simple syntax , that this parser should be sufficient to parse those.
|
||||||
|
* It does no support joins, merge, sub-selects and other complicated cases, which should be OK for Log Miner case
|
||||||
|
*/
|
||||||
|
public class SimpleDmlParser {
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(SimpleDmlParser.class);
|
||||||
|
protected String catalogName;
|
||||||
|
protected String schemaName;
|
||||||
|
protected Table table;
|
||||||
|
private final OracleChangeRecordValueConverter converter;
|
||||||
|
private String aliasName;
|
||||||
|
private Map<String, LogMinerColumnValueWrapper> newColumnValues = new LinkedHashMap<>();
|
||||||
|
private Map<String, LogMinerColumnValueWrapper> oldColumnValues = new LinkedHashMap<>();
|
||||||
|
private CCJSqlParserManager pm;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor
|
||||||
|
* @param catalogName database name
|
||||||
|
* @param schemaName user name
|
||||||
|
* @param converter value converter
|
||||||
|
*/
|
||||||
|
public SimpleDmlParser(String catalogName, String schemaName, OracleChangeRecordValueConverter converter) {
|
||||||
|
this.catalogName = catalogName;
|
||||||
|
this.schemaName = schemaName;
|
||||||
|
this.converter = converter;
|
||||||
|
pm = new CCJSqlParserManager();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This parses a DML
|
||||||
|
* @param dmlContent DML
|
||||||
|
* @param tables debezium Tables
|
||||||
|
* @return parsed value holder class
|
||||||
|
*/
|
||||||
|
public LogMinerDmlEntry parse(String dmlContent, Tables tables, String txId) {
|
||||||
|
try {
|
||||||
|
|
||||||
|
// If a table contains Spatial data type, DML input generates two entries in REDO LOG.
|
||||||
|
// First with actual statement and second with NULL. It is not relevant at this point
|
||||||
|
if (dmlContent == null) {
|
||||||
|
LOGGER.debug("Cannot parse NULL , transaction: {}", txId);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
// todo investigate: happens on CTAS
|
||||||
|
if (dmlContent.endsWith(";null;")) {
|
||||||
|
dmlContent = dmlContent.substring(0, dmlContent.lastIndexOf(";null;"));
|
||||||
|
}
|
||||||
|
if (!dmlContent.endsWith(";")) {
|
||||||
|
dmlContent = dmlContent + ";";
|
||||||
|
}
|
||||||
|
// this is to handle cases when a record contains escape character(s). This parser throws.
|
||||||
|
dmlContent = dmlContent.replaceAll("\\\\", "\\\\\\\\");
|
||||||
|
dmlContent = dmlContent.replaceAll("= Unsupported Type", "= null"); // todo address spatial data types
|
||||||
|
|
||||||
|
newColumnValues.clear();
|
||||||
|
oldColumnValues.clear();
|
||||||
|
|
||||||
|
Statement st = pm.parse(new StringReader(dmlContent));
|
||||||
|
if (st instanceof Update) {
|
||||||
|
parseUpdate(tables, (Update) st);
|
||||||
|
List<LogMinerColumnValue> actualNewValues = newColumnValues.values().stream()
|
||||||
|
.filter(LogMinerColumnValueWrapper::isProcessed).map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||||
|
List<LogMinerColumnValue> actualOldValues = oldColumnValues.values().stream()
|
||||||
|
.filter(LogMinerColumnValueWrapper::isProcessed).map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||||
|
return new LogMinerDmlEntryImpl(Envelope.Operation.UPDATE, actualNewValues, actualOldValues);
|
||||||
|
|
||||||
|
}
|
||||||
|
else if (st instanceof Insert) {
|
||||||
|
parseInsert(tables, (Insert) st);
|
||||||
|
List<LogMinerColumnValue> actualNewValues = newColumnValues.values()
|
||||||
|
.stream().map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||||
|
return new LogMinerDmlEntryImpl(Envelope.Operation.CREATE, actualNewValues, Collections.emptyList());
|
||||||
|
|
||||||
|
}
|
||||||
|
else if (st instanceof Delete) {
|
||||||
|
parseDelete(tables, (Delete) st);
|
||||||
|
List<LogMinerColumnValue> actualOldValues = oldColumnValues.values()
|
||||||
|
.stream().map(LogMinerColumnValueWrapper::getColumnValue).collect(Collectors.toList());
|
||||||
|
return new LogMinerDmlEntryImpl(Envelope.Operation.DELETE, Collections.emptyList(), actualOldValues);
|
||||||
|
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
LOGGER.error("Operation {} is not supported yet", st);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
catch (Throwable e) {
|
||||||
|
LOGGER.error("Cannot parse statement : {}, transaction: {}, due to the {}", dmlContent, txId, e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initColumns(Tables tables, String tableName) {
|
||||||
|
table = tables.forTable(catalogName, schemaName, tableName);
|
||||||
|
if (table == null) {
|
||||||
|
TableId id = new TableId(catalogName, schemaName, tableName);
|
||||||
|
throw new ParsingException(null, "Trying to parse a table '" + id + "', which does not exist.");
|
||||||
|
}
|
||||||
|
for (int i = 0; i < table.columns().size(); i++) {
|
||||||
|
Column column = table.columns().get(i);
|
||||||
|
int type = column.jdbcType();
|
||||||
|
String key = column.name();
|
||||||
|
String name = ParserUtils.stripeQuotes(column.name().toUpperCase());
|
||||||
|
newColumnValues.put(key, new LogMinerColumnValueWrapper(new LogMinerColumnValueImpl(name, type)));
|
||||||
|
oldColumnValues.put(key, new LogMinerColumnValueWrapper(new LogMinerColumnValueImpl(name, type)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// this parses simple statement with only one table
|
||||||
|
private void parseUpdate(Tables tables, Update st) throws JSQLParserException {
|
||||||
|
int tableCount = st.getTables().size();
|
||||||
|
if (tableCount > 1 || tableCount == 0) {
|
||||||
|
throw new JSQLParserException("DML includes " + tableCount + " tables");
|
||||||
|
}
|
||||||
|
net.sf.jsqlparser.schema.Table parseTable = st.getTables().get(0);
|
||||||
|
initColumns(tables, ParserUtils.stripeQuotes(parseTable.getName()));
|
||||||
|
|
||||||
|
List<net.sf.jsqlparser.schema.Column> columns = st.getColumns();
|
||||||
|
Alias alias = parseTable.getAlias();
|
||||||
|
aliasName = alias == null ? "" : alias.getName().trim();
|
||||||
|
|
||||||
|
List<Expression> expressions = st.getExpressions(); // new values
|
||||||
|
setNewValues(expressions, columns);
|
||||||
|
Expression where = st.getWhere(); // old values
|
||||||
|
if (where != null) {
|
||||||
|
parseWhereClause(where);
|
||||||
|
ParserUtils.cloneOldToNewColumnValues(newColumnValues, oldColumnValues, table);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
oldColumnValues.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void parseInsert(Tables tables, Insert st) {
|
||||||
|
initColumns(tables, ParserUtils.stripeQuotes(st.getTable().getName()));
|
||||||
|
Alias alias = st.getTable().getAlias();
|
||||||
|
aliasName = alias == null ? "" : alias.getName().trim();
|
||||||
|
|
||||||
|
List<net.sf.jsqlparser.schema.Column> columns = st.getColumns();
|
||||||
|
ItemsList values = st.getItemsList();
|
||||||
|
values.accept(new ItemsListVisitorAdapter() {
|
||||||
|
@Override
|
||||||
|
public void visit(ExpressionList expressionList) {
|
||||||
|
super.visit(expressionList);
|
||||||
|
List<Expression> expressions = expressionList.getExpressions();
|
||||||
|
setNewValues(expressions, columns);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
oldColumnValues.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void parseDelete(Tables tables, Delete st) {
|
||||||
|
initColumns(tables, ParserUtils.stripeQuotes(st.getTable().getName()));
|
||||||
|
Alias alias = st.getTable().getAlias();
|
||||||
|
aliasName = alias == null ? "" : alias.getName().trim();
|
||||||
|
|
||||||
|
newColumnValues.clear();
|
||||||
|
|
||||||
|
Expression where = st.getWhere();
|
||||||
|
if (where != null) {
|
||||||
|
parseWhereClause(where);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
oldColumnValues.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void setNewValues(List<Expression> expressions, List<net.sf.jsqlparser.schema.Column> columns) {
|
||||||
|
if (expressions.size() != columns.size()) {
|
||||||
|
throw new RuntimeException("DML has " + expressions.size() + " column values, but Table object has " + columns.size() + " columns");
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < columns.size(); i++) {
|
||||||
|
String columnName = ParserUtils.stripeQuotes(columns.get(i).getColumnName().toUpperCase());
|
||||||
|
String value = ParserUtils.stripeQuotes(expressions.get(i).toString());
|
||||||
|
Object stripedValue = ParserUtils.removeApostrophes(value);
|
||||||
|
Column column = table.columnWithName(columnName);
|
||||||
|
if (column == null) {
|
||||||
|
LOGGER.trace("blacklisted column: {}", columnName);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Object valueObject = ParserUtils.convertValueToSchemaType(column, stripedValue, converter);
|
||||||
|
|
||||||
|
LogMinerColumnValueWrapper logMinerColumnValueWrapper = newColumnValues.get(columnName);
|
||||||
|
if (logMinerColumnValueWrapper != null) {
|
||||||
|
logMinerColumnValueWrapper.setProcessed(true);
|
||||||
|
logMinerColumnValueWrapper.getColumnValue().setColumnData(valueObject);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void parseWhereClause(Expression logicalExpression) {
|
||||||
|
|
||||||
|
logicalExpression.accept(new ExpressionVisitorAdapter() {
|
||||||
|
@Override
|
||||||
|
public void visit(EqualsTo expr) {
|
||||||
|
super.visit(expr);
|
||||||
|
String columnName = expr.getLeftExpression().toString();
|
||||||
|
columnName = ParserUtils.stripeAlias(columnName, aliasName);
|
||||||
|
String value = expr.getRightExpression().toString();
|
||||||
|
columnName = ParserUtils.stripeQuotes(columnName);
|
||||||
|
|
||||||
|
Column column = table.columnWithName(columnName);
|
||||||
|
if (column == null) {
|
||||||
|
LOGGER.trace("blacklisted column in where clause: {}", columnName);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
value = ParserUtils.removeApostrophes(value);
|
||||||
|
|
||||||
|
LogMinerColumnValueWrapper logMinerColumnValueWrapper = oldColumnValues.get(columnName.toUpperCase());
|
||||||
|
if (logMinerColumnValueWrapper != null) {
|
||||||
|
Object valueObject = ParserUtils.convertValueToSchemaType(column, value, converter);
|
||||||
|
logMinerColumnValueWrapper.setProcessed(true);
|
||||||
|
logMinerColumnValueWrapper.getColumnValue().setColumnData(valueObject);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void visit(IsNullExpression expr) {
|
||||||
|
super.visit(expr);
|
||||||
|
String columnName = expr.getLeftExpression().toString();
|
||||||
|
columnName = ParserUtils.stripeAlias(columnName, aliasName);
|
||||||
|
columnName = ParserUtils.stripeQuotes(columnName);
|
||||||
|
Column column = table.columnWithName(columnName);
|
||||||
|
if (column == null) {
|
||||||
|
LOGGER.trace("blacklisted column in where clause: {}", columnName);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LogMinerColumnValueWrapper logMinerColumnValueWrapper = oldColumnValues.get(columnName.toUpperCase());
|
||||||
|
if (logMinerColumnValueWrapper != null) {
|
||||||
|
logMinerColumnValueWrapper.setProcessed(true);
|
||||||
|
logMinerColumnValueWrapper.getColumnValue().setColumnData(null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,60 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.BaseChangeRecordEmitter;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||||
|
import io.debezium.data.Envelope.Operation;
|
||||||
|
import io.debezium.pipeline.spi.OffsetContext;
|
||||||
|
import io.debezium.relational.Table;
|
||||||
|
import io.debezium.util.Clock;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Emits change record based on a single {@link LogMinerDmlEntry} event.
|
||||||
|
*/
|
||||||
|
public class LogMinerChangeRecordEmitter extends BaseChangeRecordEmitter<LogMinerColumnValue> {
|
||||||
|
|
||||||
|
private LogMinerDmlEntry dmlEntry;
|
||||||
|
protected final Table table;
|
||||||
|
|
||||||
|
public LogMinerChangeRecordEmitter(OffsetContext offset, LogMinerDmlEntry dmlEntry, Table table, Clock clock) {
|
||||||
|
super(offset, table, clock);
|
||||||
|
this.dmlEntry = dmlEntry;
|
||||||
|
this.table = table;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Operation getOperation() {
|
||||||
|
return dmlEntry.getCommandType();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object[] getOldColumnValues() {
|
||||||
|
List<LogMinerColumnValue> valueList = dmlEntry.getOldValues();
|
||||||
|
LogMinerColumnValue[] result = Arrays.copyOf(valueList.toArray(), valueList.size(), LogMinerColumnValue[].class);
|
||||||
|
return getColumnValues(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object[] getNewColumnValues() {
|
||||||
|
List<LogMinerColumnValue> valueList = dmlEntry.getNewValues();
|
||||||
|
LogMinerColumnValue[] result = Arrays.copyOf(valueList.toArray(), valueList.size(), LogMinerColumnValue[].class);
|
||||||
|
return getColumnValues(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String getColumnName(LogMinerColumnValue columnValue) {
|
||||||
|
return columnValue.getColumnName();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Object getColumnData(LogMinerColumnValue columnValue) {
|
||||||
|
return columnValue.getColumnData();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,431 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.CallableStatement;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.PreparedStatement;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Statement;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.OracleConnection;
|
||||||
|
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||||
|
import io.debezium.jdbc.JdbcConnection;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class contains methods to configure and manage Log Miner utility
|
||||||
|
*/
|
||||||
|
public class LogMinerHelper {
|
||||||
|
|
||||||
|
private final static String UNKNOWN = "unknown";
|
||||||
|
private final static Logger LOGGER = LoggerFactory.getLogger(LogMinerHelper.class);
|
||||||
|
|
||||||
|
private enum DATATYPE {
|
||||||
|
LONG,
|
||||||
|
TIMESTAMP,
|
||||||
|
STRING
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This builds data dictionary objects in redo log files.
|
||||||
|
* During this build, Oracle does an additional REDO LOG switch.
|
||||||
|
* This call may take time, which leads to delay in delivering incremental changes.
|
||||||
|
* With this option the lag between source database and dispatching event fluctuates.
|
||||||
|
*
|
||||||
|
* @param connection connection to the database as log miner user (connection to the container)
|
||||||
|
* @throws SQLException any exception
|
||||||
|
*/
|
||||||
|
static void buildDataDictionary(Connection connection) throws SQLException {
|
||||||
|
executeCallableStatement(connection, SqlUtils.BUILD_DICTIONARY);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method returns current SCN from the database
|
||||||
|
*
|
||||||
|
* @param connection container level database connection
|
||||||
|
* @return current SCN
|
||||||
|
* @throws SQLException if anything unexpected happens
|
||||||
|
*/
|
||||||
|
public static long getCurrentScn(Connection connection) throws SQLException {
|
||||||
|
try (Statement statement = connection.createStatement();
|
||||||
|
ResultSet rs = statement.executeQuery(SqlUtils.CURRENT_SCN)) {
|
||||||
|
|
||||||
|
if (!rs.next()) {
|
||||||
|
throw new IllegalStateException("Couldn't get SCN");
|
||||||
|
}
|
||||||
|
|
||||||
|
long currentScn = rs.getLong(1);
|
||||||
|
rs.close();
|
||||||
|
return currentScn;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void createAuditTable(Connection connection) throws SQLException {
|
||||||
|
String tableExists = (String) getSingleResult(connection, SqlUtils.AUDIT_TABLE_EXISTS, DATATYPE.STRING);
|
||||||
|
if (tableExists == null) {
|
||||||
|
executeCallableStatement(connection, SqlUtils.CREATE_AUDIT_TABLE);
|
||||||
|
}
|
||||||
|
|
||||||
|
String recordExists = (String) getSingleResult(connection, SqlUtils.AUDIT_TABLE_RECORD_EXISTS, DATATYPE.STRING);
|
||||||
|
if (recordExists == null) {
|
||||||
|
executeCallableStatement(connection, SqlUtils.INSERT_AUDIT_TABLE);
|
||||||
|
connection.commit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method returns next SCN for mining and also updates MBean metrics
|
||||||
|
* We use a configurable limit, because the larger mining range, the slower query from Log Miner content view.
|
||||||
|
* In addition capturing unlimited number of changes can blow up Java heap.
|
||||||
|
* Gradual querying helps to catch up faster after long delays in mining.
|
||||||
|
*
|
||||||
|
* @param connection container level database connection
|
||||||
|
* @param metrics MBean accessible metrics
|
||||||
|
* @param startScn start SCN
|
||||||
|
* @return next SCN to mine to
|
||||||
|
* @throws SQLException if anything unexpected happens
|
||||||
|
*/
|
||||||
|
static long getEndScn(Connection connection, long startScn, LogMinerMetrics metrics) throws SQLException {
|
||||||
|
long currentScn = getCurrentScn(connection);
|
||||||
|
metrics.setCurrentScn(currentScn);
|
||||||
|
int miningDiapason = metrics.getBatchSize();
|
||||||
|
|
||||||
|
// it is critical to flush LogWriter buffer
|
||||||
|
executeCallableStatement(connection, SqlUtils.UPDATE_AUDIT_TABLE + currentScn);
|
||||||
|
if (!connection.getAutoCommit()) {
|
||||||
|
connection.commit();
|
||||||
|
}
|
||||||
|
|
||||||
|
// adjust sleeping time to optimize DB impact and catchup faster when behind
|
||||||
|
boolean isNextScnCloseToDbCurrent = currentScn < (startScn + miningDiapason);
|
||||||
|
metrics.changeSleepingTime(isNextScnCloseToDbCurrent);
|
||||||
|
|
||||||
|
return isNextScnCloseToDbCurrent ? currentScn : startScn + miningDiapason;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate time difference between database and connector timers. It could be negative if DB time is ahead.
|
||||||
|
* @param connection connection
|
||||||
|
* @return difference in milliseconds
|
||||||
|
*/
|
||||||
|
static long getTimeDifference(Connection connection) throws SQLException {
|
||||||
|
Timestamp dbCurrentMillis = (Timestamp) getSingleResult(connection, SqlUtils.CURRENT_TIMESTAMP, DATATYPE.TIMESTAMP);
|
||||||
|
if (dbCurrentMillis == null) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
Instant fromDb = dbCurrentMillis.toInstant();
|
||||||
|
Instant now = Instant.now();
|
||||||
|
return Duration.between(fromDb, now).toMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method builds mining view to query changes from.
|
||||||
|
* This view is built for online redo log files.
|
||||||
|
* It starts log mining session.
|
||||||
|
* It uses data dictionary objects, incorporated in previous steps.
|
||||||
|
* It tracks DDL changes and mines committed data only.
|
||||||
|
*
|
||||||
|
* @param connection container level database connection
|
||||||
|
* @param startScn the SCN to mine from
|
||||||
|
* @param endScn the SCN to mine to
|
||||||
|
* @param strategy this is about dictionary location
|
||||||
|
* @param isContinuousMining works < 19 version only
|
||||||
|
* @throws SQLException if anything unexpected happens
|
||||||
|
*/
|
||||||
|
static void startOnlineMining(Connection connection, Long startScn, Long endScn,
|
||||||
|
OracleConnectorConfig.LogMiningStrategy strategy, boolean isContinuousMining)
|
||||||
|
throws SQLException {
|
||||||
|
String statement = SqlUtils.getStartLogMinerStatement(startScn, endScn, strategy, isContinuousMining);
|
||||||
|
executeCallableStatement(connection, statement);
|
||||||
|
// todo dbms_logmnr.STRING_LITERALS_IN_STMT?
|
||||||
|
// todo If the log file is corrupted/bad, logmnr will not be able to access it, we have to switch to another one?
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method query the database to get CURRENT online redo log file(s). Multiple is applicable for RAC systems.
|
||||||
|
* @param connection connection to reuse
|
||||||
|
* @param metrics MBean accessible metrics
|
||||||
|
* @return full redo log file name(s), including path
|
||||||
|
* @throws SQLException if anything unexpected happens
|
||||||
|
*/
|
||||||
|
static Set<String> getCurrentRedoLogFiles(Connection connection, LogMinerMetrics metrics) throws SQLException {
|
||||||
|
String checkQuery = SqlUtils.CURRENT_REDO_LOG_NAME;
|
||||||
|
|
||||||
|
Set<String> fileNames = new HashSet<>();
|
||||||
|
PreparedStatement st = connection.prepareStatement(checkQuery);
|
||||||
|
ResultSet result = st.executeQuery();
|
||||||
|
while (result.next()) {
|
||||||
|
fileNames.add(result.getString(1));
|
||||||
|
LOGGER.trace(" Current Redo log fileName: {} ", fileNames);
|
||||||
|
}
|
||||||
|
st.close();
|
||||||
|
result.close();
|
||||||
|
|
||||||
|
updateRedoLogMetrics(connection, metrics, fileNames);
|
||||||
|
return fileNames;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method fetches the oldest SCN from online redo log files
|
||||||
|
*
|
||||||
|
* @param connection container level database connection
|
||||||
|
* @return oldest SCN from online redo log
|
||||||
|
* @throws SQLException if anything unexpected happens
|
||||||
|
*/
|
||||||
|
static long getFirstOnlineLogScn(Connection connection) throws SQLException {
|
||||||
|
LOGGER.trace("getting first scn of all online logs");
|
||||||
|
Statement s = connection.createStatement();
|
||||||
|
ResultSet res = s.executeQuery(SqlUtils.OLDEST_FIRST_CHANGE);
|
||||||
|
res.next();
|
||||||
|
long firstScnOfOnlineLog = res.getLong(1);
|
||||||
|
res.close();
|
||||||
|
return firstScnOfOnlineLog;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets NLS parameters for mining session.
|
||||||
|
*
|
||||||
|
* @param connection session level database connection
|
||||||
|
* @throws SQLException if anything unexpected happens
|
||||||
|
*/
|
||||||
|
static void setNlsSessionParameters(JdbcConnection connection) throws SQLException {
|
||||||
|
connection.executeWithoutCommitting(SqlUtils.NLS_SESSION_PARAMETERS);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to update MBean metrics associated with REDO LOG groups
|
||||||
|
* @param connection connection
|
||||||
|
* @param fileNames name of current REDO LOG files
|
||||||
|
* @param metrics current metrics
|
||||||
|
*/
|
||||||
|
private static void updateRedoLogMetrics(Connection connection, LogMinerMetrics metrics, Set<String> fileNames) {
|
||||||
|
try {
|
||||||
|
// update metrics
|
||||||
|
Map<String, String> logStatuses = getRedoLogStatus(connection);
|
||||||
|
metrics.setRedoLogStatus(logStatuses);
|
||||||
|
|
||||||
|
int counter = getSwitchCount(connection);
|
||||||
|
metrics.setSwitchCount(counter);
|
||||||
|
metrics.setCurrentLogFileName(fileNames);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
LOGGER.error("Cannot update metrics");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This fetches online redo log statuses
|
||||||
|
* @param connection privileged connection
|
||||||
|
* @return REDO LOG statuses Map, where key is the REDO name and value is the status
|
||||||
|
* @throws SQLException if anything unexpected happens
|
||||||
|
*/
|
||||||
|
private static Map<String, String> getRedoLogStatus(Connection connection) throws SQLException {
|
||||||
|
return getMap(connection, SqlUtils.REDO_LOGS_STATUS, UNKNOWN);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This fetches REDO LOG switch count for the last day
|
||||||
|
* @param connection privileged connection
|
||||||
|
* @return counter
|
||||||
|
*/
|
||||||
|
private static int getSwitchCount(Connection connection) {
|
||||||
|
try {
|
||||||
|
Map<String, String> total = getMap(connection, SqlUtils.SWITCH_HISTORY_TOTAL_COUNT, UNKNOWN);
|
||||||
|
if (total != null && total.get("total") != null) {
|
||||||
|
return Integer.parseInt(total.get("total"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception e) {
|
||||||
|
LOGGER.error("Cannot get switch counter due to the {}", e);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method checks if supplemental logging was set on the database level. This is critical check, cannot work if not.
|
||||||
|
* @param connection oracle connection on logminer level
|
||||||
|
* @param pdbName pdb name
|
||||||
|
* @throws SQLException if anything unexpected happens
|
||||||
|
*/
|
||||||
|
static void checkSupplementalLogging(OracleConnection connection, String pdbName) throws SQLException {
|
||||||
|
try {
|
||||||
|
if (pdbName != null) {
|
||||||
|
connection.setSessionToPdb(pdbName);
|
||||||
|
}
|
||||||
|
|
||||||
|
final String key = "KEY";
|
||||||
|
String validateGlobalLogging = "SELECT '" + key + "', " + " SUPPLEMENTAL_LOG_DATA_ALL from V$DATABASE";
|
||||||
|
Map<String, String> globalLogging = getMap(connection.connection(false), validateGlobalLogging, UNKNOWN);
|
||||||
|
if ("no".equalsIgnoreCase(globalLogging.get(key))) {
|
||||||
|
throw new RuntimeException("Supplemental logging was not set. Use command: ALTER DATABASE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
if (pdbName != null) {
|
||||||
|
connection.resetSessionToCdb();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This call completes log miner session.
|
||||||
|
* Complete gracefully.
|
||||||
|
*
|
||||||
|
* @param connection container level database connection
|
||||||
|
*/
|
||||||
|
static void endMining(Connection connection) {
|
||||||
|
String stopMining = SqlUtils.END_LOGMNR;
|
||||||
|
try {
|
||||||
|
executeCallableStatement(connection, stopMining);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
if (e.getMessage().toUpperCase().contains("ORA-01307")) {
|
||||||
|
LOGGER.info("Log Miner session was already closed");
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
LOGGER.error("Cannot close Log Miner session gracefully: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method substitutes CONTINUOUS_MINE functionality for online files only
|
||||||
|
* @param connection connection
|
||||||
|
* @param lastProcessedScn current offset
|
||||||
|
* @throws SQLException if anything unexpected happens
|
||||||
|
*/
|
||||||
|
static void setRedoLogFilesForMining(Connection connection, Long lastProcessedScn) throws SQLException {
|
||||||
|
|
||||||
|
Map<String, Long> logFilesForMining = getLogFilesForOffsetScn(connection, lastProcessedScn);
|
||||||
|
if (logFilesForMining.isEmpty()) {
|
||||||
|
throw new IllegalStateException("The online log files do not contain offset SCN: " + lastProcessedScn + ", re-snapshot is required.");
|
||||||
|
}
|
||||||
|
|
||||||
|
// The following check seems to be problematic and removal of it does not appear to cause
|
||||||
|
// any detrimental impact to the connector's ability to stream changes. This was left as
|
||||||
|
// is but commented in case we find a reason for it to exist.
|
||||||
|
// int redoLogGroupSize = getRedoLogGroupSize(connection);
|
||||||
|
// if (logFilesForMining.size() == redoLogGroupSize) {
|
||||||
|
// throw new IllegalStateException("All online log files needed for mining the offset SCN: " + lastProcessedScn + ", re-snapshot is required.");
|
||||||
|
// }
|
||||||
|
|
||||||
|
List<String> logFilesNamesForMining = logFilesForMining.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList());
|
||||||
|
|
||||||
|
for (String file : logFilesNamesForMining) {
|
||||||
|
String addLogFileStatement = SqlUtils.getAddLogFileStatement("DBMS_LOGMNR.ADDFILE", file);
|
||||||
|
executeCallableStatement(connection, addLogFileStatement);
|
||||||
|
LOGGER.trace("add log file to the mining session = {}", file);
|
||||||
|
}
|
||||||
|
|
||||||
|
LOGGER.debug("Last mined SCN: {}, Log file list to mine: {}\n", lastProcessedScn, logFilesForMining);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method returns SCN as a watermark to abandon long lasting transactions.
|
||||||
|
*
|
||||||
|
* @param connection connection
|
||||||
|
* @param offsetScn current offset
|
||||||
|
* @return Optional last SCN in a redo log
|
||||||
|
* @throws SQLException if anything unexpected happens
|
||||||
|
*/
|
||||||
|
static Optional<Long> getLastScnFromTheOldestOnlineRedo(Connection connection, Long offsetScn) throws SQLException {
|
||||||
|
|
||||||
|
Map<String, String> allOnlineRedoLogFiles = getMap(connection, SqlUtils.ALL_ONLINE_LOGS, "-1");
|
||||||
|
Map<String, Long> logFilesToMine = getLogFilesForOffsetScn(connection, offsetScn);
|
||||||
|
|
||||||
|
LOGGER.debug("Redo log size = {}, needed for mining files size = {}", allOnlineRedoLogFiles.size(), logFilesToMine.size());
|
||||||
|
if (allOnlineRedoLogFiles.size() - logFilesToMine.size() <= 1) {
|
||||||
|
List<Long> lastScnsInRedoLogToMine = logFilesToMine.entrySet().stream().map(Map.Entry::getValue).collect(Collectors.toList());
|
||||||
|
return lastScnsInRedoLogToMine.stream().min(Long::compareTo);
|
||||||
|
}
|
||||||
|
return Optional.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
static void logWarn(TransactionalBufferMetrics metrics, String format, Object... args) {
|
||||||
|
LOGGER.warn(format, args);
|
||||||
|
metrics.incrementWarningCounter();
|
||||||
|
}
|
||||||
|
|
||||||
|
static void logError(TransactionalBufferMetrics metrics, String format, Object... args) {
|
||||||
|
LOGGER.error(format, args);
|
||||||
|
metrics.incrementErrorCounter();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get size of online REDO groups
|
||||||
|
* @param connection connection
|
||||||
|
* @return size
|
||||||
|
*/
|
||||||
|
private static int getRedoLogGroupSize(Connection connection) throws SQLException {
|
||||||
|
return getMap(connection, SqlUtils.ALL_ONLINE_LOGS, "-1").size();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method returns all online log files, starting from one which contains offset SCN and ending with one containing largest SCN
|
||||||
|
* 18446744073709551615 on Ora 19c is the max value of the nextScn in the current redo todo replace all Long with BigDecimal for SCN
|
||||||
|
*/
|
||||||
|
private static Map<String, Long> getLogFilesForOffsetScn(Connection connection, Long offsetScn) throws SQLException {
|
||||||
|
Map<String, String> redoLogFiles = getMap(connection, SqlUtils.ALL_ONLINE_LOGS, "-1");
|
||||||
|
return redoLogFiles.entrySet().stream()
|
||||||
|
.filter(entry -> new BigDecimal(entry.getValue()).longValue() > offsetScn || new BigDecimal(entry.getValue()).longValue() == -1).collect(Collectors
|
||||||
|
.toMap(Map.Entry::getKey, e -> new BigDecimal(e.getValue()).longValue() == -1 ? Long.MAX_VALUE : new BigDecimal(e.getValue()).longValue()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void executeCallableStatement(Connection connection, String statement) throws SQLException {
|
||||||
|
Objects.requireNonNull(statement);
|
||||||
|
CallableStatement s;
|
||||||
|
s = connection.prepareCall(statement);
|
||||||
|
s.execute();
|
||||||
|
s.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Map<String, String> getMap(Connection connection, String query, String nullReplacement) throws SQLException {
|
||||||
|
Map<String, String> result = new LinkedHashMap<>();
|
||||||
|
try (
|
||||||
|
PreparedStatement statement = connection.prepareStatement(query);
|
||||||
|
ResultSet rs = statement.executeQuery()) {
|
||||||
|
while (rs.next()) {
|
||||||
|
String value = rs.getString(2);
|
||||||
|
value = value == null ? nullReplacement : value;
|
||||||
|
result.put(rs.getString(1), value);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Object getSingleResult(Connection connection, String query, DATATYPE type) throws SQLException {
|
||||||
|
try (PreparedStatement statement = connection.prepareStatement(query);
|
||||||
|
ResultSet rs = statement.executeQuery()) {
|
||||||
|
if (rs.next()) {
|
||||||
|
switch (type) {
|
||||||
|
case LONG:
|
||||||
|
return rs.getLong(1);
|
||||||
|
case TIMESTAMP:
|
||||||
|
return rs.getTimestamp(1);
|
||||||
|
case STRING:
|
||||||
|
return rs.getString(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,218 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
|
import io.debezium.annotation.ThreadSafe;
|
||||||
|
import io.debezium.connector.common.CdcSourceTaskContext;
|
||||||
|
import io.debezium.metrics.Metrics;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class contains methods to be exposed via MBean server
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
@ThreadSafe
|
||||||
|
public class LogMinerMetrics extends Metrics implements LogMinerMetricsMXBean {
|
||||||
|
private AtomicLong currentScn = new AtomicLong();
|
||||||
|
private AtomicInteger capturedDmlCount = new AtomicInteger();
|
||||||
|
private AtomicReference<String[]> currentLogFileName;
|
||||||
|
private AtomicReference<String[]> redoLogStatus;
|
||||||
|
private AtomicInteger switchCounter = new AtomicInteger();
|
||||||
|
private AtomicReference<Duration> lastLogMinerQueryDuration = new AtomicReference<>();
|
||||||
|
private AtomicReference<Duration> averageLogMinerQueryDuration = new AtomicReference<>();
|
||||||
|
private AtomicInteger logMinerQueryCount = new AtomicInteger();
|
||||||
|
private AtomicReference<Duration> lastProcessedCapturedBatchDuration = new AtomicReference<>();
|
||||||
|
private AtomicInteger processedCapturedBatchCount = new AtomicInteger();
|
||||||
|
private AtomicReference<Duration> averageProcessedCapturedBatchDuration = new AtomicReference<>();
|
||||||
|
private AtomicInteger batchSize = new AtomicInteger();
|
||||||
|
private AtomicInteger millisecondToSleepBetweenMiningQuery = new AtomicInteger();
|
||||||
|
|
||||||
|
private final int MAX_SLEEP_TIME = 3_000;
|
||||||
|
private final int DEFAULT_SLEEP_TIME = 1_000;
|
||||||
|
private final int MIN_SLEEP_TIME = 100;
|
||||||
|
|
||||||
|
private final int MIN_BATCH_SIZE = 1_000;
|
||||||
|
private final int MAX_BATCH_SIZE = 100_000;
|
||||||
|
private final int DEFAULT_BATCH_SIZE = 5_000;
|
||||||
|
|
||||||
|
private final int SLEEP_TIME_INCREMENT = 200;
|
||||||
|
|
||||||
|
LogMinerMetrics(CdcSourceTaskContext taskContext) {
|
||||||
|
super(taskContext, "log-miner");
|
||||||
|
|
||||||
|
batchSize.set(DEFAULT_BATCH_SIZE);
|
||||||
|
millisecondToSleepBetweenMiningQuery.set(DEFAULT_SLEEP_TIME);
|
||||||
|
|
||||||
|
currentScn.set(-1);
|
||||||
|
capturedDmlCount.set(0);
|
||||||
|
currentLogFileName = new AtomicReference<>();
|
||||||
|
redoLogStatus = new AtomicReference<>();
|
||||||
|
switchCounter.set(0);
|
||||||
|
averageLogMinerQueryDuration.set(Duration.ZERO);
|
||||||
|
lastLogMinerQueryDuration.set(Duration.ZERO);
|
||||||
|
logMinerQueryCount.set(0);
|
||||||
|
lastProcessedCapturedBatchDuration.set(Duration.ZERO);
|
||||||
|
processedCapturedBatchCount.set(0);
|
||||||
|
averageProcessedCapturedBatchDuration.set(Duration.ZERO);
|
||||||
|
}
|
||||||
|
|
||||||
|
// setters
|
||||||
|
public void setCurrentScn(Long scn) {
|
||||||
|
currentScn.set(scn);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void incrementCapturedDmlCount() {
|
||||||
|
capturedDmlCount.incrementAndGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCurrentLogFileName(Set<String> names) {
|
||||||
|
currentLogFileName.set(names.stream().toArray(String[]::new));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRedoLogStatus(Map<String, String> status) {
|
||||||
|
String[] statusArray = status.entrySet().stream().map(e -> e.getKey() + " | " + e.getValue()).toArray(String[]::new);
|
||||||
|
redoLogStatus.set(statusArray);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSwitchCount(int counter) {
|
||||||
|
switchCounter.set(counter);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setLastLogMinerQueryDuration(Duration fetchDuration) {
|
||||||
|
setDurationMetrics(fetchDuration, lastLogMinerQueryDuration, logMinerQueryCount, averageLogMinerQueryDuration);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setProcessedCapturedBatchDuration(Duration processDuration) {
|
||||||
|
setDurationMetrics(processDuration, lastProcessedCapturedBatchDuration, processedCapturedBatchCount, averageProcessedCapturedBatchDuration);
|
||||||
|
}
|
||||||
|
|
||||||
|
// implemented getters
|
||||||
|
@Override
|
||||||
|
public Long getCurrentScn() {
|
||||||
|
return currentScn.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getCapturedDmlCount() {
|
||||||
|
return capturedDmlCount.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String[] getCurrentRedoLogFileName() {
|
||||||
|
return currentLogFileName.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String[] getRedoLogStatus() {
|
||||||
|
return redoLogStatus.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getSwitchCounter() {
|
||||||
|
return switchCounter.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Long getLastLogMinerQueryDuration() {
|
||||||
|
return lastLogMinerQueryDuration.get() == null ? 0 : lastLogMinerQueryDuration.get().toMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Long getAverageLogMinerQueryDuration() {
|
||||||
|
return averageLogMinerQueryDuration.get() == null ? 0 : averageLogMinerQueryDuration.get().toMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Long getLastProcessedCapturedBatchDuration() {
|
||||||
|
return lastProcessedCapturedBatchDuration.get() == null ? 0 : lastProcessedCapturedBatchDuration.get().toMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getLogMinerQueryCount() {
|
||||||
|
return logMinerQueryCount.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getProcessedCapturedBatchCount() {
|
||||||
|
return processedCapturedBatchCount.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Long getAverageProcessedCapturedBatchDuration() {
|
||||||
|
return averageProcessedCapturedBatchDuration.get() == null ? 0 : averageProcessedCapturedBatchDuration.get().toMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getBatchSize() {
|
||||||
|
return batchSize.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Integer getMillisecondToSleepBetweenMiningQuery() {
|
||||||
|
return millisecondToSleepBetweenMiningQuery.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
// MBean accessible setters
|
||||||
|
@Override
|
||||||
|
public void setBatchSize(int size) {
|
||||||
|
if (size >= MIN_BATCH_SIZE && size <= MAX_BATCH_SIZE) {
|
||||||
|
batchSize.set(size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setMillisecondToSleepBetweenMiningQuery(Integer milliseconds) {
|
||||||
|
if (milliseconds != null && milliseconds >= MIN_SLEEP_TIME && milliseconds < MAX_SLEEP_TIME) {
|
||||||
|
millisecondToSleepBetweenMiningQuery.set(milliseconds);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void changeSleepingTime(boolean increment) {
|
||||||
|
int sleepTime = millisecondToSleepBetweenMiningQuery.get();
|
||||||
|
int change = increment ? SLEEP_TIME_INCREMENT : -SLEEP_TIME_INCREMENT;
|
||||||
|
if (sleepTime >= MIN_SLEEP_TIME && sleepTime < MAX_SLEEP_TIME) {
|
||||||
|
millisecondToSleepBetweenMiningQuery.getAndAdd(change);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// private methods
|
||||||
|
private void setDurationMetrics(Duration duration, AtomicReference<Duration> lastDuration, AtomicInteger counter,
|
||||||
|
AtomicReference<Duration> averageDuration) {
|
||||||
|
if (duration != null) {
|
||||||
|
lastDuration.set(duration);
|
||||||
|
int count = counter.incrementAndGet();
|
||||||
|
Duration currentAverage = averageDuration.get() == null ? Duration.ZERO : averageDuration.get();
|
||||||
|
averageDuration.set(currentAverage.multipliedBy(count - 1).plus(duration).dividedBy(count));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "LogMinerMetrics{" +
|
||||||
|
"currentEndScn=" + currentScn.get() +
|
||||||
|
", currentLogFileNames=" + Arrays.toString(currentLogFileName.get()) +
|
||||||
|
", redoLogStatus=" + Arrays.toString(redoLogStatus.get()) +
|
||||||
|
", capturedDmlCount=" + capturedDmlCount.get() +
|
||||||
|
", switchCounter=" + switchCounter.get() +
|
||||||
|
", lastLogMinerQueryDuration=" + lastLogMinerQueryDuration.get() +
|
||||||
|
", logMinerQueryCount=" + logMinerQueryCount.get() +
|
||||||
|
", averageLogMinerQueryDuration=" + averageLogMinerQueryDuration.get() +
|
||||||
|
", lastProcessedCapturedBatchDuration=" + lastProcessedCapturedBatchDuration.get() +
|
||||||
|
", processedCapturedBatchCount=" + processedCapturedBatchCount.get() +
|
||||||
|
", averageProcessedCapturedBatchDuration=" + averageProcessedCapturedBatchDuration.get() +
|
||||||
|
", millisecondToSleepBetweenMiningQuery=" + millisecondToSleepBetweenMiningQuery.get() +
|
||||||
|
", batchSize=" + batchSize.get() +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,107 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This interface is exposed for JMX
|
||||||
|
*/
|
||||||
|
public interface LogMinerMetricsMXBean {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exposes current SCN in the database. This is very efficient query and will not affect overall performance
|
||||||
|
*
|
||||||
|
* @return current SCN
|
||||||
|
*/
|
||||||
|
Long getCurrentScn();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exposes current redo log file. This is very efficient query and will not affect overall performance
|
||||||
|
*
|
||||||
|
* @return full path or NULL if an exception occurs.
|
||||||
|
*/
|
||||||
|
String[] getCurrentRedoLogFileName();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exposes states of redo logs: current, active, inactive, unused ...
|
||||||
|
* @return array of: (redo log name | status) elements
|
||||||
|
*/
|
||||||
|
String[] getRedoLogStatus();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* fetches counter of redo switches for the last day.
|
||||||
|
* If this number is high , like once in 3 minutes, the troubleshooting on the database level is required.
|
||||||
|
* @return counter
|
||||||
|
*/
|
||||||
|
int getSwitchCounter();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return number of milliseconds last Log Miner query took
|
||||||
|
*/
|
||||||
|
Long getLastLogMinerQueryDuration();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return number of captured DML since the connector is up
|
||||||
|
*/
|
||||||
|
int getCapturedDmlCount();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return number of Log Miner view queries since the connector is up
|
||||||
|
*/
|
||||||
|
int getLogMinerQueryCount();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return average duration of Log Miner view query
|
||||||
|
*/
|
||||||
|
Long getAverageLogMinerQueryDuration();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Log Miner view query returns number of captured DML , Commit and Rollback. This is what we call a batch.
|
||||||
|
* @return duration of the last batch processing, which includes parsing and dispatching
|
||||||
|
*/
|
||||||
|
Long getLastProcessedCapturedBatchDuration();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Log Miner view query returns number of captured DML , Commit and Rollback. This is what we call a batch.
|
||||||
|
* @return number of all processed batches , which includes parsing and dispatching
|
||||||
|
*/
|
||||||
|
int getProcessedCapturedBatchCount();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return average time of processing captured batch from Log Miner view
|
||||||
|
*/
|
||||||
|
Long getAverageProcessedCapturedBatchDuration();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maximum number of entries in Log Miner view to fetch. This is used to set the diapason of the SCN in mining query.
|
||||||
|
* If difference between "start SCN" and "end SCN" to mine exceeds this limit, end SCN will be set to "start SCN" + batchSize
|
||||||
|
* @return the limit
|
||||||
|
*/
|
||||||
|
int getBatchSize();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* this gives ability to manipulate number of entries in Log Miner view to fetch.
|
||||||
|
* It has limits to prevent abnormal values
|
||||||
|
* @param size limit
|
||||||
|
*/
|
||||||
|
void setBatchSize(int size);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return number of milliseconds for connector to sleep before fetching another batch from the Log Miner view
|
||||||
|
*/
|
||||||
|
Integer getMillisecondToSleepBetweenMiningQuery();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* sets number of milliseconds for connector to sleep before fetching another batch from the Log Miner view
|
||||||
|
* @param milliseconds to sleep
|
||||||
|
*/
|
||||||
|
void setMillisecondToSleepBetweenMiningQuery(Integer milliseconds);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* change sleeping time
|
||||||
|
* @param increment true to add, false to deduct
|
||||||
|
*/
|
||||||
|
void changeSleepingTime(boolean increment);
|
||||||
|
}
|
@ -0,0 +1,245 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.OracleDatabaseSchema;
|
||||||
|
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||||
|
import io.debezium.connector.oracle.jsqlparser.SimpleDmlParser;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||||
|
import io.debezium.data.Envelope;
|
||||||
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
|
import io.debezium.pipeline.source.spi.ChangeEventSource;
|
||||||
|
import io.debezium.relational.Table;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
import io.debezium.util.Clock;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class process entries obtained from LogMiner view.
|
||||||
|
* It parses each entry.
|
||||||
|
* On each DML it registers a callback in TransactionalBuffer.
|
||||||
|
* On rollback it removes registered entries from TransactionalBuffer.
|
||||||
|
* On commit it executes all registered callbacks, which dispatch ChangeRecords.
|
||||||
|
* This also calculates metrics
|
||||||
|
*/
|
||||||
|
class LogMinerQueryResultProcessor {
|
||||||
|
|
||||||
|
private final ChangeEventSource.ChangeEventSourceContext context;
|
||||||
|
private final LogMinerMetrics metrics;
|
||||||
|
private final TransactionalBuffer transactionalBuffer;
|
||||||
|
private final SimpleDmlParser dmlParser;
|
||||||
|
private final OracleOffsetContext offsetContext;
|
||||||
|
private final OracleDatabaseSchema schema;
|
||||||
|
private final EventDispatcher<TableId> dispatcher;
|
||||||
|
private final TransactionalBufferMetrics transactionalBufferMetrics;
|
||||||
|
private final String catalogName;
|
||||||
|
private final Clock clock;
|
||||||
|
private final Logger LOGGER = LoggerFactory.getLogger(LogMinerQueryResultProcessor.class);
|
||||||
|
private long currentOffsetScn = 0;
|
||||||
|
private long currentOffsetCommitScn = 0;
|
||||||
|
private long stuckScnCounter = 0;
|
||||||
|
|
||||||
|
LogMinerQueryResultProcessor(ChangeEventSource.ChangeEventSourceContext context, LogMinerMetrics metrics,
|
||||||
|
TransactionalBuffer transactionalBuffer, SimpleDmlParser dmlParser,
|
||||||
|
OracleOffsetContext offsetContext, OracleDatabaseSchema schema,
|
||||||
|
EventDispatcher<TableId> dispatcher, TransactionalBufferMetrics transactionalBufferMetrics,
|
||||||
|
String catalogName, Clock clock) {
|
||||||
|
this.context = context;
|
||||||
|
this.metrics = metrics;
|
||||||
|
this.transactionalBuffer = transactionalBuffer;
|
||||||
|
this.dmlParser = dmlParser;
|
||||||
|
this.offsetContext = offsetContext;
|
||||||
|
this.schema = schema;
|
||||||
|
this.dispatcher = dispatcher;
|
||||||
|
this.transactionalBufferMetrics = transactionalBufferMetrics;
|
||||||
|
this.catalogName = catalogName;
|
||||||
|
this.clock = clock;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method does all the job
|
||||||
|
* @param resultSet the info from Log Miner view
|
||||||
|
* @return number of processed DMLs from the given resultSet
|
||||||
|
*/
|
||||||
|
int processResult(ResultSet resultSet) {
|
||||||
|
int dmlCounter = 0;
|
||||||
|
int commitCounter = 0;
|
||||||
|
int rollbackCounter = 0;
|
||||||
|
Duration cumulativeCommitTime = Duration.ZERO;
|
||||||
|
Duration cumulativeParseTime = Duration.ZERO;
|
||||||
|
Duration cumulativeOtherTime = Duration.ZERO;
|
||||||
|
Instant startTime = Instant.now();
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
if (!resultSet.next()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
LogMinerHelper.logError(transactionalBufferMetrics, "Closed resultSet");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
Instant iterationStart = Instant.now();
|
||||||
|
|
||||||
|
BigDecimal scn = RowMapper.getScn(transactionalBufferMetrics, resultSet);
|
||||||
|
String redo_sql = RowMapper.getSqlRedo(transactionalBufferMetrics, resultSet);
|
||||||
|
String tableName = RowMapper.getTableName(transactionalBufferMetrics, resultSet);
|
||||||
|
String segOwner = RowMapper.getSegOwner(transactionalBufferMetrics, resultSet);
|
||||||
|
int operationCode = RowMapper.getOperationCode(transactionalBufferMetrics, resultSet);
|
||||||
|
Timestamp changeTime = RowMapper.getChangeTime(transactionalBufferMetrics, resultSet);
|
||||||
|
String txId = RowMapper.getTransactionId(transactionalBufferMetrics, resultSet);
|
||||||
|
String operation = RowMapper.getOperation(transactionalBufferMetrics, resultSet);
|
||||||
|
String userName = RowMapper.getUsername(transactionalBufferMetrics, resultSet);
|
||||||
|
|
||||||
|
LOGGER.trace("scn={}, operationCode={}, operation={}, table={}, segOwner={}, userName={}", scn, operationCode, operation, tableName, segOwner, userName);
|
||||||
|
|
||||||
|
String logMessage = String.format("transactionId = %s, SCN= %s, table_name= %s, segOwner= %s, operationCode=%s, offsetSCN= %s, " +
|
||||||
|
" commitOffsetSCN= %s", txId, scn, tableName, segOwner, operationCode, offsetContext.getScn(), offsetContext.getCommitScn());
|
||||||
|
|
||||||
|
if (scn == null) {
|
||||||
|
LogMinerHelper.logWarn(transactionalBufferMetrics, "Scn is null for {}", logMessage);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Commit
|
||||||
|
if (operationCode == RowMapper.COMMIT) {
|
||||||
|
if (transactionalBuffer.commit(txId, scn, offsetContext, changeTime, context, logMessage)) {
|
||||||
|
LOGGER.trace("COMMIT, {}", logMessage);
|
||||||
|
commitCounter++;
|
||||||
|
cumulativeCommitTime = cumulativeCommitTime.plus(Duration.between(iterationStart, Instant.now()));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rollback
|
||||||
|
if (operationCode == RowMapper.ROLLBACK) {
|
||||||
|
if (transactionalBuffer.rollback(txId, logMessage)) {
|
||||||
|
LOGGER.trace("ROLLBACK, {}", logMessage);
|
||||||
|
rollbackCounter++;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// DDL
|
||||||
|
if (operationCode == RowMapper.DDL) {
|
||||||
|
// todo: DDL operations are not yet supported during streaming while using Log Miner.
|
||||||
|
LOGGER.info("DDL: {}, REDO_SQL: {}", logMessage, redo_sql);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// MISSING_SCN
|
||||||
|
if (operationCode == RowMapper.MISSING_SCN) {
|
||||||
|
LogMinerHelper.logWarn(transactionalBufferMetrics, "Missing SCN, {}", logMessage);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// DML
|
||||||
|
if (operationCode == RowMapper.INSERT || operationCode == RowMapper.DELETE || operationCode == RowMapper.UPDATE) {
|
||||||
|
LOGGER.trace("DML, {}, sql {}", logMessage, redo_sql);
|
||||||
|
dmlCounter++;
|
||||||
|
metrics.incrementCapturedDmlCount();
|
||||||
|
iterationStart = Instant.now();
|
||||||
|
LogMinerDmlEntry dmlEntry = dmlParser.parse(redo_sql, schema.getTables(), txId);
|
||||||
|
cumulativeParseTime = cumulativeParseTime.plus(Duration.between(iterationStart, Instant.now()));
|
||||||
|
iterationStart = Instant.now();
|
||||||
|
|
||||||
|
if (dmlEntry == null || redo_sql == null) {
|
||||||
|
LOGGER.trace("Following statement was not parsed: {}, details: {}", redo_sql, logMessage);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// this will happen for instance on a blacklisted column change, we will omit this update
|
||||||
|
if (dmlEntry.getCommandType().equals(Envelope.Operation.UPDATE)
|
||||||
|
&& dmlEntry.getOldValues().size() == dmlEntry.getNewValues().size()
|
||||||
|
&& dmlEntry.getNewValues().containsAll(dmlEntry.getOldValues())) {
|
||||||
|
LOGGER.trace("Following DML was skipped, " +
|
||||||
|
"most likely because of ignored blacklisted column change: {}, details: {}", redo_sql, logMessage);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
dmlEntry.setObjectOwner(segOwner);
|
||||||
|
dmlEntry.setSourceTime(changeTime);
|
||||||
|
dmlEntry.setTransactionId(txId);
|
||||||
|
dmlEntry.setObjectName(tableName);
|
||||||
|
dmlEntry.setScn(scn);
|
||||||
|
|
||||||
|
try {
|
||||||
|
TableId tableId = RowMapper.getTableId(catalogName, resultSet);
|
||||||
|
transactionalBuffer.registerCommitCallback(txId, scn, changeTime.toInstant(), redo_sql, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
// update SCN in offset context only if processed SCN less than SCN among other transactions
|
||||||
|
if (smallestScn == null || scn.compareTo(smallestScn) < 0) {
|
||||||
|
offsetContext.setScn(scn.longValue());
|
||||||
|
transactionalBufferMetrics.setOldestScn(scn.longValue());
|
||||||
|
}
|
||||||
|
offsetContext.setTransactionId(txId);
|
||||||
|
offsetContext.setSourceTime(timestamp.toInstant());
|
||||||
|
offsetContext.setTableId(tableId);
|
||||||
|
if (counter == 0) {
|
||||||
|
offsetContext.setCommitScn(commitScn.longValue());
|
||||||
|
}
|
||||||
|
Table table = schema.tableFor(tableId);
|
||||||
|
LOGGER.trace("Processing DML event {} scn {}", dmlEntry.toString(), scn);
|
||||||
|
|
||||||
|
dispatcher.dispatchDataChangeEvent(tableId,
|
||||||
|
new LogMinerChangeRecordEmitter(offsetContext, dmlEntry, table, clock));
|
||||||
|
});
|
||||||
|
cumulativeOtherTime = cumulativeOtherTime.plus(Duration.between(iterationStart, Instant.now()));
|
||||||
|
|
||||||
|
}
|
||||||
|
catch (Exception e) {
|
||||||
|
LogMinerHelper.logError(transactionalBufferMetrics, "Following dmlEntry: {} cannot be dispatched due to the : {}", dmlEntry, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
metrics.setProcessedCapturedBatchDuration(Duration.between(startTime, Instant.now()));
|
||||||
|
if (dmlCounter > 0 || commitCounter > 0 || rollbackCounter > 0) {
|
||||||
|
warnStuckScn();
|
||||||
|
currentOffsetScn = offsetContext.getScn();
|
||||||
|
if (offsetContext.getCommitScn() != null) {
|
||||||
|
currentOffsetCommitScn = offsetContext.getCommitScn();
|
||||||
|
}
|
||||||
|
LOGGER.debug("{} DMLs, {} Commits, {} Rollbacks. Millis - (total:{}, commit:{}, parse:{}, other:{}). " +
|
||||||
|
"Lag:{}. Offset scn:{}. Offset commit scn:{}. Active transactions:{}. Sleep time:{}",
|
||||||
|
dmlCounter, commitCounter, rollbackCounter, (Duration.between(startTime, Instant.now()).toMillis()),
|
||||||
|
cumulativeCommitTime.toMillis(), cumulativeParseTime.toMillis(), cumulativeOtherTime.toMillis(),
|
||||||
|
transactionalBufferMetrics.getLagFromSource(), offsetContext.getScn(), offsetContext.getCommitScn(),
|
||||||
|
transactionalBufferMetrics.getNumberOfActiveTransactions(), metrics.getMillisecondToSleepBetweenMiningQuery());
|
||||||
|
}
|
||||||
|
return dmlCounter;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method is warning if a long running transaction is discovered and could be abandoned in the future.
|
||||||
|
* The criteria is the offset SCN remains the same in five mining cycles
|
||||||
|
*/
|
||||||
|
private void warnStuckScn() {
|
||||||
|
if (offsetContext != null && offsetContext.getCommitScn() != null) {
|
||||||
|
if (currentOffsetScn == offsetContext.getScn() && currentOffsetCommitScn != offsetContext.getCommitScn()) {
|
||||||
|
stuckScnCounter++;
|
||||||
|
// logWarn only once
|
||||||
|
if (stuckScnCounter == 5) {
|
||||||
|
LogMinerHelper.logWarn(transactionalBufferMetrics,
|
||||||
|
"Offset SCN {} did not change in five mining cycles, hence the oldest transaction was not committed. Offset commit SCN: {}", currentOffsetScn,
|
||||||
|
offsetContext.getCommitScn());
|
||||||
|
transactionalBufferMetrics.incrementScnFreezeCounter();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
stuckScnCounter = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,27 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.BaseOracleSchemaChangeEventEmitter;
|
||||||
|
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDdlEntry;
|
||||||
|
import io.debezium.pipeline.spi.SchemaChangeEventEmitter;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@link SchemaChangeEventEmitter} implementation based on Oracle LogMiner utility.
|
||||||
|
*/
|
||||||
|
public class LogMinerSchemaChangeEventEmitter extends BaseOracleSchemaChangeEventEmitter {
|
||||||
|
|
||||||
|
public LogMinerSchemaChangeEventEmitter(OracleOffsetContext offsetContext, TableId tableId, LogMinerDdlEntry ddlLcr) {
|
||||||
|
super(offsetContext,
|
||||||
|
tableId,
|
||||||
|
tableId.catalog(), // todo tableId should be enough
|
||||||
|
tableId.schema(), // todo same here
|
||||||
|
ddlLcr.getDdlText(),
|
||||||
|
ddlLcr.getCommandType());
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,263 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.buildDataDictionary;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.checkSupplementalLogging;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.createAuditTable;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.endMining;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.getCurrentRedoLogFiles;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.getEndScn;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.getFirstOnlineLogScn;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.getLastScnFromTheOldestOnlineRedo;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.getTimeDifference;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.logError;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.logWarn;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.setNlsSessionParameters;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.setRedoLogFilesForMining;
|
||||||
|
import static io.debezium.connector.oracle.logminer.LogMinerHelper.startOnlineMining;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.PreparedStatement;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.SQLRecoverableException;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.OracleConnection;
|
||||||
|
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||||
|
import io.debezium.connector.oracle.OracleDatabaseSchema;
|
||||||
|
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||||
|
import io.debezium.connector.oracle.OracleTaskContext;
|
||||||
|
import io.debezium.connector.oracle.jsqlparser.SimpleDmlParser;
|
||||||
|
import io.debezium.pipeline.ErrorHandler;
|
||||||
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
|
import io.debezium.pipeline.source.spi.StreamingChangeEventSource;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
import io.debezium.util.Clock;
|
||||||
|
import io.debezium.util.Metronome;
|
||||||
|
|
||||||
|
import oracle.net.ns.NetException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A {@link StreamingChangeEventSource} based on Oracle's LogMiner utility.
|
||||||
|
* The event handler loop is executed in a separate executor.
|
||||||
|
*/
|
||||||
|
public class LogMinerStreamingChangeEventSource implements StreamingChangeEventSource {
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(LogMinerStreamingChangeEventSource.class);
|
||||||
|
|
||||||
|
private final OracleConnection jdbcConnection;
|
||||||
|
private final EventDispatcher<TableId> dispatcher;
|
||||||
|
private final Clock clock;
|
||||||
|
private final OracleDatabaseSchema schema;
|
||||||
|
private final OracleOffsetContext offsetContext;
|
||||||
|
private final SimpleDmlParser dmlParser;
|
||||||
|
private final String catalogName;
|
||||||
|
private OracleConnectorConfig connectorConfig;
|
||||||
|
private TransactionalBufferMetrics transactionalBufferMetrics;
|
||||||
|
private LogMinerMetrics logMinerMetrics;
|
||||||
|
private TransactionalBuffer transactionalBuffer;
|
||||||
|
private final OracleConnectorConfig.LogMiningStrategy strategy;
|
||||||
|
private final OracleTaskContext taskContext;
|
||||||
|
private final ErrorHandler errorHandler;
|
||||||
|
private final boolean isContinuousMining;
|
||||||
|
private long startScn;
|
||||||
|
private long endScn;
|
||||||
|
|
||||||
|
public LogMinerStreamingChangeEventSource(OracleConnectorConfig connectorConfig, OracleOffsetContext offsetContext,
|
||||||
|
OracleConnection jdbcConnection, EventDispatcher<TableId> dispatcher,
|
||||||
|
ErrorHandler errorHandler, Clock clock, OracleDatabaseSchema schema,
|
||||||
|
OracleTaskContext taskContext) {
|
||||||
|
this.jdbcConnection = jdbcConnection;
|
||||||
|
this.dispatcher = dispatcher;
|
||||||
|
this.clock = clock;
|
||||||
|
this.schema = schema;
|
||||||
|
this.offsetContext = offsetContext;
|
||||||
|
OracleChangeRecordValueConverter converters = new OracleChangeRecordValueConverter(jdbcConnection);
|
||||||
|
this.connectorConfig = connectorConfig;
|
||||||
|
this.catalogName = (connectorConfig.getPdbName() != null) ? connectorConfig.getPdbName() : connectorConfig.getDatabaseName();
|
||||||
|
this.dmlParser = new SimpleDmlParser(catalogName, connectorConfig.getSchemaName(), converters);
|
||||||
|
this.strategy = connectorConfig.getLogMiningStrategy();
|
||||||
|
this.isContinuousMining = connectorConfig.isContinuousMining();
|
||||||
|
this.errorHandler = errorHandler;
|
||||||
|
this.taskContext = taskContext;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is the loop to get changes from LogMiner
|
||||||
|
*
|
||||||
|
* @param context change event source context
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void execute(ChangeEventSourceContext context) {
|
||||||
|
Metronome metronome;
|
||||||
|
|
||||||
|
this.transactionalBufferMetrics = new TransactionalBufferMetrics(taskContext);
|
||||||
|
this.transactionalBufferMetrics.register(LOGGER);
|
||||||
|
|
||||||
|
this.transactionalBuffer = new TransactionalBuffer(connectorConfig.getLogicalName(), errorHandler, transactionalBufferMetrics, connectorConfig.getMaxQueueSize());
|
||||||
|
|
||||||
|
this.logMinerMetrics = new LogMinerMetrics(taskContext);
|
||||||
|
this.logMinerMetrics.register(LOGGER);
|
||||||
|
|
||||||
|
// The top outer loop gives the resiliency on the network disconnections. This is critical for cloud deployment.
|
||||||
|
while (context.isRunning()) {
|
||||||
|
try (Connection connection = jdbcConnection.connection(false);
|
||||||
|
PreparedStatement fetchFromMiningView = connection
|
||||||
|
.prepareStatement(SqlUtils.queryLogMinerContents(connectorConfig.getSchemaName(), jdbcConnection.username(), schema))) {
|
||||||
|
|
||||||
|
startScn = offsetContext.getScn();
|
||||||
|
createAuditTable(connection);
|
||||||
|
LOGGER.trace("current millis {}, db time {}", System.currentTimeMillis(), getTimeDifference(connection));
|
||||||
|
transactionalBufferMetrics.setTimeDifference(new AtomicLong(getTimeDifference(connection)));
|
||||||
|
|
||||||
|
if (!isContinuousMining && startScn < getFirstOnlineLogScn(connection)) {
|
||||||
|
throw new RuntimeException("Online REDO LOG files don't contain the offset SCN. Clean offset and start over");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Configure Log Miner to mine online redo logs
|
||||||
|
setNlsSessionParameters(jdbcConnection);
|
||||||
|
checkSupplementalLogging(jdbcConnection, connectorConfig.getPdbName());
|
||||||
|
|
||||||
|
if (strategy == OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO) {
|
||||||
|
buildDataDictionary(connection);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isContinuousMining) {
|
||||||
|
setRedoLogFilesForMining(connection, startScn);
|
||||||
|
}
|
||||||
|
|
||||||
|
LogMinerQueryResultProcessor processor = new LogMinerQueryResultProcessor(context, logMinerMetrics, transactionalBuffer,
|
||||||
|
dmlParser, offsetContext, schema, dispatcher, transactionalBufferMetrics, catalogName, clock);
|
||||||
|
|
||||||
|
// 2. Querying LogMiner view while running
|
||||||
|
Set<String> currentRedoLogFiles = getCurrentRedoLogFiles(connection, logMinerMetrics);
|
||||||
|
while (context.isRunning()) {
|
||||||
|
|
||||||
|
endScn = getEndScn(connection, startScn, logMinerMetrics);
|
||||||
|
// LOGGER.trace("startScn: {}, endScn: {}", startScn, endScn);
|
||||||
|
|
||||||
|
metronome = Metronome.sleeper(Duration.ofMillis(logMinerMetrics.getMillisecondToSleepBetweenMiningQuery()), clock);
|
||||||
|
metronome.pause();
|
||||||
|
|
||||||
|
Set<String> possibleNewCurrentLogFile = getCurrentRedoLogFiles(connection, logMinerMetrics);
|
||||||
|
if (!currentRedoLogFiles.equals(possibleNewCurrentLogFile)) {
|
||||||
|
LOGGER.debug("\n\n***** SWITCH occurred *****\n" + " from:{} , to:{} \n\n", currentRedoLogFiles, possibleNewCurrentLogFile);
|
||||||
|
|
||||||
|
// This is the way to mitigate PGA leak.
|
||||||
|
// With one mining session it grows and maybe there is another way to flush PGA, but at this point we use new mining session
|
||||||
|
endMining(connection);
|
||||||
|
|
||||||
|
if (!isContinuousMining) {
|
||||||
|
if (strategy == OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO) {
|
||||||
|
buildDataDictionary(connection);
|
||||||
|
}
|
||||||
|
|
||||||
|
abandonOldTransactionsIfExist(connection);
|
||||||
|
setRedoLogFilesForMining(connection, startScn);
|
||||||
|
}
|
||||||
|
|
||||||
|
currentRedoLogFiles = getCurrentRedoLogFiles(connection, logMinerMetrics);
|
||||||
|
}
|
||||||
|
|
||||||
|
startOnlineMining(connection, startScn, endScn, strategy, isContinuousMining);
|
||||||
|
|
||||||
|
Instant startTime = Instant.now();
|
||||||
|
fetchFromMiningView.setFetchSize(10_000);
|
||||||
|
fetchFromMiningView.setLong(1, startScn);
|
||||||
|
fetchFromMiningView.setLong(2, endScn);
|
||||||
|
|
||||||
|
ResultSet res = fetchFromMiningView.executeQuery();
|
||||||
|
logMinerMetrics.setLastLogMinerQueryDuration(Duration.between(startTime, Instant.now()));
|
||||||
|
processor.processResult(res);
|
||||||
|
|
||||||
|
updateStartScn();
|
||||||
|
// LOGGER.trace("largest scn = {}", transactionalBuffer.getLargestScn());
|
||||||
|
|
||||||
|
// update SCN in offset context only if buffer is empty, otherwise we update offset in TransactionalBuffer
|
||||||
|
if (transactionalBuffer.isEmpty()) {
|
||||||
|
offsetContext.setScn(startScn);
|
||||||
|
transactionalBuffer.resetLargestScn(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
res.close();
|
||||||
|
// we don't do it for other modes to save time on building data dictionary
|
||||||
|
// if (strategy == OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG) {
|
||||||
|
// endMining(connection);
|
||||||
|
// updateRedoLogMetrics(connection, logMinerMetrics);
|
||||||
|
// currentRedoLogFiles = getCurrentRedoLogFiles(connection, logMinerMetrics);
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Throwable e) {
|
||||||
|
if (connectionProblem(e)) {
|
||||||
|
logWarn(transactionalBufferMetrics, "Disconnection occurred. {} ", e.toString());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
logError(transactionalBufferMetrics, "Mining session was stopped due to the {} ", e.toString());
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
LOGGER.info("startScn={}, endScn={}, offsetContext.getScn()={}", startScn, endScn, offsetContext.getScn());
|
||||||
|
LOGGER.info("Transactional buffer metrics dump: {}", transactionalBufferMetrics.toString());
|
||||||
|
LOGGER.info("Transactional buffer dump: {}", transactionalBuffer.toString());
|
||||||
|
LOGGER.info("LogMiner metrics dump: {}", logMinerMetrics.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logMinerMetrics.unregister(LOGGER);
|
||||||
|
transactionalBufferMetrics.unregister(LOGGER);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void abandonOldTransactionsIfExist(Connection connection) throws SQLException {
|
||||||
|
Optional<Long> lastScnToAbandonTransactions = getLastScnFromTheOldestOnlineRedo(connection, offsetContext.getScn());
|
||||||
|
lastScnToAbandonTransactions.ifPresent(thresholdScn -> {
|
||||||
|
logWarn(transactionalBufferMetrics, "All transactions with first SCN <= {} will be abandoned, offset: {}", thresholdScn, offsetContext.getScn());
|
||||||
|
transactionalBuffer.abandonLongTransactions(thresholdScn);
|
||||||
|
offsetContext.setScn(thresholdScn);
|
||||||
|
updateStartScn();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private void updateStartScn() {
|
||||||
|
long nextStartScn = transactionalBuffer.getLargestScn().equals(BigDecimal.ZERO) ? endScn : transactionalBuffer.getLargestScn().longValue();
|
||||||
|
if (nextStartScn <= startScn) {
|
||||||
|
// When system is idle, largest SCN may stay unchanged, move it forward then
|
||||||
|
transactionalBuffer.resetLargestScn(endScn);
|
||||||
|
}
|
||||||
|
startScn = endScn;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void commitOffset(Map<String, ?> offset) {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean connectionProblem(Throwable e) {
|
||||||
|
if (e.getMessage() == null || e.getCause() == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return e.getMessage().startsWith("ORA-03135") || // connection lost contact
|
||||||
|
e.getMessage().startsWith("ORA-12543") || // TNS:destination host unreachable
|
||||||
|
e.getMessage().startsWith("ORA-00604") || // error occurred at recursive SQL level 1
|
||||||
|
e.getMessage().startsWith("ORA-01089") || // Oracle immediate shutdown in progress
|
||||||
|
e.getCause() instanceof IOException ||
|
||||||
|
e instanceof SQLRecoverableException ||
|
||||||
|
e.getMessage().toUpperCase().startsWith("NO MORE DATA TO READ FROM SOCKET") ||
|
||||||
|
e.getCause().getCause() instanceof NetException;
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,569 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import static io.debezium.util.NumberConversions.BYTE_FALSE;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Types;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.ZoneId;
|
||||||
|
import java.time.ZonedDateTime;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.time.format.DateTimeFormatterBuilder;
|
||||||
|
import java.time.temporal.ChronoField;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
import org.apache.kafka.connect.data.Field;
|
||||||
|
import org.apache.kafka.connect.data.SchemaBuilder;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.config.CommonConnectorConfig.BinaryHandlingMode;
|
||||||
|
import io.debezium.data.SpecialValueDecimal;
|
||||||
|
import io.debezium.data.VariableScaleDecimal;
|
||||||
|
import io.debezium.jdbc.JdbcConnection;
|
||||||
|
import io.debezium.jdbc.JdbcValueConverters;
|
||||||
|
import io.debezium.relational.Column;
|
||||||
|
import io.debezium.relational.ValueConverter;
|
||||||
|
import io.debezium.time.Date;
|
||||||
|
import io.debezium.time.MicroDuration;
|
||||||
|
import io.debezium.time.ZonedTimestamp;
|
||||||
|
import io.debezium.util.NumberConversions;
|
||||||
|
import io.debezium.util.Strings;
|
||||||
|
|
||||||
|
import oracle.jdbc.OracleTypes;
|
||||||
|
import oracle.sql.BINARY_DOUBLE;
|
||||||
|
import oracle.sql.BINARY_FLOAT;
|
||||||
|
import oracle.sql.BLOB;
|
||||||
|
import oracle.sql.CHAR;
|
||||||
|
import oracle.sql.CLOB;
|
||||||
|
import oracle.sql.DATE;
|
||||||
|
import oracle.sql.INTERVALDS;
|
||||||
|
import oracle.sql.INTERVALYM;
|
||||||
|
import oracle.sql.NUMBER;
|
||||||
|
import oracle.sql.TIMESTAMP;
|
||||||
|
import oracle.sql.TIMESTAMPLTZ;
|
||||||
|
import oracle.sql.TIMESTAMPTZ;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class is used as a value converter by the DML parser.
|
||||||
|
* todo this class is a replica of OracleValueConverters which is used by XStream and Logminer for snapshot mode
|
||||||
|
*/
|
||||||
|
public class OracleChangeRecordValueConverter extends JdbcValueConverters {
|
||||||
|
|
||||||
|
private static final Pattern INTERVAL_DAY_SECOND_PATTERN = Pattern.compile("([+\\-])?(\\d+) (\\d+):(\\d+):(\\d+).(\\d+)");
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(OracleChangeRecordValueConverter.class);
|
||||||
|
|
||||||
|
private static final DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder()
|
||||||
|
.parseCaseInsensitive()
|
||||||
|
.appendPattern("yyyy-MM-dd")
|
||||||
|
.toFormatter();
|
||||||
|
private static final DateTimeFormatter TIMESTAMP_FORMATTER = new DateTimeFormatterBuilder()
|
||||||
|
.parseCaseInsensitive()
|
||||||
|
.appendPattern("yyyy-MM-dd HH:mm:ss")
|
||||||
|
.optionalStart()
|
||||||
|
.appendPattern(".")
|
||||||
|
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, false)
|
||||||
|
.optionalEnd()
|
||||||
|
.toFormatter();
|
||||||
|
private static final DateTimeFormatter TIMESTAMP_TZ_FORMATTER = new DateTimeFormatterBuilder()
|
||||||
|
.parseCaseInsensitive()
|
||||||
|
.appendPattern("yyyy-MM-dd HH:mm:ss")
|
||||||
|
.optionalStart()
|
||||||
|
.appendPattern(".")
|
||||||
|
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, false)
|
||||||
|
.optionalEnd()
|
||||||
|
.appendPattern(" XXX")
|
||||||
|
.toFormatter();
|
||||||
|
|
||||||
|
private final JdbcConnection connection;
|
||||||
|
|
||||||
|
public OracleChangeRecordValueConverter(JdbcConnection connection) {
|
||||||
|
this.connection = connection;
|
||||||
|
}
|
||||||
|
|
||||||
|
// todo some methods are just a clones from OracleValueConverters, remove or implement differently
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public SchemaBuilder schemaBuilder(Column column) {
|
||||||
|
if (column == null) { // todo: we will address it if happens
|
||||||
|
LOGGER.warn("Column is null, investigate");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
LOGGER.trace("Building schema for column {} of type {} named {} with constraints ({},{})",
|
||||||
|
column.name(),
|
||||||
|
column.jdbcType(),
|
||||||
|
column.typeName(),
|
||||||
|
column.length(),
|
||||||
|
column.scale());
|
||||||
|
|
||||||
|
switch (column.jdbcType()) {
|
||||||
|
// Oracle's float is not float as in Java but a NUMERIC without scale
|
||||||
|
// case Types.FLOAT:
|
||||||
|
// return VariableScaleDecimal.builder();
|
||||||
|
case Types.NUMERIC:
|
||||||
|
return getNumericSchema(column);
|
||||||
|
case OracleTypes.BINARY_FLOAT:
|
||||||
|
return SchemaBuilder.float32();
|
||||||
|
case OracleTypes.BINARY_DOUBLE:
|
||||||
|
return SchemaBuilder.float64();
|
||||||
|
// case OracleTypes.TIMESTAMP:
|
||||||
|
// return io.debezium.time.Timestamp.builder();
|
||||||
|
case OracleTypes.TIMESTAMPTZ:
|
||||||
|
case OracleTypes.TIMESTAMPLTZ:
|
||||||
|
return ZonedTimestamp.builder();
|
||||||
|
case OracleTypes.INTERVALYM:
|
||||||
|
case OracleTypes.INTERVALDS:
|
||||||
|
return MicroDuration.builder();
|
||||||
|
case OracleTypes.STRUCT:
|
||||||
|
return SchemaBuilder.string();
|
||||||
|
default:
|
||||||
|
return super.schemaBuilder(column);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private SchemaBuilder getNumericSchema(Column column) {
|
||||||
|
if (column.scale().isPresent()) {
|
||||||
|
// return sufficiently sized int schema for non-floating point types
|
||||||
|
Integer scale = column.scale().get();
|
||||||
|
|
||||||
|
// a negative scale means rounding, e.g. NUMBER(10, -2) would be rounded to hundreds
|
||||||
|
if (scale <= 0) {
|
||||||
|
// Boolean represtented as Number(1,0)
|
||||||
|
if (scale == 0 && column.length() == 1) {
|
||||||
|
return SchemaBuilder.bool();
|
||||||
|
}
|
||||||
|
|
||||||
|
int width = column.length() - scale;
|
||||||
|
|
||||||
|
if (width < 3) {
|
||||||
|
return SchemaBuilder.int8();
|
||||||
|
}
|
||||||
|
else if (width < 5) {
|
||||||
|
return SchemaBuilder.int16();
|
||||||
|
}
|
||||||
|
else if (width < 10 || (width == 10 && scale == 0)) {
|
||||||
|
return SchemaBuilder.int32();
|
||||||
|
}
|
||||||
|
else if (width < 19) {
|
||||||
|
return SchemaBuilder.int64();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// larger non-floating point types and floating point types use Decimal
|
||||||
|
return super.schemaBuilder(column);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
return VariableScaleDecimal.builder();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ValueConverter converter(Column column, Field fieldDefn) {
|
||||||
|
switch (column.jdbcType()) {
|
||||||
|
case Types.CHAR:
|
||||||
|
case Types.VARCHAR:
|
||||||
|
case Types.NCHAR:
|
||||||
|
case Types.NVARCHAR:
|
||||||
|
case OracleTypes.STRUCT:
|
||||||
|
case Types.CLOB:
|
||||||
|
return data -> convertString(column, fieldDefn, data);
|
||||||
|
case Types.BLOB:
|
||||||
|
return data -> convertBinary(column, fieldDefn, data, binaryMode);
|
||||||
|
case OracleTypes.BINARY_FLOAT:
|
||||||
|
return data -> convertFloat(column, fieldDefn, data);
|
||||||
|
case OracleTypes.BINARY_DOUBLE:
|
||||||
|
return data -> convertDouble(column, fieldDefn, data);
|
||||||
|
case Types.NUMERIC:
|
||||||
|
return getNumericConverter(column, fieldDefn);
|
||||||
|
case Types.FLOAT:
|
||||||
|
return data -> getFloatConverter(column, fieldDefn, data);
|
||||||
|
case OracleTypes.TIMESTAMP:
|
||||||
|
return data -> convertToLocalDateTime(column, fieldDefn, data);
|
||||||
|
case OracleTypes.TIMESTAMPTZ:
|
||||||
|
case OracleTypes.TIMESTAMPLTZ:
|
||||||
|
return (data) -> convertTimestampWithZone(column, fieldDefn, data);
|
||||||
|
case OracleTypes.INTERVALYM:
|
||||||
|
return (data) -> convertIntervalYearMonth(column, fieldDefn, data);
|
||||||
|
case OracleTypes.INTERVALDS:
|
||||||
|
return (data) -> convertIntervalDaySecond(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
return super.converter(column, fieldDefn);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Converts a string object for an object type of {@link LocalDateTime}.
|
||||||
|
* If the column definition allows null and default value is 0000-00-00 00:00:00, we need return null,
|
||||||
|
* else 0000-00-00 00:00:00 will be replaced with 1970-01-01 00:00:00;
|
||||||
|
*
|
||||||
|
* @param column the column definition describing the {@code data} value; never null
|
||||||
|
* @param fieldDefn field definition
|
||||||
|
* @param value the string object to be converted into a {@link LocalDateTime} type;
|
||||||
|
* @return the converted value;
|
||||||
|
*/
|
||||||
|
private Object convertToLocalDateTime(Column column, Field fieldDefn, Object value) {
|
||||||
|
|
||||||
|
// todo make it better
|
||||||
|
String dateText;
|
||||||
|
if (value instanceof String) {
|
||||||
|
String valueString = (String) value;
|
||||||
|
if (valueString.toLowerCase().startsWith("to_timestamp")) {
|
||||||
|
dateText = valueString.substring("to_timestamp".length() + 2, valueString.length() - 2);
|
||||||
|
LocalDateTime dateTime = LocalDateTime.from(TIMESTAMP_FORMATTER.parse(dateText.trim()));
|
||||||
|
// todo: DBZ-137 this originally converted to ZoneId.systemDefault() but this should be GMT
|
||||||
|
// this is so that this behavior is compatible with the Xstreams implementation
|
||||||
|
return dateTime.atZone(ZoneId.of("GMT")).toInstant().toEpochMilli() * 1000; // timestamp(6) is converted as microTimestamp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private ValueConverter getNumericConverter(Column column, Field fieldDefn) {
|
||||||
|
if (column.scale().isPresent()) {
|
||||||
|
Integer scale = column.scale().get();
|
||||||
|
|
||||||
|
if (scale <= 0) {
|
||||||
|
// Boolean represtented as Number(1,0)
|
||||||
|
if (scale == 0 && column.length() == 1) {
|
||||||
|
return data -> convertBoolean(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
int width = column.length() - scale;
|
||||||
|
|
||||||
|
if (width < 3) {
|
||||||
|
return data -> convertNumericAsTinyInt(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
else if (width < 5) {
|
||||||
|
return data -> convertNumericAsSmallInt(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
else if (width < 10 || (width == 10 && scale == 0)) {
|
||||||
|
return data -> convertNumericAsInteger(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
else if (width < 19) {
|
||||||
|
return data -> convertNumericAsBigInteger(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// larger non-floating point types and floating point types use Decimal
|
||||||
|
return data -> convertNumeric(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
return data -> convertVariableScale(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object getFloatConverter(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof String) {
|
||||||
|
return Float.parseFloat((String) data);
|
||||||
|
}
|
||||||
|
return convertVariableScale(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertString(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof CHAR) {
|
||||||
|
return ((CHAR) data).stringValue();
|
||||||
|
}
|
||||||
|
if (data instanceof CLOB) {
|
||||||
|
return ((CLOB) data).toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
return super.convertString(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertBinary(Column column, Field fieldDefn, Object data, BinaryHandlingMode mode) {
|
||||||
|
if (data instanceof BLOB) {
|
||||||
|
return ((BLOB) data).getBytes();
|
||||||
|
}
|
||||||
|
return super.convertBinary(column, fieldDefn, data, mode);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertInteger(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof NUMBER) {
|
||||||
|
try {
|
||||||
|
data = ((NUMBER) data).intValue();
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return super.convertInteger(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertFloat(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof Float) {
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
else if (data instanceof NUMBER) {
|
||||||
|
return ((NUMBER) data).floatValue();
|
||||||
|
}
|
||||||
|
else if (data instanceof BINARY_FLOAT) {
|
||||||
|
try {
|
||||||
|
return ((BINARY_FLOAT) data).floatValue();
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return super.convertFloat(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertDouble(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof BINARY_DOUBLE) {
|
||||||
|
try {
|
||||||
|
return ((BINARY_DOUBLE) data).doubleValue();
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return super.convertDouble(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertDecimal(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof NUMBER) {
|
||||||
|
try {
|
||||||
|
data = ((NUMBER) data).bigDecimalValue();
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// adjust scale to column's scale if the column's scale is larger than the one from
|
||||||
|
// the value (e.g. 4.4444 -> 4.444400)
|
||||||
|
if (data instanceof BigDecimal) {
|
||||||
|
data = withScaleAdjustedIfNeeded(column, (BigDecimal) data);
|
||||||
|
}
|
||||||
|
|
||||||
|
return super.convertDecimal(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected BigDecimal withScaleAdjustedIfNeeded(Column column, BigDecimal data) {
|
||||||
|
if (column.scale().isPresent() && column.scale().get() > data.scale()) {
|
||||||
|
data = data.setScale(column.scale().get());
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertNumeric(Column column, Field fieldDefn, Object data) {
|
||||||
|
return convertDecimal(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object convertNumericAsTinyInt(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof NUMBER) {
|
||||||
|
try {
|
||||||
|
data = ((NUMBER) data).byteValue();
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return convertTinyInt(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object convertNumericAsSmallInt(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof NUMBER) {
|
||||||
|
try {
|
||||||
|
data = ((NUMBER) data).shortValue();
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return super.convertSmallInt(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object convertNumericAsInteger(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof NUMBER) {
|
||||||
|
try {
|
||||||
|
data = ((NUMBER) data).intValue();
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return super.convertInteger(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object convertNumericAsBigInteger(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof NUMBER) {
|
||||||
|
try {
|
||||||
|
data = ((NUMBER) data).longValue();
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return super.convertBigInt(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Converts a value object for an expected JDBC type of {@link Types#BOOLEAN}.
|
||||||
|
*
|
||||||
|
* @param column the column definition describing the {@code data} value; never null
|
||||||
|
* @param fieldDefn the field definition; never null
|
||||||
|
* @param data the data object to be converted into a {@link Date Kafka Connect date} type; never null
|
||||||
|
* @return the converted value, or null if the conversion could not be made and the column allows nulls
|
||||||
|
* @throws IllegalArgumentException if the value could not be converted but the column does not allow nulls
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
protected Object convertBoolean(Column column, Field fieldDefn, Object data) {
|
||||||
|
if (data instanceof String) {
|
||||||
|
return Byte.parseByte((String) data) == 0 ? Boolean.FALSE : Boolean.TRUE;
|
||||||
|
}
|
||||||
|
return super.convertBoolean(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertTinyInt(Column column, Field fieldDefn, Object data) {
|
||||||
|
return convertValue(column, fieldDefn, data, BYTE_FALSE, (r) -> {
|
||||||
|
if (data instanceof Byte) {
|
||||||
|
r.deliver(data);
|
||||||
|
}
|
||||||
|
else if (data instanceof Number) {
|
||||||
|
Number value = (Number) data;
|
||||||
|
r.deliver(value.byteValue());
|
||||||
|
}
|
||||||
|
else if (data instanceof Boolean) {
|
||||||
|
r.deliver(NumberConversions.getByte((boolean) data));
|
||||||
|
}
|
||||||
|
else if (data instanceof String) {
|
||||||
|
r.deliver(Byte.parseByte((String) data));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object convertVariableScale(Column column, Field fieldDefn, Object data) {
|
||||||
|
data = convertNumeric(column, fieldDefn, data); // provides default value
|
||||||
|
|
||||||
|
if (data == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
// TODO Need to handle special values, it is not supported in variable scale decimal
|
||||||
|
else if (data instanceof SpecialValueDecimal) {
|
||||||
|
return VariableScaleDecimal.fromLogical(fieldDefn.schema(), (SpecialValueDecimal) data);
|
||||||
|
}
|
||||||
|
else if (data instanceof BigDecimal) {
|
||||||
|
return VariableScaleDecimal.fromLogical(fieldDefn.schema(), new SpecialValueDecimal((BigDecimal) data));
|
||||||
|
}
|
||||||
|
return handleUnknownData(column, fieldDefn, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object fromOracleTimeClasses(Column column, Object data) {
|
||||||
|
try {
|
||||||
|
if (data instanceof TIMESTAMP) {
|
||||||
|
data = ((TIMESTAMP) data).timestampValue();
|
||||||
|
}
|
||||||
|
else if (data instanceof DATE) {
|
||||||
|
data = ((DATE) data).timestampValue();
|
||||||
|
}
|
||||||
|
else if (data instanceof TIMESTAMPTZ) {
|
||||||
|
final TIMESTAMPTZ ts = (TIMESTAMPTZ) data;
|
||||||
|
data = ZonedDateTime.ofInstant(ts.timestampValue(connection.connection()).toInstant(), ts.getTimeZone().toZoneId());
|
||||||
|
}
|
||||||
|
else if (data instanceof TIMESTAMPLTZ) {
|
||||||
|
// JDBC driver throws an exception
|
||||||
|
// final TIMESTAMPLTZ ts = (TIMESTAMPLTZ)data;
|
||||||
|
// data = ts.offsetDateTimeValue(connection.connection());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertTimestampToEpochMicros(Column column, Field fieldDefn, Object data) {
|
||||||
|
return super.convertTimestampToEpochMicros(column, fieldDefn, fromOracleTimeClasses(column, data));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertTimestampToEpochMillis(Column column, Field fieldDefn, Object data) {
|
||||||
|
return super.convertTimestampToEpochMillis(column, fieldDefn, fromOracleTimeClasses(column, data));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertTimestampToEpochNanos(Column column, Field fieldDefn, Object data) {
|
||||||
|
return super.convertTimestampToEpochNanos(column, fieldDefn, fromOracleTimeClasses(column, data));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Object convertTimestampWithZone(Column column, Field fieldDefn, Object data) {
|
||||||
|
return super.convertTimestampWithZone(column, fieldDefn, fromOracleTimeClasses(column, data));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object convertIntervalYearMonth(Column column, Field fieldDefn, Object data) {
|
||||||
|
return convertValue(column, fieldDefn, data, NumberConversions.DOUBLE_FALSE, (r) -> {
|
||||||
|
if (data instanceof Number) {
|
||||||
|
// we expect to get back from the plugin a double value
|
||||||
|
r.deliver(((Number) data).doubleValue());
|
||||||
|
}
|
||||||
|
else if (data instanceof INTERVALYM) {
|
||||||
|
final String interval = ((INTERVALYM) data).stringValue();
|
||||||
|
int sign = 1;
|
||||||
|
int start = 0;
|
||||||
|
if (interval.charAt(0) == '-') {
|
||||||
|
sign = -1;
|
||||||
|
start = 1;
|
||||||
|
}
|
||||||
|
for (int i = 1; i < interval.length(); i++) {
|
||||||
|
if (interval.charAt(i) == '-') {
|
||||||
|
final int year = sign * Integer.parseInt(interval.substring(start, i));
|
||||||
|
final int month = sign * Integer.parseInt(interval.substring(i + 1));
|
||||||
|
r.deliver(MicroDuration.durationMicros(year, month, 0, 0,
|
||||||
|
0, 0, MicroDuration.DAYS_PER_MONTH_AVG));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object convertIntervalDaySecond(Column column, Field fieldDefn, Object data) {
|
||||||
|
return convertValue(column, fieldDefn, data, NumberConversions.DOUBLE_FALSE, (r) -> {
|
||||||
|
if (data instanceof Number) {
|
||||||
|
// we expect to get back from the plugin a double value
|
||||||
|
r.deliver(((Number) data).doubleValue());
|
||||||
|
}
|
||||||
|
else if (data instanceof INTERVALDS) {
|
||||||
|
final String interval = ((INTERVALDS) data).stringValue();
|
||||||
|
final Matcher m = INTERVAL_DAY_SECOND_PATTERN.matcher(interval);
|
||||||
|
if (m.matches()) {
|
||||||
|
final int sign = "-".equals(m.group(1)) ? -1 : 1;
|
||||||
|
r.deliver(MicroDuration.durationMicros(
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
sign * Integer.valueOf(m.group(2)),
|
||||||
|
sign * Integer.valueOf(m.group(3)),
|
||||||
|
sign * Integer.valueOf(m.group(4)),
|
||||||
|
sign * Integer.valueOf(m.group(5)),
|
||||||
|
sign * Integer.valueOf(Strings.pad(m.group(6), 6, '0')),
|
||||||
|
MicroDuration.DAYS_PER_MONTH_AVG));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,175 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.Instant;
|
||||||
|
|
||||||
|
import javax.xml.bind.DatatypeConverter;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A utility class to map LogMiner content resultSet values.
|
||||||
|
* This class gracefully logs errors, loosing an entry is not critical.
|
||||||
|
* The loss will be logged
|
||||||
|
*/
|
||||||
|
public class RowMapper {
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(RowMapper.class);
|
||||||
|
|
||||||
|
// operations
|
||||||
|
public static final int INSERT = 1;
|
||||||
|
public static final int DELETE = 2;
|
||||||
|
public static final int UPDATE = 3;
|
||||||
|
public static final int DDL = 5;
|
||||||
|
public static final int COMMIT = 7;
|
||||||
|
public static final int MISSING_SCN = 34;
|
||||||
|
public static final int ROLLBACK = 36;
|
||||||
|
|
||||||
|
private static final int SCN = 1;
|
||||||
|
private static final int SQL_REDO = 2;
|
||||||
|
private static final int OPERATION_CODE = 3;
|
||||||
|
private static final int CHANGE_TIME = 4;
|
||||||
|
private static final int TX_ID = 5;
|
||||||
|
private static final int CSF = 6;
|
||||||
|
private static final int TABLE_NAME = 7;
|
||||||
|
private static final int SEG_OWNER = 8;
|
||||||
|
private static final int OPERATION = 9;
|
||||||
|
private static final int USERNAME = 10;
|
||||||
|
|
||||||
|
public static String getOperation(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||||
|
try {
|
||||||
|
return rs.getString(OPERATION);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
logError(metrics, e, "OPERATION");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getUsername(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||||
|
try {
|
||||||
|
return rs.getString(USERNAME);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
logError(metrics, e, "USERNAME");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int getOperationCode(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||||
|
try {
|
||||||
|
return rs.getInt(OPERATION_CODE);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
logError(metrics, e, "OPERATION_CODE");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getTableName(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||||
|
try {
|
||||||
|
return rs.getString(TABLE_NAME);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
logError(metrics, e, "TABLE_NAME");
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getSegOwner(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||||
|
try {
|
||||||
|
return rs.getString(SEG_OWNER);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
logError(metrics, e, "SEG_OWNER");
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Timestamp getChangeTime(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||||
|
try {
|
||||||
|
return rs.getTimestamp(CHANGE_TIME);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
logError(metrics, e, "CHANGE_TIME");
|
||||||
|
return new Timestamp(Instant.now().getEpochSecond());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static BigDecimal getScn(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||||
|
try {
|
||||||
|
return rs.getBigDecimal(SCN);
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
logError(metrics, e, "SCN");
|
||||||
|
return new BigDecimal(-1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getTransactionId(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||||
|
try {
|
||||||
|
return DatatypeConverter.printHexBinary(rs.getBytes(TX_ID));
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
logError(metrics, e, "TX_ID");
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* It constructs REDO_SQL. If REDO_SQL is in a few lines, it truncates after first 40_000 characters
|
||||||
|
* @param metrics metrics
|
||||||
|
* @param rs result set
|
||||||
|
* @return REDO_SQL
|
||||||
|
*/
|
||||||
|
public static String getSqlRedo(TransactionalBufferMetrics metrics, ResultSet rs) {
|
||||||
|
int lobLimitCounter = 9; // todo : decide on approach ( XStream chunk option) and Lob limit
|
||||||
|
StringBuilder result = new StringBuilder(4000);
|
||||||
|
try {
|
||||||
|
String redo_sql = rs.getString(SQL_REDO);
|
||||||
|
if (redo_sql == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
result = new StringBuilder(redo_sql);
|
||||||
|
|
||||||
|
int csf = rs.getInt(CSF);
|
||||||
|
// 0 - indicates SQL_REDO is contained within the same row
|
||||||
|
// 1 - indicates that either SQL_REDO is greater than 4000 bytes in size and is continued in
|
||||||
|
// the next row returned by the ResultSet
|
||||||
|
while (csf == 1) {
|
||||||
|
rs.next();
|
||||||
|
if (lobLimitCounter-- == 0) {
|
||||||
|
LOGGER.warn("LOB value was truncated due to the connector limitation of {} MB", 40);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
result.append(rs.getString(SQL_REDO));
|
||||||
|
csf = rs.getInt(CSF);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
logError(metrics, e, "SQL_REDO");
|
||||||
|
}
|
||||||
|
return result.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void logError(TransactionalBufferMetrics metrics, SQLException e, String s) {
|
||||||
|
LogMinerHelper.logError(metrics, "Cannot get {}. This entry from log miner will be lost due to the {}", s, e);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static TableId getTableId(String catalogName, ResultSet rs) throws SQLException {
|
||||||
|
return new TableId(catalogName.toUpperCase(), rs.getString(SEG_OWNER).toUpperCase(), rs.getString(TABLE_NAME).toUpperCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,134 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.StringJoiner;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||||
|
import io.debezium.connector.oracle.OracleDatabaseSchema;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This utility class contains SQL statements to configure, manage and query Oracle LogMiner
|
||||||
|
*/
|
||||||
|
class SqlUtils {
|
||||||
|
|
||||||
|
private static final String LOGMNR_CONTENTS_VIEW = "V$LOGMNR_CONTENTS";
|
||||||
|
private static final String LOGMNR_AUDIT_TABLE = "LOG_MINING_AUDIT";
|
||||||
|
|
||||||
|
static final String BUILD_DICTIONARY = "BEGIN DBMS_LOGMNR_D.BUILD (options => DBMS_LOGMNR_D.STORE_IN_REDO_LOGS); END;";
|
||||||
|
static final String CURRENT_SCN = "SELECT CURRENT_SCN FROM V$DATABASE";
|
||||||
|
static final String CURRENT_TIMESTAMP = "select current_timestamp from dual";
|
||||||
|
static final String END_LOGMNR = "BEGIN SYS.DBMS_LOGMNR.END_LOGMNR(); END;";
|
||||||
|
static final String OLDEST_FIRST_CHANGE = "SELECT MIN(FIRST_CHANGE#) FROM V$LOG";
|
||||||
|
static final String ALL_ONLINE_LOGS = "SELECT MIN(F.MEMBER) AS FILE_NAME, L.NEXT_CHANGE# AS NEXT_CHANGE, F.GROUP# " +
|
||||||
|
" FROM V$LOG L, V$LOGFILE F " +
|
||||||
|
" WHERE F.GROUP# = L.GROUP# AND L.NEXT_CHANGE# > 0 " +
|
||||||
|
" GROUP BY F.GROUP#, L.NEXT_CHANGE# ORDER BY 3";
|
||||||
|
|
||||||
|
static final String REDO_LOGS_STATUS = "SELECT F.MEMBER, R.STATUS FROM V$LOGFILE F, V$LOG R WHERE F.GROUP# = R.GROUP# ORDER BY 2";
|
||||||
|
static final String SWITCH_HISTORY_TOTAL_COUNT = "select 'total', count(1) from v$archived_log where first_time > trunc(sysdate)" +
|
||||||
|
" and dest_id = (select dest_id from V$ARCHIVE_DEST_STATUS where status='VALID' and type='LOCAL')";
|
||||||
|
static final String CURRENT_REDO_LOG_NAME = "select f.member from v$log log, v$logfile f where log.group#=f.group# and log.status='CURRENT'";
|
||||||
|
static final String AUDIT_TABLE_EXISTS = "SELECT '1' AS ONE FROM USER_TABLES WHERE TABLE_NAME = '" + LOGMNR_AUDIT_TABLE + "'";
|
||||||
|
static final String AUDIT_TABLE_RECORD_EXISTS = "SELECT '1' AS ONE FROM " + LOGMNR_AUDIT_TABLE;
|
||||||
|
static final String CREATE_AUDIT_TABLE = "CREATE TABLE " + LOGMNR_AUDIT_TABLE + "(LAST_SCN NUMBER(19,0))";
|
||||||
|
static final String INSERT_AUDIT_TABLE = "INSERT INTO " + LOGMNR_AUDIT_TABLE + " VALUES(0)";
|
||||||
|
static final String UPDATE_AUDIT_TABLE = "UPDATE " + LOGMNR_AUDIT_TABLE + " SET LAST_SCN =";
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(SqlUtils.class);
|
||||||
|
|
||||||
|
// todo handle INVALID file member (report somehow and continue to work with valid file), handle adding multiplexed files,
|
||||||
|
// todo SELECT name, value FROM v$sysstat WHERE name = 'redo wastage';
|
||||||
|
// todo SELECT GROUP#, STATUS, MEMBER FROM V$LOGFILE WHERE STATUS='INVALID'; (drop and recreate? or do it manually?)
|
||||||
|
// todo SELECT BLOCKSIZE FROM V$LOG;
|
||||||
|
|
||||||
|
static final String NLS_SESSION_PARAMETERS = "ALTER SESSION SET "
|
||||||
|
+ " NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS'"
|
||||||
|
+ " NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF'"
|
||||||
|
+ " NLS_TIMESTAMP_TZ_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM'"
|
||||||
|
+ " NLS_NUMERIC_CHARACTERS = '.,'";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This returns statement to build log miner view for online redo log files
|
||||||
|
* @param startScn mine from
|
||||||
|
* @param endScn mine till
|
||||||
|
* @param strategy Log Mining strategy
|
||||||
|
* @return statement todo: handle corruption. STATUS (Double) — value of 0 indicates it is executable
|
||||||
|
*/
|
||||||
|
static String getStartLogMinerStatement(Long startScn, Long endScn, OracleConnectorConfig.LogMiningStrategy strategy, boolean isContinuousMining) {
|
||||||
|
String miningStrategy;
|
||||||
|
if (strategy.equals(OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO)) {
|
||||||
|
miningStrategy = "DBMS_LOGMNR.DICT_FROM_REDO_LOGS + DBMS_LOGMNR.DDL_DICT_TRACKING ";
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
miningStrategy = "DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG ";
|
||||||
|
}
|
||||||
|
if (isContinuousMining) {
|
||||||
|
miningStrategy += " + DBMS_LOGMNR.CONTINUOUS_MINE ";
|
||||||
|
}
|
||||||
|
return "BEGIN sys.dbms_logmnr.start_logmnr(" +
|
||||||
|
"startScn => '" + startScn + "', " +
|
||||||
|
"endScn => '" + endScn + "', " +
|
||||||
|
"OPTIONS => " + miningStrategy +
|
||||||
|
" + DBMS_LOGMNR.NO_ROWID_IN_STMT);" +
|
||||||
|
"END;";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is the query from the log miner view to get changes. Columns of the view we using are:
|
||||||
|
* NOTE. Currently we do not capture changes from other schemas
|
||||||
|
* SCN - The SCN at which a change was made
|
||||||
|
* COMMIT_SCN - The SCN at which a change was committed
|
||||||
|
* USERNAME - Name of the user who executed the transaction
|
||||||
|
* SQL_REDO Reconstructed SQL statement that is equivalent to the original SQL statement that made the change
|
||||||
|
* OPERATION_CODE - Number of the operation code.
|
||||||
|
* TABLE_NAME - Name of the modified table
|
||||||
|
* TIMESTAMP - Timestamp when the database change was made
|
||||||
|
*
|
||||||
|
* @param schemaName user name
|
||||||
|
* @param logMinerUser log mining session user name
|
||||||
|
* @param schema schema
|
||||||
|
* @return the query
|
||||||
|
*/
|
||||||
|
static String queryLogMinerContents(String schemaName, String logMinerUser, OracleDatabaseSchema schema) {
|
||||||
|
List<String> whiteListTableNames = schema.tableIds().stream().map(TableId::table).collect(Collectors.toList());
|
||||||
|
|
||||||
|
String sorting = "ORDER BY SCN";
|
||||||
|
return "SELECT SCN, SQL_REDO, OPERATION_CODE, TIMESTAMP, XID, CSF, TABLE_NAME, SEG_OWNER, OPERATION, USERNAME " +
|
||||||
|
" FROM " + LOGMNR_CONTENTS_VIEW + " WHERE OPERATION_CODE in (1,2,3,5) " + // 5 - DDL
|
||||||
|
" AND SEG_OWNER = '" + schemaName.toUpperCase() + "' " +
|
||||||
|
buildTableInPredicate(whiteListTableNames) +
|
||||||
|
" AND SCN >= ? AND SCN < ? " +
|
||||||
|
" OR (OPERATION_CODE IN (5,7,34,36) AND USERNAME NOT IN ('SYS','SYSTEM','" + logMinerUser.toUpperCase() + "'))" + sorting; // todo username = schemaName?
|
||||||
|
}
|
||||||
|
|
||||||
|
static String getAddLogFileStatement(String option, String fileName) {
|
||||||
|
return "BEGIN sys.dbms_logmnr.add_logfile(LOGFILENAME => '" + fileName + "', OPTIONS => " + option + ");END;";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method builds table_name IN predicate, filtering out non whitelisted tables from Log Mining.
|
||||||
|
* It limits joining condition over 1000 tables, Oracle will throw exception in such predicate.
|
||||||
|
* @param tables white listed table names
|
||||||
|
* @return IN predicate or empty string if number of whitelisted tables exceeds 1000
|
||||||
|
*/
|
||||||
|
private static String buildTableInPredicate(List<String> tables) {
|
||||||
|
if (tables.size() == 0 || tables.size() > 1000) {
|
||||||
|
LOGGER.warn(" Cannot apply {} whitelisted tables condition", tables.size());
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
StringJoiner tableNames = new StringJoiner(",");
|
||||||
|
tables.forEach(table -> tableNames.add("'" + table + "'"));
|
||||||
|
return " AND table_name IN (" + tableNames + ") AND SEG_NAME IN (" + tableNames + ") ";
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,399 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.ArrayBlockingQueue;
|
||||||
|
import java.util.concurrent.BlockingQueue;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.ThreadPoolExecutor;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
|
import org.apache.kafka.connect.errors.DataException;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.annotation.NotThreadSafe;
|
||||||
|
import io.debezium.connector.oracle.OracleConnector;
|
||||||
|
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||||
|
import io.debezium.pipeline.ErrorHandler;
|
||||||
|
import io.debezium.pipeline.source.spi.ChangeEventSource;
|
||||||
|
import io.debezium.util.Threads;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Andrey Pustovetov
|
||||||
|
* <p>
|
||||||
|
* Transactional buffer is designed to register callbacks, to execute them when transaction commits and to clear them
|
||||||
|
* when transaction rollbacks.
|
||||||
|
*/
|
||||||
|
@NotThreadSafe
|
||||||
|
public final class TransactionalBuffer {
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(TransactionalBuffer.class);
|
||||||
|
|
||||||
|
private final Map<String, Transaction> transactions;
|
||||||
|
private final ExecutorService executor;
|
||||||
|
private final AtomicInteger taskCounter;
|
||||||
|
private final ErrorHandler errorHandler;
|
||||||
|
private final Supplier<Integer> commitQueueCapacity;
|
||||||
|
private TransactionalBufferMetrics metrics;
|
||||||
|
private final Set<String> abandonedTransactionIds;
|
||||||
|
|
||||||
|
// storing rolledBackTransactionIds is for debugging purposes to check what was rolled back to research, todo delete in future releases
|
||||||
|
private final Set<String> rolledBackTransactionIds;
|
||||||
|
|
||||||
|
// It holds the latest captured SCN.
|
||||||
|
// This number tracks starting point for the next mining cycle.
|
||||||
|
private BigDecimal largestScn;
|
||||||
|
private BigDecimal lastCommittedScn;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor to create a new instance.
|
||||||
|
*
|
||||||
|
* @param logicalName logical name
|
||||||
|
* @param errorHandler logError handler
|
||||||
|
* @param metrics metrics MBean
|
||||||
|
* @param inCommitQueueCapacity commit queue capacity. On overflow, caller runs task
|
||||||
|
*/
|
||||||
|
TransactionalBuffer(String logicalName, ErrorHandler errorHandler, TransactionalBufferMetrics metrics, int inCommitQueueCapacity) {
|
||||||
|
this.transactions = new HashMap<>();
|
||||||
|
final BlockingQueue<Runnable> workQueue = new ArrayBlockingQueue<>(inCommitQueueCapacity);
|
||||||
|
executor = new ThreadPoolExecutor(1, 1,
|
||||||
|
Integer.MAX_VALUE, TimeUnit.MILLISECONDS,
|
||||||
|
workQueue,
|
||||||
|
Threads.threadFactory(OracleConnector.class, logicalName, "transactional-buffer", true, false),
|
||||||
|
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||||
|
commitQueueCapacity = workQueue::remainingCapacity;
|
||||||
|
this.taskCounter = new AtomicInteger();
|
||||||
|
this.errorHandler = errorHandler;
|
||||||
|
this.metrics = metrics;
|
||||||
|
largestScn = BigDecimal.ZERO;
|
||||||
|
lastCommittedScn = BigDecimal.ZERO;
|
||||||
|
this.abandonedTransactionIds = new HashSet<>();
|
||||||
|
this.rolledBackTransactionIds = new HashSet<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return largest last SCN in the buffer among all transactions
|
||||||
|
*/
|
||||||
|
BigDecimal getLargestScn() {
|
||||||
|
return largestScn;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return rolled back transactions
|
||||||
|
*/
|
||||||
|
Set<String> getRolledBackTransactionIds() {
|
||||||
|
return new HashSet<>(rolledBackTransactionIds);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset Largest SCN
|
||||||
|
*/
|
||||||
|
void resetLargestScn(Long value) {
|
||||||
|
if (value != null) {
|
||||||
|
largestScn = new BigDecimal(value);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
largestScn = BigDecimal.ZERO;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Registers callback to execute when transaction commits.
|
||||||
|
*
|
||||||
|
* @param transactionId transaction identifier
|
||||||
|
* @param scn SCN
|
||||||
|
* @param changeTime time of DML parsing completion
|
||||||
|
* @param redoSql statement from redo
|
||||||
|
* @param callback callback to execute when transaction commits
|
||||||
|
*/
|
||||||
|
void registerCommitCallback(String transactionId, BigDecimal scn, Instant changeTime, String redoSql, CommitCallback callback) {
|
||||||
|
if (abandonedTransactionIds.contains(transactionId)) {
|
||||||
|
LogMinerHelper.logWarn(metrics, "Another DML for an abandoned transaction {} : {}, ignored", transactionId, redoSql);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
transactions.computeIfAbsent(transactionId, s -> new Transaction(scn));
|
||||||
|
|
||||||
|
metrics.setActiveTransactions(transactions.size());
|
||||||
|
metrics.incrementCapturedDmlCounter();
|
||||||
|
metrics.calculateLagMetrics(changeTime);
|
||||||
|
|
||||||
|
// The transaction object is not a lightweight object anymore having all REDO_SQL stored.
|
||||||
|
Transaction transaction = transactions.get(transactionId);
|
||||||
|
if (transaction != null) {
|
||||||
|
|
||||||
|
// todo this should never happen, delete when tested and confirmed
|
||||||
|
if (rolledBackTransactionIds.contains(transactionId)) {
|
||||||
|
LogMinerHelper.logWarn(metrics, "Ignore DML for rolled back transaction: SCN={}, REDO_SQL={}", scn, redoSql);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
transaction.commitCallbacks.add(callback);
|
||||||
|
transaction.addRedoSql(scn, redoSql);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (scn.compareTo(largestScn) > 0) {
|
||||||
|
largestScn = scn;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If the commit executor queue is full, back-pressure will be applied by letting execution of the callback
|
||||||
|
* be performed by the calling thread.
|
||||||
|
*
|
||||||
|
* @param transactionId transaction identifier
|
||||||
|
* @param scn SCN of the commit.
|
||||||
|
* @param offsetContext Oracle offset
|
||||||
|
* @param timestamp commit timestamp
|
||||||
|
* @param context context to check that source is running
|
||||||
|
* @param debugMessage message
|
||||||
|
* @return true if committed transaction is in the buffer, was not processed yet and processed now
|
||||||
|
*/
|
||||||
|
boolean commit(String transactionId, BigDecimal scn, OracleOffsetContext offsetContext, Timestamp timestamp,
|
||||||
|
ChangeEventSource.ChangeEventSourceContext context, String debugMessage) {
|
||||||
|
|
||||||
|
Transaction transaction = transactions.get(transactionId);
|
||||||
|
if (transaction == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
calculateLargestScn();
|
||||||
|
transaction = transactions.remove(transactionId);
|
||||||
|
BigDecimal smallestScn = calculateSmallestScn();
|
||||||
|
|
||||||
|
taskCounter.incrementAndGet();
|
||||||
|
abandonedTransactionIds.remove(transactionId);
|
||||||
|
|
||||||
|
// On the restarting connector, we start from SCN in the offset. There is possibility to commit a transaction(s) which were already committed.
|
||||||
|
// Currently we cannot use ">=", because we may lose normal commit which may happen at the same time. TODO use audit table to prevent duplications
|
||||||
|
if ((offsetContext.getCommitScn() != null && offsetContext.getCommitScn() > scn.longValue()) || lastCommittedScn.longValue() > scn.longValue()) {
|
||||||
|
LogMinerHelper.logWarn(metrics,
|
||||||
|
"Transaction {} was already processed, ignore. Committed SCN in offset is {}, commit SCN of the transaction is {}, last committed SCN is {}",
|
||||||
|
transactionId, offsetContext.getCommitScn(), scn, lastCommittedScn);
|
||||||
|
metrics.setActiveTransactions(transactions.size());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<CommitCallback> commitCallbacks = transaction.commitCallbacks;
|
||||||
|
LOGGER.trace("COMMIT, {}, smallest SCN: {}, largest SCN {}", debugMessage, smallestScn, largestScn);
|
||||||
|
executor.execute(() -> {
|
||||||
|
try {
|
||||||
|
int counter = commitCallbacks.size();
|
||||||
|
for (CommitCallback callback : commitCallbacks) {
|
||||||
|
if (!context.isRunning()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
callback.execute(timestamp, smallestScn, scn, --counter);
|
||||||
|
}
|
||||||
|
|
||||||
|
lastCommittedScn = new BigDecimal(scn.longValue());
|
||||||
|
}
|
||||||
|
catch (InterruptedException e) {
|
||||||
|
LogMinerHelper.logError(metrics, "Thread interrupted during running", e);
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
}
|
||||||
|
catch (Exception e) {
|
||||||
|
errorHandler.setProducerThrowable(e);
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
metrics.incrementCommittedTransactions();
|
||||||
|
metrics.setActiveTransactions(transactions.size());
|
||||||
|
metrics.incrementCommittedDmlCounter(commitCallbacks.size());
|
||||||
|
metrics.setCommittedScn(scn.longValue());
|
||||||
|
metrics.setCommitQueueCapacity(commitQueueCapacity.get());
|
||||||
|
taskCounter.decrementAndGet();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
metrics.setCommitQueueCapacity(commitQueueCapacity.get());
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clears registered callbacks for given transaction identifier.
|
||||||
|
*
|
||||||
|
* @param transactionId transaction id
|
||||||
|
* @param debugMessage message
|
||||||
|
* @return true if the rollback is for a transaction in the buffer
|
||||||
|
*/
|
||||||
|
boolean rollback(String transactionId, String debugMessage) {
|
||||||
|
|
||||||
|
Transaction transaction = transactions.get(transactionId);
|
||||||
|
if (transaction != null) {
|
||||||
|
LOGGER.debug("Transaction rolled back, {} , Statements: {}", debugMessage, transaction.redoSqlMap.values().toArray());
|
||||||
|
|
||||||
|
calculateLargestScn(); // in case if largest SCN was in this transaction
|
||||||
|
transactions.remove(transactionId);
|
||||||
|
|
||||||
|
abandonedTransactionIds.remove(transactionId);
|
||||||
|
rolledBackTransactionIds.add(transactionId);
|
||||||
|
|
||||||
|
metrics.setActiveTransactions(transactions.size());
|
||||||
|
metrics.incrementRolledBackTransactions();
|
||||||
|
metrics.addRolledBackTransactionId(transactionId); // todo decide if we need both metrics
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If for some reason the connector got restarted, the offset will point to the beginning of the oldest captured transaction.
|
||||||
|
* Taking in consideration offset flush interval, the offset could be even older.
|
||||||
|
* If that transaction was lasted for a long time, let say > 30 minutes, the offset will be not accessible after restart,
|
||||||
|
* because we don't mine archived logs, neither rely on continuous_mine configuration option.
|
||||||
|
* Hence we have to address these cases manually.
|
||||||
|
* <p>
|
||||||
|
* It is limited by following condition:
|
||||||
|
* allOnlineRedoLogFiles.size() - currentlyMinedLogFiles.size() <= 1
|
||||||
|
* <p>
|
||||||
|
* If each redo lasts for 10 minutes and 7 redo group have been configured, any transaction cannot lasts longer than 1 hour.
|
||||||
|
* <p>
|
||||||
|
* In case of an abandonment, all DMLs/Commits/Rollbacs for this transaction will be ignored
|
||||||
|
* <p>
|
||||||
|
* In other words connector will not send any part of this transaction to Kafka
|
||||||
|
*
|
||||||
|
* @param thresholdScn the smallest SVN of any transaction to keep in the buffer. All others will be removed.
|
||||||
|
*/
|
||||||
|
void abandonLongTransactions(Long thresholdScn) {
|
||||||
|
BigDecimal threshold = new BigDecimal(thresholdScn);
|
||||||
|
Iterator<Map.Entry<String, Transaction>> iter = transactions.entrySet().iterator();
|
||||||
|
while (iter.hasNext()) {
|
||||||
|
Map.Entry<String, Transaction> transaction = iter.next();
|
||||||
|
if (transaction.getValue().firstScn.compareTo(threshold) <= 0) {
|
||||||
|
LogMinerHelper.logWarn(metrics, "Following long running transaction {} will be abandoned and ignored: {} ", transaction.getKey(),
|
||||||
|
transaction.getValue().toString());
|
||||||
|
abandonedTransactionIds.add(transaction.getKey());
|
||||||
|
iter.remove();
|
||||||
|
calculateLargestScn();
|
||||||
|
|
||||||
|
metrics.addAbandonedTransactionId(transaction.getKey());
|
||||||
|
metrics.setActiveTransactions(transactions.size());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigDecimal calculateSmallestScn() {
|
||||||
|
BigDecimal scn = transactions.isEmpty() ? null
|
||||||
|
: transactions.values()
|
||||||
|
.stream()
|
||||||
|
.map(transaction -> transaction.firstScn)
|
||||||
|
.min(BigDecimal::compareTo)
|
||||||
|
.orElseThrow(() -> new DataException("Cannot calculate smallest SCN"));
|
||||||
|
metrics.setOldestScn(scn == null ? -1 : scn.longValue());
|
||||||
|
return scn;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void calculateLargestScn() {
|
||||||
|
largestScn = transactions.isEmpty() ? BigDecimal.ZERO
|
||||||
|
: transactions.values()
|
||||||
|
.stream()
|
||||||
|
.map(transaction -> transaction.lastScn)
|
||||||
|
.max(BigDecimal::compareTo)
|
||||||
|
.orElseThrow(() -> new DataException("Cannot calculate largest SCN"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns {@code true} if buffer is empty, otherwise {@code false}.
|
||||||
|
*
|
||||||
|
* @return {@code true} if buffer is empty, otherwise {@code false}
|
||||||
|
*/
|
||||||
|
boolean isEmpty() {
|
||||||
|
return transactions.isEmpty() && taskCounter.get() == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
StringBuilder result = new StringBuilder();
|
||||||
|
this.transactions.values().forEach(t -> result.append(t.toString()));
|
||||||
|
return result.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Closes buffer.
|
||||||
|
*/
|
||||||
|
void close() {
|
||||||
|
transactions.clear();
|
||||||
|
executor.shutdown();
|
||||||
|
try {
|
||||||
|
if (!executor.awaitTermination(1000L, TimeUnit.MILLISECONDS)) {
|
||||||
|
executor.shutdownNow();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (InterruptedException e) {
|
||||||
|
LogMinerHelper.logError(metrics, "Thread interrupted during shutdown", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Callback is designed to execute when transaction commits.
|
||||||
|
*/
|
||||||
|
public interface CommitCallback {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Executes callback.
|
||||||
|
*
|
||||||
|
* @param timestamp commit timestamp
|
||||||
|
* @param smallestScn smallest SCN among other transactions
|
||||||
|
* @param commitScn commit SCN
|
||||||
|
* @param callbackNumber number of the callback in the transaction
|
||||||
|
*/
|
||||||
|
void execute(Timestamp timestamp, BigDecimal smallestScn, BigDecimal commitScn, int callbackNumber) throws InterruptedException;
|
||||||
|
}
|
||||||
|
|
||||||
|
@NotThreadSafe
|
||||||
|
private static final class Transaction {
|
||||||
|
|
||||||
|
private final BigDecimal firstScn;
|
||||||
|
private BigDecimal lastScn;
|
||||||
|
private final List<CommitCallback> commitCallbacks;
|
||||||
|
private final Map<BigDecimal, List<String>> redoSqlMap;
|
||||||
|
|
||||||
|
private Transaction(BigDecimal firstScn) {
|
||||||
|
this.firstScn = firstScn;
|
||||||
|
this.commitCallbacks = new ArrayList<>();
|
||||||
|
this.redoSqlMap = new HashMap<>();
|
||||||
|
this.lastScn = firstScn;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addRedoSql(BigDecimal scn, String redoSql) {
|
||||||
|
this.lastScn = scn;
|
||||||
|
|
||||||
|
List<String> sqlList = redoSqlMap.get(scn);
|
||||||
|
if (sqlList == null) {
|
||||||
|
redoSqlMap.put(scn, new ArrayList<>(Collections.singletonList(redoSql)));
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
sqlList.add(redoSql);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "Transaction{" +
|
||||||
|
"firstScn=" + firstScn +
|
||||||
|
", lastScn=" + lastScn +
|
||||||
|
", redoSqls=" + Arrays.toString(redoSqlMap.values().toArray()) +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,284 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
|
import io.debezium.annotation.ThreadSafe;
|
||||||
|
import io.debezium.connector.common.CdcSourceTaskContext;
|
||||||
|
import io.debezium.metrics.Metrics;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class contains MBean methods
|
||||||
|
*/
|
||||||
|
@ThreadSafe
|
||||||
|
public class TransactionalBufferMetrics extends Metrics implements TransactionalBufferMetricsMXBean {
|
||||||
|
private AtomicLong oldestScn = new AtomicLong();
|
||||||
|
private AtomicLong committedScn = new AtomicLong();
|
||||||
|
private AtomicReference<Duration> lagFromTheSource = new AtomicReference<>();
|
||||||
|
private AtomicInteger activeTransactions = new AtomicInteger();
|
||||||
|
private AtomicLong rolledBackTransactions = new AtomicLong();
|
||||||
|
private AtomicLong committedTransactions = new AtomicLong();
|
||||||
|
private AtomicLong capturedDmlCounter = new AtomicLong();
|
||||||
|
private AtomicLong committedDmlCounter = new AtomicLong();
|
||||||
|
private AtomicInteger commitQueueCapacity = new AtomicInteger();
|
||||||
|
private AtomicReference<Duration> maxLagFromTheSource = new AtomicReference<>();
|
||||||
|
private AtomicReference<Duration> minLagFromTheSource = new AtomicReference<>();
|
||||||
|
private AtomicReference<Duration> averageLagsFromTheSource = new AtomicReference<>();
|
||||||
|
private AtomicReference<Set<String>> abandonedTransactionIds = new AtomicReference<>();
|
||||||
|
private AtomicReference<Set<String>> rolledBackTransactionIds = new AtomicReference<>();
|
||||||
|
private Instant startTime;
|
||||||
|
private static long MILLIS_PER_SECOND = 1000L;
|
||||||
|
private AtomicLong timeDifference = new AtomicLong();
|
||||||
|
private AtomicInteger errorCounter = new AtomicInteger();
|
||||||
|
private AtomicInteger warningCounter = new AtomicInteger();
|
||||||
|
private AtomicInteger scnFreezeCounter = new AtomicInteger();
|
||||||
|
|
||||||
|
TransactionalBufferMetrics(CdcSourceTaskContext taskContext) {
|
||||||
|
super(taskContext, "log-miner-transactional-buffer");
|
||||||
|
startTime = Instant.now();
|
||||||
|
oldestScn.set(-1);
|
||||||
|
committedScn.set(-1);
|
||||||
|
timeDifference.set(0);
|
||||||
|
reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
// setters
|
||||||
|
void setOldestScn(Long scn) {
|
||||||
|
oldestScn.set(scn);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCommittedScn(Long scn) {
|
||||||
|
committedScn.set(scn);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTimeDifference(AtomicLong timeDifference) {
|
||||||
|
this.timeDifference = timeDifference;
|
||||||
|
}
|
||||||
|
|
||||||
|
void calculateLagMetrics(Instant changeTime) {
|
||||||
|
if (changeTime != null) {
|
||||||
|
Instant correctedChangeTime = changeTime.plus(Duration.ofMillis(timeDifference.longValue()));
|
||||||
|
lagFromTheSource.set(Duration.between(correctedChangeTime, Instant.now()).abs());
|
||||||
|
|
||||||
|
if (maxLagFromTheSource.get().toMillis() < lagFromTheSource.get().toMillis()) {
|
||||||
|
maxLagFromTheSource.set(lagFromTheSource.get());
|
||||||
|
}
|
||||||
|
if (minLagFromTheSource.get().toMillis() > lagFromTheSource.get().toMillis()) {
|
||||||
|
minLagFromTheSource.set(lagFromTheSource.get());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (averageLagsFromTheSource.get().isZero()) {
|
||||||
|
averageLagsFromTheSource.set(lagFromTheSource.get());
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
averageLagsFromTheSource.set(averageLagsFromTheSource.get().plus(lagFromTheSource.get()).dividedBy(2));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void setActiveTransactions(Integer counter) {
|
||||||
|
if (counter != null) {
|
||||||
|
activeTransactions.set(counter);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void incrementRolledBackTransactions() {
|
||||||
|
rolledBackTransactions.incrementAndGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
void incrementCommittedTransactions() {
|
||||||
|
committedTransactions.incrementAndGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
void incrementCapturedDmlCounter() {
|
||||||
|
capturedDmlCounter.incrementAndGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
void incrementCommittedDmlCounter(int counter) {
|
||||||
|
committedDmlCounter.getAndAdd(counter);
|
||||||
|
}
|
||||||
|
|
||||||
|
void addAbandonedTransactionId(String transactionId) {
|
||||||
|
if (transactionId != null) {
|
||||||
|
abandonedTransactionIds.get().add(transactionId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void addRolledBackTransactionId(String transactionId) {
|
||||||
|
if (transactionId != null) {
|
||||||
|
rolledBackTransactionIds.get().add(transactionId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to increase logged logError counter.
|
||||||
|
* There are other ways to monitor the log, but this is just to check if there are any.
|
||||||
|
*/
|
||||||
|
void incrementErrorCounter() {
|
||||||
|
errorCounter.incrementAndGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to increase logged warning counter
|
||||||
|
* There are other ways to monitor the log, but this is just to check if there are any.
|
||||||
|
*/
|
||||||
|
void incrementWarningCounter() {
|
||||||
|
warningCounter.incrementAndGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This counter to accumulate number of encountered observations when SCN does not change in the offset.
|
||||||
|
* This call indicates an uncommitted oldest transaction in the buffer.
|
||||||
|
*/
|
||||||
|
void incrementScnFreezeCounter() {
|
||||||
|
scnFreezeCounter.incrementAndGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
// implemented getters
|
||||||
|
@Override
|
||||||
|
public Long getOldestScn() {
|
||||||
|
return oldestScn.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Long getCommittedScn() {
|
||||||
|
return committedScn.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getNumberOfActiveTransactions() {
|
||||||
|
return activeTransactions.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getNumberOfRolledBackTransactions() {
|
||||||
|
return rolledBackTransactions.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getNumberOfCommittedTransactions() {
|
||||||
|
return committedTransactions.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getCommitThroughput() {
|
||||||
|
long timeSpent = Duration.between(startTime, Instant.now()).isZero() ? 1 : Duration.between(startTime, Instant.now()).toMillis();
|
||||||
|
return committedTransactions.get() * MILLIS_PER_SECOND / timeSpent;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getCapturedDmlThroughput() {
|
||||||
|
long timeSpent = Duration.between(startTime, Instant.now()).isZero() ? 1 : Duration.between(startTime, Instant.now()).toMillis();
|
||||||
|
return committedDmlCounter.get() * MILLIS_PER_SECOND / timeSpent;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getCapturedDmlCount() {
|
||||||
|
return capturedDmlCounter.longValue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getLagFromSource() {
|
||||||
|
return lagFromTheSource.get().toMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getMaxLagFromSource() {
|
||||||
|
return maxLagFromTheSource.get().toMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getMinLagFromSource() {
|
||||||
|
return minLagFromTheSource.get().toMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getAverageLagFromSource() {
|
||||||
|
return averageLagsFromTheSource.get().toMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<String> getAbandonedTransactionIds() {
|
||||||
|
return abandonedTransactionIds.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<String> getRolledBackTransactionIds() {
|
||||||
|
return rolledBackTransactionIds.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getErrorCounter() {
|
||||||
|
return errorCounter.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getWarningCounter() {
|
||||||
|
return warningCounter.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getScnFreezeCounter() {
|
||||||
|
return scnFreezeCounter.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getCommitQueueCapacity() {
|
||||||
|
return commitQueueCapacity.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
void setCommitQueueCapacity(int commitQueueCapacity) {
|
||||||
|
this.commitQueueCapacity.set(commitQueueCapacity);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void reset() {
|
||||||
|
maxLagFromTheSource.set(Duration.ZERO);
|
||||||
|
minLagFromTheSource.set(Duration.ZERO);
|
||||||
|
averageLagsFromTheSource.set(Duration.ZERO);
|
||||||
|
activeTransactions.set(0);
|
||||||
|
rolledBackTransactions.set(0);
|
||||||
|
committedTransactions.set(0);
|
||||||
|
capturedDmlCounter.set(0);
|
||||||
|
committedDmlCounter.set(0);
|
||||||
|
abandonedTransactionIds.set(new HashSet<>());
|
||||||
|
rolledBackTransactionIds.set(new HashSet<>());
|
||||||
|
lagFromTheSource.set(Duration.ZERO);
|
||||||
|
errorCounter.set(0);
|
||||||
|
warningCounter.set(0);
|
||||||
|
scnFreezeCounter.set(0);
|
||||||
|
commitQueueCapacity.set(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "TransactionalBufferMetrics{" +
|
||||||
|
"oldestScn=" + oldestScn.get() +
|
||||||
|
", committedScn=" + committedScn.get() +
|
||||||
|
", lagFromTheSource=" + lagFromTheSource.get() +
|
||||||
|
", activeTransactions=" + activeTransactions.get() +
|
||||||
|
", rolledBackTransactions=" + rolledBackTransactions.get() +
|
||||||
|
", committedTransactions=" + committedTransactions.get() +
|
||||||
|
", capturedDmlCounter=" + capturedDmlCounter.get() +
|
||||||
|
", committedDmlCounter=" + committedDmlCounter.get() +
|
||||||
|
", maxLagFromTheSource=" + maxLagFromTheSource.get() +
|
||||||
|
", minLagFromTheSource=" + minLagFromTheSource.get() +
|
||||||
|
", averageLagsFromTheSource=" + averageLagsFromTheSource.get() +
|
||||||
|
", abandonedTransactionIds=" + abandonedTransactionIds.get() +
|
||||||
|
", errorCounter=" + errorCounter.get() +
|
||||||
|
", warningCounter=" + warningCounter.get() +
|
||||||
|
", scnFreezeCounter=" + scnFreezeCounter.get() +
|
||||||
|
", commitQueueCapacity=" + commitQueueCapacity.get() +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,146 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This interface exposes TransactionalBuffer metrics
|
||||||
|
*/
|
||||||
|
public interface TransactionalBufferMetricsMXBean {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exposes number of rolled back transactions
|
||||||
|
*
|
||||||
|
* @return number of rolled back transaction in the in-memory buffer
|
||||||
|
*/
|
||||||
|
long getNumberOfRolledBackTransactions();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exposes number of committed transactions
|
||||||
|
*
|
||||||
|
* @return number of committed transaction in the in-memory buffer
|
||||||
|
*/
|
||||||
|
long getNumberOfCommittedTransactions();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exposes average number of committed transactions per second
|
||||||
|
*
|
||||||
|
* @return average number of committed transactions per second in the in-memory buffer
|
||||||
|
*/
|
||||||
|
long getCommitThroughput();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exposes average number of captured and parsed DML per second
|
||||||
|
*
|
||||||
|
* @return average number of captured and parsed DML per second in the in-memory buffer
|
||||||
|
*/
|
||||||
|
long getCapturedDmlThroughput();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* exposes total number of captured DMLs
|
||||||
|
*
|
||||||
|
* @return captured DML count
|
||||||
|
*/
|
||||||
|
long getCapturedDmlCount();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exposes number of transaction, buffered in memory
|
||||||
|
*
|
||||||
|
* @return number of currently buffered transactions
|
||||||
|
*/
|
||||||
|
int getNumberOfActiveTransactions();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exposes the oldest(smallest) SCN in the Transactional Buffer
|
||||||
|
*
|
||||||
|
* @return oldest SCN
|
||||||
|
*/
|
||||||
|
Long getOldestScn();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* It shows last committed SCN
|
||||||
|
*
|
||||||
|
* @return committed SCN
|
||||||
|
*/
|
||||||
|
Long getCommittedScn();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to get the lag between latest captured change timestamp in REDO LOG and time of it's placement in the buffer
|
||||||
|
*
|
||||||
|
* @return lag in milliseconds
|
||||||
|
*/
|
||||||
|
long getLagFromSource();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to get max value of the time difference between logging of source DB records into redo log and capturing it by Log Miner
|
||||||
|
*
|
||||||
|
* @return value in milliseconds
|
||||||
|
*/
|
||||||
|
long getMaxLagFromSource();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to get min value of the time difference between logging of source DB records into redo log and capturing it by Log Miner
|
||||||
|
*
|
||||||
|
* @return value in milliseconds
|
||||||
|
*/
|
||||||
|
long getMinLagFromSource();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to get average value of the time difference between logging of source DB records into redo log and capturing it by Log Miner.
|
||||||
|
* Average is calculated as summary of all lags / number of captured DB changes
|
||||||
|
*
|
||||||
|
* @return value in milliseconds
|
||||||
|
*/
|
||||||
|
long getAverageLagFromSource();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to get list of removed transactions from the Transactional Buffer
|
||||||
|
*
|
||||||
|
* @return count abandoned transaction ids
|
||||||
|
*/
|
||||||
|
Set<String> getAbandonedTransactionIds();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* See which transactions were rolled back
|
||||||
|
*
|
||||||
|
* @return set of transaction IDs
|
||||||
|
*/
|
||||||
|
Set<String> getRolledBackTransactionIds();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets commit queue capacity. As the queue fills up, this reduces to zero
|
||||||
|
*
|
||||||
|
* @return the commit queue capacity
|
||||||
|
*/
|
||||||
|
int getCommitQueueCapacity();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* action to reset some metrics
|
||||||
|
*/
|
||||||
|
void reset();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to get logged logError counter.
|
||||||
|
*
|
||||||
|
* @return the error counter
|
||||||
|
*/
|
||||||
|
int getErrorCounter();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is to get logged warning counter
|
||||||
|
*
|
||||||
|
* @return the warning counter
|
||||||
|
*/
|
||||||
|
int getWarningCounter();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get counter of encountered observations when SCN does not change in the offset.
|
||||||
|
*
|
||||||
|
* @return the scn freeze counter
|
||||||
|
*/
|
||||||
|
int getScnFreezeCounter();
|
||||||
|
}
|
@ -0,0 +1,26 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer.valueholder;
|
||||||
|
|
||||||
|
public interface LogMinerColumnValue {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return value of the database record
|
||||||
|
* with exception of LOB types
|
||||||
|
*/
|
||||||
|
Object getColumnData();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return column name
|
||||||
|
*/
|
||||||
|
String getColumnName();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This sets the database record value with the exception of LOBs
|
||||||
|
* @param columnData data
|
||||||
|
*/
|
||||||
|
void setColumnData(Object columnData);
|
||||||
|
}
|
@ -0,0 +1,65 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer.valueholder;
|
||||||
|
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.antlr.listener.ParserUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class stores parsed column info
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class LogMinerColumnValueImpl implements LogMinerColumnValue {
|
||||||
|
|
||||||
|
private String columnName;
|
||||||
|
private Object columnData;
|
||||||
|
private int columnType;
|
||||||
|
|
||||||
|
public LogMinerColumnValueImpl(String columnName, int columnType) {
|
||||||
|
this.columnName = columnName;
|
||||||
|
this.columnType = columnType;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object getColumnData() {
|
||||||
|
return columnData;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getColumnName() {
|
||||||
|
return columnName;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setColumnData(Object columnData) {
|
||||||
|
if (columnData instanceof String) {
|
||||||
|
this.columnData = ParserUtils.replaceDoubleBackSlashes((String) columnData);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
this.columnData = columnData;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (this == o) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (o == null || getClass() != o.getClass()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
LogMinerColumnValueImpl that = (LogMinerColumnValueImpl) o;
|
||||||
|
return columnType == that.columnType &&
|
||||||
|
Objects.equals(columnName, that.columnName) &&
|
||||||
|
Objects.equals(columnData, that.columnData);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Objects.hash(columnName, columnData, columnType);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,37 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer.valueholder;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class is a wrapper class which holds LogMinerColumnValue
|
||||||
|
* and the indicator if the column was processed by a parser listener.
|
||||||
|
* The "processed" is "true" means a listener has parsed a value.
|
||||||
|
* The "false" value means the this value was not parsed yet
|
||||||
|
* The "processed" flag helps to filter the resulting collection of "new" and "old" values.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class LogMinerColumnValueWrapper {
|
||||||
|
|
||||||
|
private boolean processed;
|
||||||
|
private final LogMinerColumnValue columnValue;
|
||||||
|
|
||||||
|
public LogMinerColumnValueWrapper(LogMinerColumnValue columnValue) {
|
||||||
|
this.columnValue = columnValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public LogMinerColumnValue getColumnValue() {
|
||||||
|
return columnValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isProcessed() {
|
||||||
|
return processed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setProcessed(boolean processed) {
|
||||||
|
this.processed = processed;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,22 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer.valueholder;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class is a placeholder of DDL data
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public interface LogMinerDdlEntry {
|
||||||
|
/**
|
||||||
|
* @return text of the DDL statement
|
||||||
|
*/
|
||||||
|
String getDdlText();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return string such as "CREATE TABLE", "ALTER TABLE", "DROP TABLE"
|
||||||
|
*/
|
||||||
|
String getCommandType();
|
||||||
|
}
|
@ -0,0 +1,92 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer.valueholder;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import io.debezium.data.Envelope;
|
||||||
|
|
||||||
|
public interface LogMinerDmlEntry {
|
||||||
|
/**
|
||||||
|
* This getter
|
||||||
|
* @return old(current) values of the database record.
|
||||||
|
* They represent values in WHERE clauses
|
||||||
|
*/
|
||||||
|
List<LogMinerColumnValue> getOldValues();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* this getter
|
||||||
|
* @return new values to be applied to the database record
|
||||||
|
* Those values are applicable for INSERT and UPDATE statements
|
||||||
|
*/
|
||||||
|
List<LogMinerColumnValue> getNewValues();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* this getter
|
||||||
|
* @return Envelope.Operation enum
|
||||||
|
*/
|
||||||
|
Envelope.Operation getCommandType();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* the scn obtained from a Log Miner entry.
|
||||||
|
* This SCN is not a final SCN, just a candidate.
|
||||||
|
* The actual SCN will be assigned after commit
|
||||||
|
* @return it's value
|
||||||
|
*/
|
||||||
|
BigDecimal getScn();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return transaction ID
|
||||||
|
*/
|
||||||
|
String getTransactionId();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return schema name
|
||||||
|
*/
|
||||||
|
String getObjectOwner();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return table name
|
||||||
|
*/
|
||||||
|
String getObjectName();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return database change time of this logical record
|
||||||
|
*/
|
||||||
|
Timestamp getSourceTime();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* sets scn obtained from a Log Miner entry
|
||||||
|
* @param scn it's value
|
||||||
|
*/
|
||||||
|
void setScn(BigDecimal scn);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets table name
|
||||||
|
* @param name table name
|
||||||
|
*/
|
||||||
|
void setObjectName(String name);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets schema owner
|
||||||
|
* @param name schema owner
|
||||||
|
*/
|
||||||
|
void setObjectOwner(String name);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the time of the database change
|
||||||
|
* @param changeTime the time of the change
|
||||||
|
*/
|
||||||
|
void setSourceTime(Timestamp changeTime);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param id unique transaction ID
|
||||||
|
*/
|
||||||
|
void setTransactionId(String id);
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,119 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer.valueholder;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
import io.debezium.data.Envelope;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class holds one parsed DML Log Miner record details
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class LogMinerDmlEntryImpl implements LogMinerDmlEntry {
|
||||||
|
|
||||||
|
private Envelope.Operation commandType;
|
||||||
|
private List<LogMinerColumnValue> newLmColumnValues;
|
||||||
|
private List<LogMinerColumnValue> oldLmColumnValues;
|
||||||
|
private String objectOwner;
|
||||||
|
private String objectName;
|
||||||
|
private Timestamp sourceTime;
|
||||||
|
private String transactionId;
|
||||||
|
private BigDecimal scn;
|
||||||
|
|
||||||
|
public LogMinerDmlEntryImpl(Envelope.Operation commandType, List<LogMinerColumnValue> newLmColumnValues, List<LogMinerColumnValue> oldLmColumnValues) {
|
||||||
|
this.commandType = commandType;
|
||||||
|
this.newLmColumnValues = newLmColumnValues;
|
||||||
|
this.oldLmColumnValues = oldLmColumnValues;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Envelope.Operation getCommandType() {
|
||||||
|
return commandType;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<LogMinerColumnValue> getOldValues() {
|
||||||
|
return oldLmColumnValues;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<LogMinerColumnValue> getNewValues() {
|
||||||
|
return newLmColumnValues;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getTransactionId() {
|
||||||
|
return transactionId;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getObjectOwner() {
|
||||||
|
return objectOwner;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getObjectName() {
|
||||||
|
return objectName;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Timestamp getSourceTime() {
|
||||||
|
return sourceTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setObjectName(String name) {
|
||||||
|
this.objectName = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setObjectOwner(String name) {
|
||||||
|
this.objectOwner = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setSourceTime(Timestamp changeTime) {
|
||||||
|
this.sourceTime = changeTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setTransactionId(String id) {
|
||||||
|
this.transactionId = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BigDecimal getScn() {
|
||||||
|
return scn;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setScn(BigDecimal scn) {
|
||||||
|
this.scn = scn;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (this == o) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (o == null || getClass() != o.getClass()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
LogMinerDmlEntryImpl that = (LogMinerDmlEntryImpl) o;
|
||||||
|
return commandType == that.commandType &&
|
||||||
|
Objects.equals(newLmColumnValues, that.newLmColumnValues) &&
|
||||||
|
Objects.equals(oldLmColumnValues, that.oldLmColumnValues);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Objects.hash(commandType, newLmColumnValues, oldLmColumnValues);
|
||||||
|
}
|
||||||
|
}
|
@ -3,11 +3,12 @@
|
|||||||
*
|
*
|
||||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
*/
|
*/
|
||||||
package io.debezium.connector.oracle;
|
package io.debezium.connector.oracle.xstream;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||||
import io.debezium.pipeline.ErrorHandler;
|
import io.debezium.pipeline.ErrorHandler;
|
||||||
import io.debezium.pipeline.EventDispatcher;
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
import io.debezium.relational.RelationalDatabaseSchema;
|
import io.debezium.relational.RelationalDatabaseSchema;
|
||||||
@ -28,7 +29,7 @@
|
|||||||
*/
|
*/
|
||||||
class LcrEventHandler implements XStreamLCRCallbackHandler {
|
class LcrEventHandler implements XStreamLCRCallbackHandler {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(OracleStreamingChangeEventSource.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(XstreamStreamingChangeEventSource.class);
|
||||||
|
|
||||||
private final ErrorHandler errorHandler;
|
private final ErrorHandler errorHandler;
|
||||||
private final EventDispatcher<TableId> dispatcher;
|
private final EventDispatcher<TableId> dispatcher;
|
||||||
@ -114,7 +115,7 @@ private void dispatchSchemaChangeEvent(DDLLCR ddlLcr) throws InterruptedExceptio
|
|||||||
|
|
||||||
dispatcher.dispatchSchemaChangeEvent(
|
dispatcher.dispatchSchemaChangeEvent(
|
||||||
tableId,
|
tableId,
|
||||||
new OracleSchemaChangeEventEmitter(offsetContext, tableId, ddlLcr));
|
new XStreamSchemaChangeEventEmitter(offsetContext, tableId, ddlLcr));
|
||||||
}
|
}
|
||||||
|
|
||||||
private TableId getTableId(LCR lcr) {
|
private TableId getTableId(LCR lcr) {
|
@ -3,7 +3,7 @@
|
|||||||
*
|
*
|
||||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
*/
|
*/
|
||||||
package io.debezium.connector.oracle;
|
package io.debezium.connector.oracle.xstream;
|
||||||
|
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
@ -49,7 +49,7 @@ public static LcrPosition valueOf(String rawPosition) {
|
|||||||
return new LcrPosition(Strings.hexStringToByteArray(rawPosition));
|
return new LcrPosition(Strings.hexStringToByteArray(rawPosition));
|
||||||
}
|
}
|
||||||
|
|
||||||
byte[] getRawPosition() {
|
public byte[] getRawPosition() {
|
||||||
return rawPosition;
|
return rawPosition;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -77,10 +77,7 @@ public boolean equals(Object obj) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
LcrPosition other = (LcrPosition) obj;
|
LcrPosition other = (LcrPosition) obj;
|
||||||
if (!Arrays.equals(rawPosition, other.rawPosition)) {
|
return Arrays.equals(rawPosition, other.rawPosition);
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
@ -0,0 +1,68 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.xstream;
|
||||||
|
|
||||||
|
import io.debezium.config.EnumeratedValue;
|
||||||
|
|
||||||
|
import oracle.streams.XStreamUtility;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This enum class includes Oracle 11 and 12 major versions
|
||||||
|
* It returns position version based on the Oracle release.
|
||||||
|
* Position version get used in conversion SCN (system change number) into position
|
||||||
|
*/
|
||||||
|
public enum OracleVersion implements EnumeratedValue {
|
||||||
|
|
||||||
|
V11("11"),
|
||||||
|
V12Plus("12+");
|
||||||
|
|
||||||
|
private final String version;
|
||||||
|
|
||||||
|
OracleVersion(String version) {
|
||||||
|
this.version = version;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getValue() {
|
||||||
|
return version;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getPosVersion() {
|
||||||
|
switch (version) {
|
||||||
|
case "11":
|
||||||
|
return XStreamUtility.POS_VERSION_V1;
|
||||||
|
case "12+":
|
||||||
|
return XStreamUtility.POS_VERSION_V2;
|
||||||
|
default:
|
||||||
|
return XStreamUtility.POS_VERSION_V2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static OracleVersion parse(String value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
value = value.trim();
|
||||||
|
|
||||||
|
for (OracleVersion option : OracleVersion.values()) {
|
||||||
|
if (option.getValue().equalsIgnoreCase(value)) {
|
||||||
|
return option;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static OracleVersion parse(String value, String defaultValue) {
|
||||||
|
OracleVersion option = parse(value);
|
||||||
|
|
||||||
|
if (option == null && defaultValue != null) {
|
||||||
|
option = parse(defaultValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
return option;
|
||||||
|
}
|
||||||
|
}
|
@ -3,11 +3,11 @@
|
|||||||
*
|
*
|
||||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
*/
|
*/
|
||||||
package io.debezium.connector.oracle;
|
package io.debezium.connector.oracle.xstream;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.BaseChangeRecordEmitter;
|
||||||
import io.debezium.data.Envelope.Operation;
|
import io.debezium.data.Envelope.Operation;
|
||||||
import io.debezium.pipeline.spi.OffsetContext;
|
import io.debezium.pipeline.spi.OffsetContext;
|
||||||
import io.debezium.relational.RelationalChangeRecordEmitter;
|
|
||||||
import io.debezium.relational.Table;
|
import io.debezium.relational.Table;
|
||||||
import io.debezium.util.Clock;
|
import io.debezium.util.Clock;
|
||||||
|
|
||||||
@ -19,16 +19,13 @@
|
|||||||
*
|
*
|
||||||
* @author Gunnar Morling
|
* @author Gunnar Morling
|
||||||
*/
|
*/
|
||||||
public class XStreamChangeRecordEmitter extends RelationalChangeRecordEmitter {
|
public class XStreamChangeRecordEmitter extends BaseChangeRecordEmitter<ColumnValue> {
|
||||||
|
|
||||||
private final RowLCR lcr;
|
private final RowLCR lcr;
|
||||||
private final Table table;
|
|
||||||
|
|
||||||
public XStreamChangeRecordEmitter(OffsetContext offset, RowLCR lcr, Table table, Clock clock) {
|
public XStreamChangeRecordEmitter(OffsetContext offset, RowLCR lcr, Table table, Clock clock) {
|
||||||
super(offset, clock);
|
super(offset, table, clock);
|
||||||
|
|
||||||
this.lcr = lcr;
|
this.lcr = lcr;
|
||||||
this.table = table;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -55,14 +52,13 @@ protected Object[] getNewColumnValues() {
|
|||||||
return getColumnValues(lcr.getNewValues());
|
return getColumnValues(lcr.getNewValues());
|
||||||
}
|
}
|
||||||
|
|
||||||
private Object[] getColumnValues(ColumnValue[] columnValues) {
|
@Override
|
||||||
Object[] values = new Object[table.columns().size()];
|
protected String getColumnName(ColumnValue columnValue) {
|
||||||
|
return columnValue.getColumnName();
|
||||||
|
}
|
||||||
|
|
||||||
for (ColumnValue columnValue : columnValues) {
|
@Override
|
||||||
int index = table.columnWithName(columnValue.getColumnName()).position() - 1;
|
protected Object getColumnData(ColumnValue columnValue) {
|
||||||
values[index] = columnValue.getColumnData();
|
return columnValue.getColumnData();
|
||||||
}
|
|
||||||
|
|
||||||
return values;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -0,0 +1,30 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.xstream;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.BaseOracleSchemaChangeEventEmitter;
|
||||||
|
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||||
|
import io.debezium.pipeline.spi.SchemaChangeEventEmitter;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
|
||||||
|
import oracle.streams.DDLLCR;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@link SchemaChangeEventEmitter} implementation based on Oracle.
|
||||||
|
*
|
||||||
|
* @author Gunnar Morling
|
||||||
|
*/
|
||||||
|
public class XStreamSchemaChangeEventEmitter extends BaseOracleSchemaChangeEventEmitter {
|
||||||
|
|
||||||
|
public XStreamSchemaChangeEventEmitter(OracleOffsetContext offsetContext, TableId tableId, DDLLCR ddlLcr) {
|
||||||
|
super(offsetContext,
|
||||||
|
tableId,
|
||||||
|
ddlLcr.getSourceDatabaseName(),
|
||||||
|
ddlLcr.getObjectOwner(),
|
||||||
|
ddlLcr.getDDLText(),
|
||||||
|
ddlLcr.getCommandType());
|
||||||
|
}
|
||||||
|
}
|
@ -3,13 +3,17 @@
|
|||||||
*
|
*
|
||||||
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
*/
|
*/
|
||||||
package io.debezium.connector.oracle;
|
package io.debezium.connector.oracle.xstream;
|
||||||
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||||
|
import io.debezium.connector.oracle.OracleDatabaseSchema;
|
||||||
|
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||||
|
import io.debezium.connector.oracle.SourceInfo;
|
||||||
import io.debezium.jdbc.JdbcConnection;
|
import io.debezium.jdbc.JdbcConnection;
|
||||||
import io.debezium.pipeline.ErrorHandler;
|
import io.debezium.pipeline.ErrorHandler;
|
||||||
import io.debezium.pipeline.EventDispatcher;
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
@ -29,9 +33,9 @@
|
|||||||
*
|
*
|
||||||
* @author Gunnar Morling
|
* @author Gunnar Morling
|
||||||
*/
|
*/
|
||||||
public class OracleStreamingChangeEventSource implements StreamingChangeEventSource {
|
public class XstreamStreamingChangeEventSource implements StreamingChangeEventSource {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(OracleStreamingChangeEventSource.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(XstreamStreamingChangeEventSource.class);
|
||||||
|
|
||||||
private final JdbcConnection jdbcConnection;
|
private final JdbcConnection jdbcConnection;
|
||||||
private final EventDispatcher<TableId> dispatcher;
|
private final EventDispatcher<TableId> dispatcher;
|
||||||
@ -44,8 +48,8 @@ public class OracleStreamingChangeEventSource implements StreamingChangeEventSou
|
|||||||
private final boolean tablenameCaseInsensitive;
|
private final boolean tablenameCaseInsensitive;
|
||||||
private final int posVersion;
|
private final int posVersion;
|
||||||
|
|
||||||
public OracleStreamingChangeEventSource(OracleConnectorConfig connectorConfig, OracleOffsetContext offsetContext, JdbcConnection jdbcConnection,
|
public XstreamStreamingChangeEventSource(OracleConnectorConfig connectorConfig, OracleOffsetContext offsetContext, JdbcConnection jdbcConnection,
|
||||||
EventDispatcher<TableId> dispatcher, ErrorHandler errorHandler, Clock clock, OracleDatabaseSchema schema) {
|
EventDispatcher<TableId> dispatcher, ErrorHandler errorHandler, Clock clock, OracleDatabaseSchema schema) {
|
||||||
this.jdbcConnection = jdbcConnection;
|
this.jdbcConnection = jdbcConnection;
|
||||||
this.dispatcher = dispatcher;
|
this.dispatcher = dispatcher;
|
||||||
this.errorHandler = errorHandler;
|
this.errorHandler = errorHandler;
|
@ -228,7 +228,7 @@ protected List<String> getAllTables() {
|
|||||||
protected abstract Builder connectorConfig();
|
protected abstract Builder connectorConfig();
|
||||||
|
|
||||||
private static void streamTable(String table) throws SQLException {
|
private static void streamTable(String table) throws SQLException {
|
||||||
connection.execute("GRANT SELECT ON " + table + " to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON " + table + " to " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE " + table + " ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE " + table + " ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -7,7 +7,12 @@
|
|||||||
|
|
||||||
import static org.fest.assertions.Assertions.assertThat;
|
import static org.fest.assertions.Assertions.assertThat;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.PreparedStatement;
|
||||||
|
import java.sql.ResultSet;
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
@ -16,10 +21,15 @@
|
|||||||
import org.junit.AfterClass;
|
import org.junit.AfterClass;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
|
||||||
import io.debezium.config.Configuration;
|
import io.debezium.config.Configuration;
|
||||||
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIs;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||||
import io.debezium.connector.oracle.util.TestHelper;
|
import io.debezium.connector.oracle.util.TestHelper;
|
||||||
import io.debezium.data.VerifyRecord;
|
import io.debezium.data.VerifyRecord;
|
||||||
import io.debezium.embedded.AbstractConnectorTest;
|
import io.debezium.embedded.AbstractConnectorTest;
|
||||||
@ -35,6 +45,9 @@ public class OracleConnectorFilterIT extends AbstractConnectorTest {
|
|||||||
private static OracleConnection connection;
|
private static OracleConnection connection;
|
||||||
private static OracleConnection adminConnection;
|
private static OracleConnection adminConnection;
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void beforeClass() throws SQLException {
|
public static void beforeClass() throws SQLException {
|
||||||
connection = TestHelper.testConnection();
|
connection = TestHelper.testConnection();
|
||||||
@ -71,8 +84,8 @@ public void before() throws SQLException {
|
|||||||
"CREATE TABLE debezium2.table2 (id NUMERIC(9,0) NOT NULL, name VARCHAR2(1000), PRIMARY KEY (id))",
|
"CREATE TABLE debezium2.table2 (id NUMERIC(9,0) NOT NULL, name VARCHAR2(1000), PRIMARY KEY (id))",
|
||||||
"CREATE TABLE debezium2.nopk (id NUMERIC(9,0) NOT NULL)",
|
"CREATE TABLE debezium2.nopk (id NUMERIC(9,0) NOT NULL)",
|
||||||
"GRANT ALL PRIVILEGES ON debezium2.table2 TO debezium",
|
"GRANT ALL PRIVILEGES ON debezium2.table2 TO debezium",
|
||||||
"GRANT SELECT ON debezium2.table2 TO " + TestHelper.CONNECTOR_USER,
|
"GRANT SELECT ON debezium2.table2 TO " + TestHelper.getConnectorUserName(),
|
||||||
"GRANT SELECT ON debezium2.nopk TO " + TestHelper.CONNECTOR_USER,
|
"GRANT SELECT ON debezium2.nopk TO " + TestHelper.getConnectorUserName(),
|
||||||
"ALTER TABLE debezium2.table2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
"ALTER TABLE debezium2.table2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
String ddl = "CREATE TABLE debezium.table1 (" +
|
String ddl = "CREATE TABLE debezium.table1 (" +
|
||||||
" id NUMERIC(9,0) NOT NULL, " +
|
" id NUMERIC(9,0) NOT NULL, " +
|
||||||
@ -81,7 +94,7 @@ public void before() throws SQLException {
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl);
|
connection.execute(ddl);
|
||||||
connection.execute("GRANT SELECT ON debezium.table1 TO " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.table1 TO " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.table1 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.table1 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
|
|
||||||
ddl = "CREATE TABLE debezium.table2 (" +
|
ddl = "CREATE TABLE debezium.table2 (" +
|
||||||
@ -91,7 +104,7 @@ public void before() throws SQLException {
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl);
|
connection.execute(ddl);
|
||||||
connection.execute("GRANT SELECT ON debezium.table2 TO " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.table2 TO " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.table2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.table2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
|
|
||||||
initializeConnectorTestFramework();
|
initializeConnectorTestFramework();
|
||||||
@ -99,10 +112,11 @@ public void before() throws SQLException {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Log Miner does not support DDL during streaming")
|
||||||
public void shouldApplyTableWhitelistConfiguration() throws Exception {
|
public void shouldApplyTableWhitelistConfiguration() throws Exception {
|
||||||
Configuration config = TestHelper.defaultConfig()
|
Configuration config = TestHelper.defaultConfig()
|
||||||
.with(
|
.with(
|
||||||
OracleConnectorConfig.TABLE_WHITELIST,
|
RelationalDatabaseConnectorConfig.TABLE_WHITELIST,
|
||||||
"DEBEZIUM2\\.TABLE2,DEBEZIUM\\.TABLE1,DEBEZIUM\\.TABLE3")
|
"DEBEZIUM2\\.TABLE2,DEBEZIUM\\.TABLE1,DEBEZIUM\\.TABLE3")
|
||||||
.with(OracleConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL_SCHEMA_ONLY)
|
.with(OracleConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL_SCHEMA_ONLY)
|
||||||
.build();
|
.build();
|
||||||
@ -111,6 +125,7 @@ public void shouldApplyTableWhitelistConfiguration() throws Exception {
|
|||||||
assertConnectorIsRunning();
|
assertConnectorIsRunning();
|
||||||
|
|
||||||
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||||
|
waitForStreamingRunning(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||||
|
|
||||||
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
||||||
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
||||||
@ -151,6 +166,7 @@ public void shouldApplyTableWhitelistConfiguration() throws Exception {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Log Miner does not support DDL during streaming")
|
||||||
public void shouldApplyTableIncludeListConfiguration() throws Exception {
|
public void shouldApplyTableIncludeListConfiguration() throws Exception {
|
||||||
Configuration config = TestHelper.defaultConfig()
|
Configuration config = TestHelper.defaultConfig()
|
||||||
.with(
|
.with(
|
||||||
@ -163,6 +179,7 @@ public void shouldApplyTableIncludeListConfiguration() throws Exception {
|
|||||||
assertConnectorIsRunning();
|
assertConnectorIsRunning();
|
||||||
|
|
||||||
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||||
|
waitForStreamingRunning(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||||
|
|
||||||
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
||||||
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
||||||
@ -175,8 +192,8 @@ public void shouldApplyTableIncludeListConfiguration() throws Exception {
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl);
|
connection.execute(ddl);
|
||||||
connection.execute("GRANT SELECT ON debezium.table3 TO c##xstrm");
|
connection.execute("GRANT SELECT ON debezium.table3 TO " + TestHelper.getConnectorUserName());
|
||||||
|
connection.execute("ALTER TABLE debezium.table3 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
connection.execute("INSERT INTO debezium.table3 VALUES (3, 'Text-3')");
|
connection.execute("INSERT INTO debezium.table3 VALUES (3, 'Text-3')");
|
||||||
connection.execute("COMMIT");
|
connection.execute("COMMIT");
|
||||||
|
|
||||||
@ -203,6 +220,7 @@ public void shouldApplyTableIncludeListConfiguration() throws Exception {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Log Miner does not support DDL during streaming")
|
||||||
public void shouldApplyTableBlacklistConfiguration() throws Exception {
|
public void shouldApplyTableBlacklistConfiguration() throws Exception {
|
||||||
Configuration config = TestHelper.defaultConfig()
|
Configuration config = TestHelper.defaultConfig()
|
||||||
.with(
|
.with(
|
||||||
@ -215,6 +233,7 @@ public void shouldApplyTableBlacklistConfiguration() throws Exception {
|
|||||||
assertConnectorIsRunning();
|
assertConnectorIsRunning();
|
||||||
|
|
||||||
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||||
|
waitForStreamingRunning(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||||
|
|
||||||
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
||||||
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
||||||
@ -227,7 +246,7 @@ public void shouldApplyTableBlacklistConfiguration() throws Exception {
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl);
|
connection.execute(ddl);
|
||||||
connection.execute("GRANT SELECT ON debezium.table3 TO " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.table3 TO " + TestHelper.getConnectorUserName());
|
||||||
|
|
||||||
connection.execute("INSERT INTO debezium.table3 VALUES (3, 'Text-3')");
|
connection.execute("INSERT INTO debezium.table3 VALUES (3, 'Text-3')");
|
||||||
connection.execute("COMMIT");
|
connection.execute("COMMIT");
|
||||||
@ -255,6 +274,7 @@ public void shouldApplyTableBlacklistConfiguration() throws Exception {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Log Miner does not support DDL during streaming")
|
||||||
public void shouldApplyTableExcludeListConfiguration() throws Exception {
|
public void shouldApplyTableExcludeListConfiguration() throws Exception {
|
||||||
Configuration config = TestHelper.defaultConfig()
|
Configuration config = TestHelper.defaultConfig()
|
||||||
.with(
|
.with(
|
||||||
@ -267,6 +287,7 @@ public void shouldApplyTableExcludeListConfiguration() throws Exception {
|
|||||||
assertConnectorIsRunning();
|
assertConnectorIsRunning();
|
||||||
|
|
||||||
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||||
|
waitForStreamingRunning(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||||
|
|
||||||
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
connection.execute("INSERT INTO debezium.table1 VALUES (1, 'Text-1')");
|
||||||
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
connection.execute("INSERT INTO debezium.table2 VALUES (2, 'Text-2')");
|
||||||
@ -279,7 +300,7 @@ public void shouldApplyTableExcludeListConfiguration() throws Exception {
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl);
|
connection.execute(ddl);
|
||||||
connection.execute("GRANT SELECT ON debezium.table3 TO " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.table3 TO " + TestHelper.getConnectorUserName());
|
||||||
|
|
||||||
connection.execute("INSERT INTO debezium.table3 VALUES (3, 'Text-3')");
|
connection.execute("INSERT INTO debezium.table3 VALUES (3, 'Text-3')");
|
||||||
connection.execute("COMMIT");
|
connection.execute("COMMIT");
|
||||||
@ -305,4 +326,21 @@ public void shouldApplyTableExcludeListConfiguration() throws Exception {
|
|||||||
assertThat(after.get("ID")).isEqualTo(3);
|
assertThat(after.get("ID")).isEqualTo(3);
|
||||||
assertThat(after.get("NAME")).isEqualTo("Text-3");
|
assertThat(after.get("NAME")).isEqualTo("Text-3");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@SkipWhenAdapterNameIsNot(value = SkipWhenAdapterNameIsNot.AdapterName.LOGMINER)
|
||||||
|
public void shouldTakeTimeDifference() throws Exception {
|
||||||
|
Testing.Print.enable();
|
||||||
|
String stmt = "select current_timestamp from dual";
|
||||||
|
try (Connection conn = connection.connection(true);
|
||||||
|
PreparedStatement ps = conn.prepareStatement(stmt);
|
||||||
|
ResultSet rs = ps.executeQuery()) {
|
||||||
|
rs.next();
|
||||||
|
java.sql.Timestamp ts = rs.getTimestamp(1);
|
||||||
|
Instant fromDb = ts.toInstant();
|
||||||
|
Instant now = Instant.now();
|
||||||
|
long diff = Duration.between(fromDb, now).toMillis();
|
||||||
|
Testing.print("diff: " + diff);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -26,10 +26,14 @@
|
|||||||
import org.junit.AfterClass;
|
import org.junit.AfterClass;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
|
||||||
import io.debezium.config.Configuration;
|
import io.debezium.config.Configuration;
|
||||||
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIs;
|
||||||
import io.debezium.connector.oracle.util.TestHelper;
|
import io.debezium.connector.oracle.util.TestHelper;
|
||||||
import io.debezium.data.Envelope;
|
import io.debezium.data.Envelope;
|
||||||
import io.debezium.data.VerifyRecord;
|
import io.debezium.data.VerifyRecord;
|
||||||
@ -48,11 +52,19 @@ public class OracleConnectorIT extends AbstractConnectorTest {
|
|||||||
private static final long MICROS_PER_SECOND = TimeUnit.SECONDS.toMicros(1);
|
private static final long MICROS_PER_SECOND = TimeUnit.SECONDS.toMicros(1);
|
||||||
private static final String SNAPSHOT_COMPLETED_KEY = "snapshot_completed";
|
private static final String SNAPSHOT_COMPLETED_KEY = "snapshot_completed";
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public final TestRule skipAdapterRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
private static OracleConnection connection;
|
private static OracleConnection connection;
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void beforeClass() throws SQLException {
|
public static void beforeClass() throws SQLException {
|
||||||
connection = TestHelper.testConnection();
|
if (TestHelper.adapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
|
||||||
|
connection = TestHelper.logMinerPdbConnection();
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
connection = TestHelper.testConnection();
|
||||||
|
}
|
||||||
|
|
||||||
TestHelper.dropTable(connection, "debezium.customer");
|
TestHelper.dropTable(connection, "debezium.customer");
|
||||||
TestHelper.dropTable(connection, "debezium.masked_hashed_column_table");
|
TestHelper.dropTable(connection, "debezium.masked_hashed_column_table");
|
||||||
@ -68,7 +80,7 @@ public static void beforeClass() throws SQLException {
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl);
|
connection.execute(ddl);
|
||||||
connection.execute("GRANT SELECT ON debezium.customer to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.customer to " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.customer ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.customer ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
|
|
||||||
String ddl2 = "create table debezium.masked_hashed_column_table (" +
|
String ddl2 = "create table debezium.masked_hashed_column_table (" +
|
||||||
@ -80,7 +92,7 @@ public static void beforeClass() throws SQLException {
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl2);
|
connection.execute(ddl2);
|
||||||
connection.execute("GRANT SELECT ON debezium.masked_hashed_column_table to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.masked_hashed_column_table to " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.masked_hashed_column_table ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.masked_hashed_column_table ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
|
|
||||||
String ddl3 = "create table debezium.truncated_column_table (" +
|
String ddl3 = "create table debezium.truncated_column_table (" +
|
||||||
@ -90,7 +102,7 @@ public static void beforeClass() throws SQLException {
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl3);
|
connection.execute(ddl3);
|
||||||
connection.execute("GRANT SELECT ON debezium.truncated_column_table to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.truncated_column_table to " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.truncated_column_table ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.truncated_column_table ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
|
|
||||||
String ddl4 = "create table dt_table (" +
|
String ddl4 = "create table dt_table (" +
|
||||||
@ -105,7 +117,7 @@ public static void beforeClass() throws SQLException {
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl4);
|
connection.execute(ddl4);
|
||||||
connection.execute("GRANT SELECT ON debezium.dt_table to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.dt_table to " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.dt_table ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.dt_table ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -139,7 +151,7 @@ public void shouldSnapshotAndStreamWithHyphenedTableName() throws Exception {
|
|||||||
" primary key (id))";
|
" primary key (id))";
|
||||||
|
|
||||||
connection.execute(ddl);
|
connection.execute(ddl);
|
||||||
connection.execute("GRANT SELECT ON debezium.\"my-table\" to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.\"my-table\" to " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.\"my-table\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.\"my-table\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
connection.execute("INSERT INTO debezium.\"my-table\" VALUES (1, 25, 'Test')");
|
connection.execute("INSERT INTO debezium.\"my-table\" VALUES (1, 25, 'Test')");
|
||||||
connection.execute("COMMIT");
|
connection.execute("COMMIT");
|
||||||
@ -300,6 +312,7 @@ public void shouldContinueWithStreamingAfterSnapshot() throws Exception {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
@FixFor("DBZ-1223")
|
@FixFor("DBZ-1223")
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "sendTxBatch randomly fails")
|
||||||
public void shouldStreamTransaction() throws Exception {
|
public void shouldStreamTransaction() throws Exception {
|
||||||
Configuration config = TestHelper.defaultConfig()
|
Configuration config = TestHelper.defaultConfig()
|
||||||
.with(OracleConnectorConfig.TABLE_INCLUDE_LIST, "DEBEZIUM\\.CUSTOMER")
|
.with(OracleConnectorConfig.TABLE_INCLUDE_LIST, "DEBEZIUM\\.CUSTOMER")
|
||||||
@ -346,20 +359,28 @@ public void shouldStreamTransaction() throws Exception {
|
|||||||
|
|
||||||
expectedRecordCount = 30;
|
expectedRecordCount = 30;
|
||||||
connection.setAutoCommit(false);
|
connection.setAutoCommit(false);
|
||||||
sendTxBatch(expectedRecordCount, 100);
|
sendTxBatch(config, expectedRecordCount, 100);
|
||||||
sendTxBatch(expectedRecordCount, 200);
|
sendTxBatch(config, expectedRecordCount, 200);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void sendTxBatch(int expectedRecordCount, int offset) throws SQLException, InterruptedException {
|
private void sendTxBatch(Configuration config, int expectedRecordCount, int offset) throws SQLException, InterruptedException {
|
||||||
|
boolean isAutoCommit = false;
|
||||||
|
if (connection.connection().getAutoCommit()) {
|
||||||
|
isAutoCommit = true;
|
||||||
|
connection.connection().setAutoCommit(false);
|
||||||
|
}
|
||||||
for (int i = offset; i < expectedRecordCount + offset; i++) {
|
for (int i = offset; i < expectedRecordCount + offset; i++) {
|
||||||
connection.executeWithoutCommitting(String.format("INSERT INTO debezium.customer VALUES (%s, 'Brian%s', 2345.67, null)", i, i));
|
connection.executeWithoutCommitting(String.format("INSERT INTO debezium.customer VALUES (%s, 'Brian%s', 2345.67, null)", i, i));
|
||||||
}
|
}
|
||||||
connection.connection().commit();
|
connection.connection().commit();
|
||||||
|
if (isAutoCommit) {
|
||||||
|
connection.connection().setAutoCommit(true);
|
||||||
|
}
|
||||||
|
|
||||||
assertTxBatch(expectedRecordCount, offset);
|
assertTxBatch(config, expectedRecordCount, offset);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void assertTxBatch(int expectedRecordCount, int offset) throws InterruptedException {
|
private void assertTxBatch(Configuration config, int expectedRecordCount, int offset) throws InterruptedException {
|
||||||
SourceRecords records;
|
SourceRecords records;
|
||||||
List<SourceRecord> testTableRecords;
|
List<SourceRecord> testTableRecords;
|
||||||
Struct after;
|
Struct after;
|
||||||
@ -367,6 +388,7 @@ private void assertTxBatch(int expectedRecordCount, int offset) throws Interrupt
|
|||||||
records = consumeRecordsByTopic(expectedRecordCount);
|
records = consumeRecordsByTopic(expectedRecordCount);
|
||||||
testTableRecords = records.recordsForTopic("server1.DEBEZIUM.CUSTOMER");
|
testTableRecords = records.recordsForTopic("server1.DEBEZIUM.CUSTOMER");
|
||||||
assertThat(testTableRecords).hasSize(expectedRecordCount);
|
assertThat(testTableRecords).hasSize(expectedRecordCount);
|
||||||
|
final String adapter = config.getString(OracleConnectorConfig.CONNECTOR_ADAPTER);
|
||||||
|
|
||||||
for (int i = 0; i < expectedRecordCount; i++) {
|
for (int i = 0; i < expectedRecordCount; i++) {
|
||||||
SourceRecord record3 = testTableRecords.get(i);
|
SourceRecord record3 = testTableRecords.get(i);
|
||||||
@ -376,13 +398,19 @@ private void assertTxBatch(int expectedRecordCount, int offset) throws Interrupt
|
|||||||
|
|
||||||
assertThat(record3.sourceOffset().containsKey(SourceInfo.SNAPSHOT_KEY)).isFalse();
|
assertThat(record3.sourceOffset().containsKey(SourceInfo.SNAPSHOT_KEY)).isFalse();
|
||||||
assertThat(record3.sourceOffset().containsKey(SNAPSHOT_COMPLETED_KEY)).isFalse();
|
assertThat(record3.sourceOffset().containsKey(SNAPSHOT_COMPLETED_KEY)).isFalse();
|
||||||
assertThat(record3.sourceOffset().containsKey(SourceInfo.LCR_POSITION_KEY)).isTrue();
|
|
||||||
assertThat(record3.sourceOffset().containsKey(SourceInfo.SCN_KEY)).isFalse();
|
if (!"LogMiner".equalsIgnoreCase(adapter)) {
|
||||||
|
assertThat(record3.sourceOffset().containsKey(SourceInfo.LCR_POSITION_KEY)).isTrue();
|
||||||
|
assertThat(record3.sourceOffset().containsKey(SourceInfo.SCN_KEY)).isFalse();
|
||||||
|
}
|
||||||
|
|
||||||
source = (Struct) ((Struct) record3.value()).get("source");
|
source = (Struct) ((Struct) record3.value()).get("source");
|
||||||
assertThat(source.get(SourceInfo.SNAPSHOT_KEY)).isEqualTo("false");
|
assertThat(source.get(SourceInfo.SNAPSHOT_KEY)).isEqualTo("false");
|
||||||
assertThat(source.get(SourceInfo.SCN_KEY)).isNotNull();
|
assertThat(source.get(SourceInfo.SCN_KEY)).isNotNull();
|
||||||
assertThat(source.get(SourceInfo.LCR_POSITION_KEY)).isNotNull();
|
if (!"LogMiner".equalsIgnoreCase(adapter)) {
|
||||||
|
assertThat(source.get(SourceInfo.LCR_POSITION_KEY)).isNotNull();
|
||||||
|
}
|
||||||
|
|
||||||
assertThat(source.get(SourceInfo.SERVER_NAME_KEY)).isEqualTo("server1");
|
assertThat(source.get(SourceInfo.SERVER_NAME_KEY)).isEqualTo("server1");
|
||||||
assertThat(source.get(SourceInfo.DEBEZIUM_VERSION_KEY)).isNotNull();
|
assertThat(source.get(SourceInfo.DEBEZIUM_VERSION_KEY)).isNotNull();
|
||||||
assertThat(source.get(SourceInfo.TXID_KEY)).isNotNull();
|
assertThat(source.get(SourceInfo.TXID_KEY)).isNotNull();
|
||||||
@ -391,6 +419,7 @@ private void assertTxBatch(int expectedRecordCount, int offset) throws Interrupt
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Test randomly fails in sendTxBatch")
|
||||||
public void shouldStreamAfterRestart() throws Exception {
|
public void shouldStreamAfterRestart() throws Exception {
|
||||||
Configuration config = TestHelper.defaultConfig()
|
Configuration config = TestHelper.defaultConfig()
|
||||||
.with(OracleConnectorConfig.TABLE_INCLUDE_LIST, "DEBEZIUM\\.CUSTOMER")
|
.with(OracleConnectorConfig.TABLE_INCLUDE_LIST, "DEBEZIUM\\.CUSTOMER")
|
||||||
@ -406,14 +435,16 @@ public void shouldStreamAfterRestart() throws Exception {
|
|||||||
start(OracleConnector.class, config);
|
start(OracleConnector.class, config);
|
||||||
assertConnectorIsRunning();
|
assertConnectorIsRunning();
|
||||||
|
|
||||||
|
waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||||
|
|
||||||
SourceRecords records = consumeRecordsByTopic(expectedRecordCount);
|
SourceRecords records = consumeRecordsByTopic(expectedRecordCount);
|
||||||
List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.CUSTOMER");
|
List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.CUSTOMER");
|
||||||
assertThat(testTableRecords).hasSize(expectedRecordCount);
|
assertThat(testTableRecords).hasSize(expectedRecordCount);
|
||||||
|
|
||||||
expectedRecordCount = 30;
|
expectedRecordCount = 30;
|
||||||
connection.setAutoCommit(false);
|
connection.setAutoCommit(false);
|
||||||
sendTxBatch(expectedRecordCount, 100);
|
sendTxBatch(config, expectedRecordCount, 100);
|
||||||
sendTxBatch(expectedRecordCount, 200);
|
sendTxBatch(config, expectedRecordCount, 200);
|
||||||
|
|
||||||
stopConnector();
|
stopConnector();
|
||||||
final int OFFSET = 300;
|
final int OFFSET = 300;
|
||||||
@ -425,9 +456,11 @@ public void shouldStreamAfterRestart() throws Exception {
|
|||||||
start(OracleConnector.class, config);
|
start(OracleConnector.class, config);
|
||||||
assertConnectorIsRunning();
|
assertConnectorIsRunning();
|
||||||
|
|
||||||
assertTxBatch(expectedRecordCount, 300);
|
waitForStreamingRunning(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
|
||||||
sendTxBatch(expectedRecordCount, 400);
|
|
||||||
sendTxBatch(expectedRecordCount, 500);
|
assertTxBatch(config, expectedRecordCount, 300);
|
||||||
|
sendTxBatch(config, expectedRecordCount, 400);
|
||||||
|
sendTxBatch(config, expectedRecordCount, 500);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -459,11 +492,14 @@ public void shouldStreamAfterRestartAfterSnapshot() throws Exception {
|
|||||||
}
|
}
|
||||||
connection.connection().commit();
|
connection.connection().commit();
|
||||||
|
|
||||||
|
connection.setAutoCommit(true);
|
||||||
|
|
||||||
|
Testing.print("=== Starting connector second time ===");
|
||||||
start(OracleConnector.class, config);
|
start(OracleConnector.class, config);
|
||||||
assertConnectorIsRunning();
|
assertConnectorIsRunning();
|
||||||
|
|
||||||
assertTxBatch(expectedRecordCount, 100);
|
assertTxBatch(config, expectedRecordCount, 100);
|
||||||
sendTxBatch(expectedRecordCount, 200);
|
sendTxBatch(config, expectedRecordCount, 200);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -599,6 +635,7 @@ public void deleteWithoutTombstone() throws Exception {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Seems to get caught in loop?")
|
||||||
public void shouldReadChangeStreamForTableCreatedWhileStreaming() throws Exception {
|
public void shouldReadChangeStreamForTableCreatedWhileStreaming() throws Exception {
|
||||||
TestHelper.dropTable(connection, "debezium.customer2");
|
TestHelper.dropTable(connection, "debezium.customer2");
|
||||||
|
|
||||||
@ -620,7 +657,7 @@ public void shouldReadChangeStreamForTableCreatedWhileStreaming() throws Excepti
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl);
|
connection.execute(ddl);
|
||||||
connection.execute("GRANT SELECT ON debezium.customer2 to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.customer2 to " + TestHelper.getConnectorUserName());
|
||||||
|
|
||||||
connection.execute("INSERT INTO debezium.customer2 VALUES (2, 'Billie-Bob', 1234.56, TO_DATE('2018/02/22', 'yyyy-mm-dd'))");
|
connection.execute("INSERT INTO debezium.customer2 VALUES (2, 'Billie-Bob', 1234.56, TO_DATE('2018/02/22', 'yyyy-mm-dd'))");
|
||||||
connection.execute("COMMIT");
|
connection.execute("COMMIT");
|
||||||
@ -640,6 +677,7 @@ public void shouldReadChangeStreamForTableCreatedWhileStreaming() throws Excepti
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
@FixFor("DBZ-800")
|
@FixFor("DBZ-800")
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Log Miner does not yet support DDL during streaming")
|
||||||
public void shouldReceiveHeartbeatAlsoWhenChangingTableIncludeListTables() throws Exception {
|
public void shouldReceiveHeartbeatAlsoWhenChangingTableIncludeListTables() throws Exception {
|
||||||
TestHelper.dropTable(connection, "debezium.dbz800a");
|
TestHelper.dropTable(connection, "debezium.dbz800a");
|
||||||
TestHelper.dropTable(connection, "debezium.dbz800b");
|
TestHelper.dropTable(connection, "debezium.dbz800b");
|
||||||
|
@ -28,7 +28,8 @@
|
|||||||
*/
|
*/
|
||||||
public class OracleDdlParserTest {
|
public class OracleDdlParserTest {
|
||||||
|
|
||||||
private static final String TABLE_NAME = "DEBEZIUM";
|
private static final String TABLE_NAME = "TEST";
|
||||||
|
private static final String PDB_NAME = "ORCLPDB1";
|
||||||
|
|
||||||
private OracleDdlParser parser;
|
private OracleDdlParser parser;
|
||||||
private Tables tables;
|
private Tables tables;
|
||||||
@ -47,7 +48,7 @@ public void shouldParseCreateAndAlterTable() throws Exception {
|
|||||||
Table table = tables.forTable(new TableId(null, null, TABLE_NAME));
|
Table table = tables.forTable(new TableId(null, null, TABLE_NAME));
|
||||||
|
|
||||||
assertThat(tables.size()).isEqualTo(1);
|
assertThat(tables.size()).isEqualTo(1);
|
||||||
assertThat(table.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL7", "COL8", "COL9", "COL10");
|
assertThat(table.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL8", "COL9", "COL10", "COL11", "COL12");
|
||||||
// ID, primary key
|
// ID, primary key
|
||||||
assertThat(table.columnWithName("ID").position()).isEqualTo(1);
|
assertThat(table.columnWithName("ID").position()).isEqualTo(1);
|
||||||
assertThat(table.isPrimaryKeyColumn("ID"));
|
assertThat(table.isPrimaryKeyColumn("ID"));
|
||||||
@ -64,26 +65,31 @@ public void shouldParseCreateAndAlterTable() throws Exception {
|
|||||||
testColumn(table, "COL5", true, Types.NCHAR, "NCHAR", 1, 0, true, null);
|
testColumn(table, "COL5", true, Types.NCHAR, "NCHAR", 1, 0, true, null);
|
||||||
// float(126)
|
// float(126)
|
||||||
testColumn(table, "COL6", true, Types.FLOAT, "FLOAT", 126, 0, true, null);
|
testColumn(table, "COL6", true, Types.FLOAT, "FLOAT", 126, 0, true, null);
|
||||||
|
// todo: DBZ-137 removed
|
||||||
// date
|
// date
|
||||||
testColumn(table, "COL7", true, Types.TIMESTAMP, "DATE", -1, null, true, null);
|
// testColumn(table, "COL7", true, Types.TIMESTAMP, "DATE", -1, null, true, null);
|
||||||
// timestamp
|
// timestamp
|
||||||
testColumn(table, "COL8", true, Types.TIMESTAMP, "TIMESTAMP", 6, null, true, null);
|
testColumn(table, "COL8", true, Types.TIMESTAMP, "TIMESTAMP", 6, null, true, null);
|
||||||
// blob
|
// blob
|
||||||
testColumn(table, "COL9", true, Types.BLOB, "BLOB", -1, null, true, null);
|
testColumn(table, "COL9", true, Types.BLOB, "BLOB", -1, null, true, null);
|
||||||
// clob
|
// clob
|
||||||
testColumn(table, "COL10", true, Types.CLOB, "CLOB", -1, null, true, null);
|
testColumn(table, "COL10", true, Types.CLOB, "CLOB", -1, null, true, null);
|
||||||
// todo sdo_geometry
|
// sdo_geometry
|
||||||
// testColumn(table, "col12", true, Types.STRUCT, "MDSYS.SDO_GEOMETRY", -1, null,true);
|
testColumn(table, "col11", true, Types.STRUCT, "MDSYS.SDO_GEOMETRY", -1, null, true, null);
|
||||||
|
// number(1,0)
|
||||||
|
testColumn(table, "col12", true, Types.NUMERIC, "NUMBER", 1, 0, true, null);
|
||||||
|
|
||||||
String ddl = "alter table " + TABLE_NAME + " add (col21 varchar2(20), col22 number(19));";
|
String ddl = "alter table " + TABLE_NAME + " add (col21 varchar2(20), col22 number(19));";
|
||||||
parser.parse(ddl, tables);
|
parser.parse(ddl, tables);
|
||||||
Table alteredTable = tables.forTable(new TableId(null, null, TABLE_NAME));
|
Table alteredTable = tables.forTable(new TableId(null, null, TABLE_NAME));
|
||||||
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL7", "COL8", "COL9", "COL10", "COL21",
|
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL8", "COL9", "COL10", "COL11", "COL12",
|
||||||
|
"COL21",
|
||||||
"COL22");
|
"COL22");
|
||||||
// varchar2(255)
|
// varchar2(255)
|
||||||
testColumn(alteredTable, "COL21", true, Types.VARCHAR, "VARCHAR2", 20, null, true, null);
|
testColumn(alteredTable, "COL21", true, Types.VARCHAR, "VARCHAR2", 20, null, true, null);
|
||||||
testColumn(alteredTable, "COL22", true, Types.NUMERIC, "NUMBER", 19, 0, true, null);
|
testColumn(alteredTable, "COL22", true, Types.NUMERIC, "NUMBER", 19, 0, true, null);
|
||||||
|
|
||||||
|
// todo check real LogMiner entry, maybe this entry never happens
|
||||||
ddl = "alter table " + TABLE_NAME + " add col23 varchar2(20);";
|
ddl = "alter table " + TABLE_NAME + " add col23 varchar2(20);";
|
||||||
try {
|
try {
|
||||||
parser.parse(ddl, tables);
|
parser.parse(ddl, tables);
|
||||||
@ -95,14 +101,16 @@ public void shouldParseCreateAndAlterTable() throws Exception {
|
|||||||
ddl = "alter table " + TABLE_NAME + " add (col23 varchar2(20) not null);";
|
ddl = "alter table " + TABLE_NAME + " add (col23 varchar2(20) not null);";
|
||||||
parser.parse(ddl, tables);
|
parser.parse(ddl, tables);
|
||||||
alteredTable = tables.forTable(new TableId(null, null, TABLE_NAME));
|
alteredTable = tables.forTable(new TableId(null, null, TABLE_NAME));
|
||||||
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL7", "COL8", "COL9", "COL10", "COL21",
|
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL8", "COL9", "COL10", "COL11", "COL12",
|
||||||
|
"COL21",
|
||||||
"COL22", "COL23");
|
"COL22", "COL23");
|
||||||
testColumn(alteredTable, "COL23", false, Types.VARCHAR, "VARCHAR2", 20, null, false, null);
|
testColumn(alteredTable, "COL23", false, Types.VARCHAR, "VARCHAR2", 20, null, false, null);
|
||||||
|
|
||||||
ddl = "alter table " + TABLE_NAME + " drop (col22, col23);";
|
ddl = "alter table " + TABLE_NAME + " drop (col22, col23);";
|
||||||
parser.parse(ddl, tables);
|
parser.parse(ddl, tables);
|
||||||
alteredTable = tables.forTable(new TableId(null, null, TABLE_NAME));
|
alteredTable = tables.forTable(new TableId(null, null, TABLE_NAME));
|
||||||
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL7", "COL8", "COL9", "COL10", "COL21");
|
assertThat(alteredTable.retrieveColumnNames()).containsExactly("ID", "COL1", "COL2", "COL3", "COL4", "COL5", "COL6", "COL8", "COL9", "COL10", "COL11", "COL12",
|
||||||
|
"COL21");
|
||||||
|
|
||||||
ddl = "drop table " + TABLE_NAME + ";";
|
ddl = "drop table " + TABLE_NAME + ";";
|
||||||
parser.parse(ddl, tables);
|
parser.parse(ddl, tables);
|
||||||
@ -120,8 +128,8 @@ public void shouldParseCreateAndAlterTable() throws Exception {
|
|||||||
@Test
|
@Test
|
||||||
public void shouldParseCreateTable() {
|
public void shouldParseCreateTable() {
|
||||||
|
|
||||||
parser.setCurrentDatabase("ORCLPDB1");
|
parser.setCurrentDatabase(PDB_NAME);
|
||||||
parser.setCurrentSchema(TABLE_NAME);
|
parser.setCurrentSchema("DEBEZIUM");
|
||||||
|
|
||||||
String CREATE_SIMPLE_TABLE = "create table debezium.customer (" +
|
String CREATE_SIMPLE_TABLE = "create table debezium.customer (" +
|
||||||
" id int not null, " +
|
" id int not null, " +
|
||||||
@ -131,7 +139,7 @@ public void shouldParseCreateTable() {
|
|||||||
" primary key (id)" +
|
" primary key (id)" +
|
||||||
");";
|
");";
|
||||||
parser.parse(CREATE_SIMPLE_TABLE, tables);
|
parser.parse(CREATE_SIMPLE_TABLE, tables);
|
||||||
Table table = tables.forTable(new TableId("ORCLPDB1", "DEBEZIUM", "CUSTOMER"));
|
Table table = tables.forTable(new TableId(PDB_NAME, "DEBEZIUM", "CUSTOMER"));
|
||||||
|
|
||||||
assertThat(table).isNotNull();
|
assertThat(table).isNotNull();
|
||||||
|
|
||||||
|
@ -47,11 +47,11 @@ public void before() throws SQLException {
|
|||||||
"CREATE TABLE debezium.tableb (id numeric(9,0) not null, colb varchar2(30), primary key(id))",
|
"CREATE TABLE debezium.tableb (id numeric(9,0) not null, colb varchar2(30), primary key(id))",
|
||||||
"CREATE TABLE debezium.tablec (id numeric(9,0) not null, colc varchar2(30), primary key(id))");
|
"CREATE TABLE debezium.tablec (id numeric(9,0) not null, colc varchar2(30), primary key(id))");
|
||||||
|
|
||||||
connection.execute("GRANT SELECT ON debezium.tablea to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.tablea to " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.tablea ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.tablea ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
connection.execute("GRANT SELECT ON debezium.tableb to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.tableb to " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.tableb ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.tableb ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
connection.execute("GRANT SELECT ON debezium.tablec to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.tablec to " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.tablec ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.tablec ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
|
|
||||||
initializeConnectorTestFramework();
|
initializeConnectorTestFramework();
|
||||||
|
@ -59,6 +59,7 @@ public void schemaIsCorrect() {
|
|||||||
.field("table", Schema.STRING_SCHEMA)
|
.field("table", Schema.STRING_SCHEMA)
|
||||||
.field("txId", Schema.OPTIONAL_STRING_SCHEMA)
|
.field("txId", Schema.OPTIONAL_STRING_SCHEMA)
|
||||||
.field("scn", Schema.OPTIONAL_INT64_SCHEMA)
|
.field("scn", Schema.OPTIONAL_INT64_SCHEMA)
|
||||||
|
.field("commit_scn", Schema.OPTIONAL_INT64_SCHEMA)
|
||||||
.field("lcr_position", Schema.OPTIONAL_STRING_SCHEMA)
|
.field("lcr_position", Schema.OPTIONAL_STRING_SCHEMA)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
@ -9,10 +9,14 @@
|
|||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
|
||||||
import io.debezium.config.Configuration;
|
import io.debezium.config.Configuration;
|
||||||
import io.debezium.config.Configuration.Builder;
|
import io.debezium.config.Configuration.Builder;
|
||||||
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIs;
|
||||||
import io.debezium.connector.oracle.util.TestHelper;
|
import io.debezium.connector.oracle.util.TestHelper;
|
||||||
import io.debezium.util.Testing;
|
import io.debezium.util.Testing;
|
||||||
|
|
||||||
@ -21,8 +25,12 @@
|
|||||||
*
|
*
|
||||||
* @author Jiri Pechanec
|
* @author Jiri Pechanec
|
||||||
*/
|
*/
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Implementation does not support creating/updating schema during streaming")
|
||||||
public class StreamingDatatypesIT extends AbstractOracleDatatypesTest {
|
public class StreamingDatatypesIT extends AbstractOracleDatatypesTest {
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void before() throws Exception {
|
public void before() throws Exception {
|
||||||
setConsumeTimeout(TestHelper.defaultMessageConsumerPollTimeout(), TimeUnit.SECONDS);
|
setConsumeTimeout(TestHelper.defaultMessageConsumerPollTimeout(), TimeUnit.SECONDS);
|
||||||
|
@ -17,10 +17,14 @@
|
|||||||
import org.junit.AfterClass;
|
import org.junit.AfterClass;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
|
||||||
import io.debezium.config.Configuration;
|
import io.debezium.config.Configuration;
|
||||||
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
import io.debezium.connector.oracle.OracleConnectorConfig.SnapshotMode;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIs;
|
||||||
import io.debezium.connector.oracle.util.TestHelper;
|
import io.debezium.connector.oracle.util.TestHelper;
|
||||||
import io.debezium.data.VerifyRecord;
|
import io.debezium.data.VerifyRecord;
|
||||||
import io.debezium.embedded.AbstractConnectorTest;
|
import io.debezium.embedded.AbstractConnectorTest;
|
||||||
@ -36,6 +40,9 @@ public class TransactionMetadataIT extends AbstractConnectorTest {
|
|||||||
|
|
||||||
private static OracleConnection connection;
|
private static OracleConnection connection;
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void beforeClass() throws SQLException {
|
public static void beforeClass() throws SQLException {
|
||||||
connection = TestHelper.testConnection();
|
connection = TestHelper.testConnection();
|
||||||
@ -51,7 +58,7 @@ public static void beforeClass() throws SQLException {
|
|||||||
")";
|
")";
|
||||||
|
|
||||||
connection.execute(ddl);
|
connection.execute(ddl);
|
||||||
connection.execute("GRANT SELECT ON debezium.customer to " + TestHelper.CONNECTOR_USER);
|
connection.execute("GRANT SELECT ON debezium.customer to " + TestHelper.getConnectorUserName());
|
||||||
connection.execute("ALTER TABLE debezium.customer ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
connection.execute("ALTER TABLE debezium.customer ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS");
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -71,6 +78,7 @@ public void before() throws SQLException {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "End transaction is not emitted")
|
||||||
public void transactionMetadata() throws Exception {
|
public void transactionMetadata() throws Exception {
|
||||||
Configuration config = TestHelper.defaultConfig()
|
Configuration config = TestHelper.defaultConfig()
|
||||||
.with(OracleConnectorConfig.TABLE_INCLUDE_LIST, "DEBEZIUM\\.CUSTOMER")
|
.with(OracleConnectorConfig.TABLE_INCLUDE_LIST, "DEBEZIUM\\.CUSTOMER")
|
||||||
|
@ -0,0 +1,44 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.junit;
|
||||||
|
|
||||||
|
import org.junit.runner.Description;
|
||||||
|
import org.junit.runners.model.Statement;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.util.TestHelper;
|
||||||
|
import io.debezium.junit.AnnotationBasedTestRule;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JUnit rule that skips a test based on the {@link SkipWhenAdapterNameIs} annotation on
|
||||||
|
* either a test method or test class.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
public class SkipTestDependingOnAdapterNameRule extends AnnotationBasedTestRule {
|
||||||
|
|
||||||
|
private static final String adapterName = determineAdapterName();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Statement apply(Statement base, Description description) {
|
||||||
|
SkipWhenAdapterNameIs skipWhenAdpterName = hasAnnotation(description, SkipWhenAdapterNameIs.class);
|
||||||
|
if (skipWhenAdpterName != null && skipWhenAdpterName.value().isEqualTo(adapterName)) {
|
||||||
|
String reasonForSkipping = "Adapter name is " + skipWhenAdpterName.value() + System.lineSeparator() + skipWhenAdpterName.reason();
|
||||||
|
return emptyStatement(reasonForSkipping, description);
|
||||||
|
}
|
||||||
|
|
||||||
|
SkipWhenAdapterNameIsNot skipWhenAdapterNameNot = hasAnnotation(description, SkipWhenAdapterNameIsNot.class);
|
||||||
|
if (skipWhenAdapterNameNot != null && skipWhenAdapterNameNot.value().isNotEqualTo(adapterName)) {
|
||||||
|
String reasonForSkipping = "Adapter name is not " + skipWhenAdapterNameNot.value() + System.lineSeparator() + skipWhenAdapterNameNot.reason();
|
||||||
|
return emptyStatement(reasonForSkipping, description);
|
||||||
|
}
|
||||||
|
|
||||||
|
return base;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String determineAdapterName() {
|
||||||
|
return TestHelper.adapter().getValue();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,46 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.junit;
|
||||||
|
|
||||||
|
import java.lang.annotation.ElementType;
|
||||||
|
import java.lang.annotation.Retention;
|
||||||
|
import java.lang.annotation.RetentionPolicy;
|
||||||
|
import java.lang.annotation.Target;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Marker annotation used together with the {@link SkipTestDependingOnAdapterNameRule} JUnit rule, that allows
|
||||||
|
* tests to be skipped based on the adapter name that is being used for testing.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
@Retention(RetentionPolicy.RUNTIME)
|
||||||
|
@Target({ ElementType.METHOD, ElementType.TYPE })
|
||||||
|
public @interface SkipWhenAdapterNameIs {
|
||||||
|
|
||||||
|
SkipWhenAdapterNameIs.AdapterName value();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the reason why the test should be skipped.
|
||||||
|
*/
|
||||||
|
String reason() default "";
|
||||||
|
|
||||||
|
enum AdapterName {
|
||||||
|
XSTREAM {
|
||||||
|
@Override
|
||||||
|
boolean isEqualTo(String adapterName) {
|
||||||
|
return adapterName.equalsIgnoreCase("xstream");
|
||||||
|
}
|
||||||
|
},
|
||||||
|
LOGMINER {
|
||||||
|
@Override
|
||||||
|
boolean isEqualTo(String adapterName) {
|
||||||
|
return adapterName.equalsIgnoreCase("logminer");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
abstract boolean isEqualTo(String adapterName);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,46 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.junit;
|
||||||
|
|
||||||
|
import java.lang.annotation.ElementType;
|
||||||
|
import java.lang.annotation.Retention;
|
||||||
|
import java.lang.annotation.RetentionPolicy;
|
||||||
|
import java.lang.annotation.Target;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Marker annotation used togehter with {@link SkipTestDependingOnAdapterNameRule} JUnit rule, that allows
|
||||||
|
* tests to not be skipped based on the adapter name that is being used for testing.
|
||||||
|
*
|
||||||
|
* @author Chris Cranford
|
||||||
|
*/
|
||||||
|
@Retention(RetentionPolicy.RUNTIME)
|
||||||
|
@Target({ ElementType.METHOD, ElementType.TYPE })
|
||||||
|
public @interface SkipWhenAdapterNameIsNot {
|
||||||
|
|
||||||
|
SkipWhenAdapterNameIsNot.AdapterName value();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the reason why the test should be skipped.
|
||||||
|
*/
|
||||||
|
String reason() default "";
|
||||||
|
|
||||||
|
enum AdapterName {
|
||||||
|
XSTREAM {
|
||||||
|
@Override
|
||||||
|
boolean isNotEqualTo(String adapterName) {
|
||||||
|
return !adapterName.equalsIgnoreCase("xstream");
|
||||||
|
}
|
||||||
|
},
|
||||||
|
LOGMINER {
|
||||||
|
@Override
|
||||||
|
boolean isNotEqualTo(String adapterName) {
|
||||||
|
return !adapterName.equalsIgnoreCase("logminer");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
abstract boolean isNotEqualTo(String adapterName);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,96 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import static org.fest.assertions.Assertions.assertThat;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashSet;
|
||||||
|
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
|
import io.debezium.connector.common.CdcSourceTaskContext;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||||
|
|
||||||
|
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||||
|
public class LogMinerMetricsTest {
|
||||||
|
|
||||||
|
private LogMinerMetrics metrics;
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void before() {
|
||||||
|
CdcSourceTaskContext taskContext = mock(CdcSourceTaskContext.class);
|
||||||
|
Mockito.when(taskContext.getConnectorName()).thenReturn("connector name");
|
||||||
|
Mockito.when(taskContext.getConnectorType()).thenReturn("connector type");
|
||||||
|
metrics = new LogMinerMetrics(taskContext);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMetrics() {
|
||||||
|
|
||||||
|
metrics.incrementCapturedDmlCount();
|
||||||
|
assertThat(metrics.getCapturedDmlCount() == 1).isTrue();
|
||||||
|
|
||||||
|
metrics.setCurrentScn(1000L);
|
||||||
|
assertThat(metrics.getCurrentScn() == 1000L).isTrue();
|
||||||
|
|
||||||
|
metrics.setBatchSize(10);
|
||||||
|
assertThat(metrics.getBatchSize() == 5_000).isTrue();
|
||||||
|
metrics.setBatchSize(1_000_000);
|
||||||
|
assertThat(metrics.getBatchSize() == 5_000).isTrue();
|
||||||
|
metrics.setBatchSize(6000);
|
||||||
|
assertThat(metrics.getBatchSize() == 6_000).isTrue();
|
||||||
|
|
||||||
|
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 1000).isTrue();
|
||||||
|
metrics.changeSleepingTime(true);
|
||||||
|
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 1200).isTrue();
|
||||||
|
metrics.changeSleepingTime(false);
|
||||||
|
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 1000).isTrue();
|
||||||
|
metrics.setMillisecondToSleepBetweenMiningQuery(20);
|
||||||
|
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 1000).isTrue();
|
||||||
|
metrics.setMillisecondToSleepBetweenMiningQuery(4000);
|
||||||
|
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 1000).isTrue();
|
||||||
|
metrics.setMillisecondToSleepBetweenMiningQuery(2000);
|
||||||
|
assertThat(metrics.getMillisecondToSleepBetweenMiningQuery() == 2000).isTrue();
|
||||||
|
|
||||||
|
metrics.setLastLogMinerQueryDuration(Duration.ofMillis(100));
|
||||||
|
assertThat(metrics.getLastLogMinerQueryDuration() == 100).isTrue();
|
||||||
|
metrics.setLastLogMinerQueryDuration(Duration.ofMillis(200));
|
||||||
|
assertThat(metrics.getLastLogMinerQueryDuration() == 200).isTrue();
|
||||||
|
assertThat(metrics.getAverageLogMinerQueryDuration() == 150).isTrue();
|
||||||
|
assertThat(metrics.getLogMinerQueryCount() == 2).isTrue();
|
||||||
|
|
||||||
|
metrics.setCurrentLogFileName(new HashSet<>(Arrays.asList("name", "name1")));
|
||||||
|
assertThat(metrics.getCurrentRedoLogFileName()[0].equals("name")).isTrue();
|
||||||
|
assertThat(metrics.getCurrentRedoLogFileName()[1].equals("name1")).isTrue();
|
||||||
|
|
||||||
|
metrics.setSwitchCount(5);
|
||||||
|
assertThat(metrics.getSwitchCounter() == 5).isTrue();
|
||||||
|
|
||||||
|
metrics.setProcessedCapturedBatchDuration(Duration.ofMillis(1000));
|
||||||
|
assertThat(metrics.getLastProcessedCapturedBatchDuration() == 1000).isTrue();
|
||||||
|
assertThat(metrics.getProcessedCapturedBatchCount() == 1).isTrue();
|
||||||
|
assertThat(metrics.getAverageProcessedCapturedBatchDuration() == 1000).isTrue();
|
||||||
|
|
||||||
|
metrics.setRedoLogStatus(Collections.singletonMap("name", "current"));
|
||||||
|
assertThat(metrics.getRedoLogStatus()[0].equals("name | current")).isTrue();
|
||||||
|
|
||||||
|
assertThat(metrics.toString().contains("logMinerQueryCount"));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,132 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import static org.fest.assertions.Assertions.assertThat;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||||
|
import io.debezium.connector.oracle.OracleSnapshotChangeEventSource;
|
||||||
|
import io.debezium.connector.oracle.antlr.OracleDdlParser;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||||
|
import io.debezium.relational.Table;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
import io.debezium.relational.Tables;
|
||||||
|
import io.debezium.util.IoUtil;
|
||||||
|
|
||||||
|
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||||
|
public class LogMinerUtilsTest {
|
||||||
|
|
||||||
|
private static final BigDecimal SCN = BigDecimal.ONE;
|
||||||
|
private static final BigDecimal OTHER_SCN = BigDecimal.TEN;
|
||||||
|
private OracleDdlParser ddlParser;
|
||||||
|
private Tables tables;
|
||||||
|
private static final String TABLE_NAME = "TEST";
|
||||||
|
private static final String CATALOG_NAME = "ORCLPDB1";
|
||||||
|
private static final String SCHEMA_NAME = "DEBEZIUM";
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testStartLogMinerStatement() {
|
||||||
|
String statement = SqlUtils.getStartLogMinerStatement(SCN.longValue(), OTHER_SCN.longValue(), OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO, false);
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isTrue();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isTrue();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isFalse();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isFalse();
|
||||||
|
statement = SqlUtils.getStartLogMinerStatement(SCN.longValue(), OTHER_SCN.longValue(), OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG, false);
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isFalse();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isFalse();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isTrue();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isFalse();
|
||||||
|
statement = SqlUtils.getStartLogMinerStatement(SCN.longValue(), OTHER_SCN.longValue(), OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO, true);
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isTrue();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isTrue();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isFalse();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isTrue();
|
||||||
|
statement = SqlUtils.getStartLogMinerStatement(SCN.longValue(), OTHER_SCN.longValue(), OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG, true);
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_REDO_LOGS")).isFalse();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DDL_DICT_TRACKING")).isFalse();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG")).isTrue();
|
||||||
|
assertThat(statement.contains("DBMS_LOGMNR.CONTINUOUS_MINE")).isTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBlacklistFiltering() throws Exception {
|
||||||
|
|
||||||
|
ddlParser = new OracleDdlParser(true, CATALOG_NAME, SCHEMA_NAME);
|
||||||
|
tables = new Tables();
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
Table table = tables.forTable(new TableId(CATALOG_NAME, SCHEMA_NAME, TABLE_NAME));
|
||||||
|
|
||||||
|
String prefix = CATALOG_NAME + "." + TABLE_NAME + ".";
|
||||||
|
String blacklistedColumns = prefix + "COL2," + prefix + "COL3";
|
||||||
|
String whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||||
|
assertThat(whitelistedColumns.contains("COL2")).isFalse();
|
||||||
|
assertThat(whitelistedColumns.contains("COL3")).isFalse();
|
||||||
|
assertThat(whitelistedColumns.contains("COL4")).isTrue();
|
||||||
|
|
||||||
|
prefix = TABLE_NAME + ".";
|
||||||
|
blacklistedColumns = prefix + "COL2," + prefix + "COL3";
|
||||||
|
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns.toLowerCase(), table);
|
||||||
|
assertThat(whitelistedColumns.contains("COL2")).isFalse();
|
||||||
|
assertThat(whitelistedColumns.contains("COL3")).isFalse();
|
||||||
|
assertThat(whitelistedColumns.contains("COL4")).isTrue();
|
||||||
|
|
||||||
|
prefix = "";
|
||||||
|
blacklistedColumns = prefix + "COL2," + prefix + "COL3";
|
||||||
|
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||||
|
assertThat(whitelistedColumns.equals("*")).isTrue();
|
||||||
|
|
||||||
|
prefix = "NONEXISTINGTABLE.";
|
||||||
|
blacklistedColumns = prefix + "COL2," + prefix + "COL3";
|
||||||
|
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||||
|
assertThat(whitelistedColumns.equals("*")).isTrue();
|
||||||
|
|
||||||
|
prefix = TABLE_NAME + ".";
|
||||||
|
blacklistedColumns = prefix + "col2," + prefix + "CO77";
|
||||||
|
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||||
|
assertThat(whitelistedColumns.contains("COL2")).isFalse();
|
||||||
|
assertThat(whitelistedColumns.contains("CO77")).isFalse();
|
||||||
|
assertThat(whitelistedColumns.contains("COL4")).isTrue();
|
||||||
|
|
||||||
|
blacklistedColumns = "";
|
||||||
|
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||||
|
assertThat(whitelistedColumns.equals("*")).isTrue();
|
||||||
|
|
||||||
|
blacklistedColumns = null;
|
||||||
|
whitelistedColumns = OracleSnapshotChangeEventSource.buildSelectColumns(blacklistedColumns, table);
|
||||||
|
assertThat(whitelistedColumns.equals("*")).isTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
// todo delete after replacement == -1 in the code
|
||||||
|
@Test
|
||||||
|
public void testConversion() {
|
||||||
|
Map<String, String> map = new HashMap<>();
|
||||||
|
map.put("one", "1001");
|
||||||
|
map.put("two", "1002");
|
||||||
|
map.put("three", "1007");
|
||||||
|
map.put("four", "18446744073709551615");
|
||||||
|
Map<String, Long> res = map.entrySet().stream()
|
||||||
|
.filter(entry -> new BigDecimal(entry.getValue()).longValue() > 1003 || new BigDecimal(entry.getValue()).longValue() == -1).collect(Collectors
|
||||||
|
.toMap(Map.Entry::getKey, e -> new BigDecimal(e.getValue()).longValue() == -1 ? Long.MAX_VALUE : new BigInteger(e.getValue()).longValue()));
|
||||||
|
|
||||||
|
assertThat(res).isNotEmpty();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,454 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import static org.fest.assertions.Assertions.assertThat;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.antlr.OracleDdlParser;
|
||||||
|
import io.debezium.connector.oracle.antlr.OracleDmlParser;
|
||||||
|
import io.debezium.connector.oracle.jsqlparser.SimpleDmlParser;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||||
|
import io.debezium.data.Envelope;
|
||||||
|
import io.debezium.relational.Tables;
|
||||||
|
import io.debezium.util.IoUtil;
|
||||||
|
|
||||||
|
import net.sf.jsqlparser.statement.update.Update;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is the test suite for Oracle Antlr and jsqlparser DML parser unit testing
|
||||||
|
*/
|
||||||
|
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||||
|
public class OracleDmlParserTest {
|
||||||
|
|
||||||
|
private OracleDdlParser ddlParser;
|
||||||
|
private OracleDmlParser antlrDmlParser;
|
||||||
|
private SimpleDmlParser sqlDmlParser;
|
||||||
|
private Tables tables;
|
||||||
|
private static final String TABLE_NAME = "TEST";
|
||||||
|
private static final String CATALOG_NAME = "ORCLPDB1";
|
||||||
|
private static final String SCHEMA_NAME = "DEBEZIUM";
|
||||||
|
private static final String FULL_TABLE_NAME = SCHEMA_NAME + "\".\"" + TABLE_NAME;
|
||||||
|
private static final String SPATIAL_DATA = "SDO_GEOMETRY(2003, NULL, NULL, SDO_ELEM_INFO_ARRAY(1, 1003, 1), SDO_ORDINATE_ARRAY" +
|
||||||
|
"(102604.878, 85772.8286, 101994.879, 85773.6633, 101992.739, 84209.6648, 102602.738, 84208.83, 102604.878, 85772.8286))";
|
||||||
|
private static final String SPATIAL_DATA_1 = "'unsupported type'";
|
||||||
|
private static String CLOB_DATA;
|
||||||
|
private static byte[] BLOB_DATA; // todo
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() {
|
||||||
|
OracleChangeRecordValueConverter converters = new OracleChangeRecordValueConverter(null);
|
||||||
|
|
||||||
|
ddlParser = new OracleDdlParser(true, CATALOG_NAME, SCHEMA_NAME);
|
||||||
|
antlrDmlParser = new OracleDmlParser(true, CATALOG_NAME, SCHEMA_NAME, converters);
|
||||||
|
sqlDmlParser = new SimpleDmlParser(CATALOG_NAME, SCHEMA_NAME, converters);
|
||||||
|
tables = new Tables();
|
||||||
|
|
||||||
|
CLOB_DATA = StringUtils.repeat("clob_", 4000);
|
||||||
|
String blobString = "blob_";
|
||||||
|
BLOB_DATA = Arrays.copyOf(blobString.getBytes(), 8000); // todo doesn't support blob
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldParseAliasUpdate() throws Exception {
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
|
||||||
|
String dml = "update \"" + FULL_TABLE_NAME + "\" a set a.\"col1\" = '9', a.col2 = 'diFFerent', a.col3 = 'anotheR', a.col4 = '123', a.col6 = 5.2, " +
|
||||||
|
"a.col8 = TO_TIMESTAMP('2019-05-14 02:28:32.302000'), a.col10 = " + CLOB_DATA + ", a.col11 = null, a.col12 = '1' " +
|
||||||
|
"where a.ID = 5 and a.COL1 = 6 and a.\"COL2\" = 'text' " +
|
||||||
|
"and a.COL3 = 'text' and a.COL4 IS NULL and a.\"COL5\" IS NULL and a.COL6 IS NULL " +
|
||||||
|
"and a.COL8 = TO_TIMESTAMP('2019-05-14 02:28:32.') and a.col11 is null;";
|
||||||
|
|
||||||
|
antlrDmlParser.parse(dml, tables);
|
||||||
|
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||||
|
verifyUpdate(record, false, true, 9);
|
||||||
|
|
||||||
|
record = sqlDmlParser.parse(dml, tables, "1");
|
||||||
|
verifyUpdate(record, false, true, 9);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldParseAliasInsert() throws Exception {
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
|
||||||
|
String dml = "insert into \"" + FULL_TABLE_NAME + "\" a (a.\"ID\",a.\"COL1\",a.\"COL2\",a.\"COL3\",a.\"COL4\",a.\"COL5\",a.\"COL6\",a.\"COL8\"," +
|
||||||
|
"a.\"COL9\",a.\"COL10\") values ('5','4','tExt','text',NULL,NULL,NULL,NULL,EMPTY_BLOB(),EMPTY_CLOB());";
|
||||||
|
antlrDmlParser.parse(dml, tables);
|
||||||
|
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||||
|
verifyInsert(record);
|
||||||
|
|
||||||
|
record = sqlDmlParser.parse(dml, tables, "1");
|
||||||
|
verifyInsert(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldParseAliasDelete() throws Exception {
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
|
||||||
|
String dml = "delete from \"" + FULL_TABLE_NAME +
|
||||||
|
"\" a where a.\"id\" = 6 and a.\"col1\" = 2 and a.\"col2\" = 'text' and a.col3 = 'tExt' and a.col4 is null and a.col5 is null " +
|
||||||
|
" and a.col6 is null and a.col8 is null and a.col9 is null and a.col10 is null and a.col11 is null and a.col12 is null";
|
||||||
|
antlrDmlParser.parse(dml, tables);
|
||||||
|
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||||
|
verifyDelete(record, true);
|
||||||
|
|
||||||
|
record = sqlDmlParser.parse(dml, tables, "1");
|
||||||
|
verifyDelete(record, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldParseNoWhereClause() throws Exception {
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
|
||||||
|
String dml = "update \"" + FULL_TABLE_NAME
|
||||||
|
+ "\" a set a.\"id\"=1, a.\"col1\" = '9', a.col2 = 'diFFerent', a.col3 = 'anotheR', a.col4 = '123', a.col5 = null, a.col6 = 5.2, " +
|
||||||
|
"a.col8 = TO_TIMESTAMP('2019-05-14 02:28:32.302000'), a.col9=null, a.col10 = " + CLOB_DATA + ", a.col11 = null, a.col12 = '1'";
|
||||||
|
|
||||||
|
antlrDmlParser.parse(dml, tables);
|
||||||
|
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||||
|
verifyUpdate(record, false, false, 9);
|
||||||
|
|
||||||
|
record = sqlDmlParser.parse(dml, tables, "1");
|
||||||
|
verifyUpdate(record, false, false, 9);
|
||||||
|
|
||||||
|
dml = "delete from \"" + FULL_TABLE_NAME + "\" a ";
|
||||||
|
antlrDmlParser.parse(dml, tables);
|
||||||
|
record = antlrDmlParser.getDmlEntry();
|
||||||
|
verifyDelete(record, false);
|
||||||
|
|
||||||
|
record = sqlDmlParser.parse(dml, tables, "1");
|
||||||
|
verifyDelete(record, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldParseInsertAndDeleteTable() throws Exception {
|
||||||
|
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
|
||||||
|
String dml = "insert into \"" + FULL_TABLE_NAME + "\"(\"ID\",\"COL1\",\"COL2\",\"COL3\",\"COL4\",\"COL5\",\"COL6\",\"COL8\"," +
|
||||||
|
"\"COL9\",\"COL10\") values ('5','4','tExt','text',NULL,NULL,NULL,NULL,EMPTY_BLOB(),EMPTY_CLOB());";
|
||||||
|
antlrDmlParser.parse(dml, tables);
|
||||||
|
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||||
|
verifyInsert(record);
|
||||||
|
|
||||||
|
record = sqlDmlParser.parse(dml, tables, "1");
|
||||||
|
verifyInsert(record);
|
||||||
|
|
||||||
|
dml = "delete from \"" + FULL_TABLE_NAME +
|
||||||
|
"\" where id = 6 and col1 = 2 and col2 = 'text' and col3 = 'tExt' and col4 is null and col5 is null " +
|
||||||
|
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col12 is null";
|
||||||
|
antlrDmlParser.parse(dml, tables);
|
||||||
|
record = antlrDmlParser.getDmlEntry();
|
||||||
|
verifyDelete(record, true);
|
||||||
|
|
||||||
|
record = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
verifyDelete(record, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
// todo encrypted columns and spatial will be represented as "Unsupported Type"
|
||||||
|
@Test
|
||||||
|
public void shouldParseUpdateTable() throws Exception {
|
||||||
|
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
|
||||||
|
String dml = "update \"" + FULL_TABLE_NAME + "\" set \"col1\" = '9', col2 = 'diFFerent', col3 = 'anotheR', col4 = '123', col6 = '5.2', " +
|
||||||
|
"col8 = TO_TIMESTAMP('2019-05-14 02:28:32.302000'), col10='clob_', col12 = '1' " +
|
||||||
|
"where ID = 5 and COL1 = 6 and \"COL2\" = 'text' " +
|
||||||
|
"and COL3 = 'text' and COL4 IS NULL and \"COL5\" IS NULL and COL6 IS NULL " +
|
||||||
|
"and COL8 = TO_TIMESTAMP('2019-05-14 02:28:32') and col11 = " + SPATIAL_DATA + ";";
|
||||||
|
|
||||||
|
antlrDmlParser.parse(dml, tables);
|
||||||
|
LogMinerDmlEntry record = antlrDmlParser.getDmlEntry();
|
||||||
|
// verifyUpdate(record, true, true);
|
||||||
|
|
||||||
|
record = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
verifyUpdate(record, true, true, 9);
|
||||||
|
|
||||||
|
dml = "update \"" + FULL_TABLE_NAME
|
||||||
|
+ "\" set \"col1\" = '9', col2 = '$2a$10$aHo.lQk.YAkGl5AkXbjJhODBqwNLkqF94slP5oZ3boNzm0d04WnE2', col3 = NULL, col4 = '123', col6 = '5.2', " +
|
||||||
|
"col8 = TO_TIMESTAMP('2019-05-14 02:28:32.302000'), col10='clob_', col12 = '1' " +
|
||||||
|
"where ID = 5 and COL1 = 6 and \"COL2\" = 'johan.philtjens@dpworld.com' " +
|
||||||
|
"and COL3 = 'text' and COL4 IS NULL and \"COL5\" IS NULL and COL6 IS NULL " +
|
||||||
|
"and COL8 = TO_TIMESTAMP('2019-05-14 02:28:32') and col11 = " + SPATIAL_DATA + ";";
|
||||||
|
|
||||||
|
record = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldParseUpdateNoChangesTable() throws Exception {
|
||||||
|
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
|
||||||
|
String dml = "update \"" + FULL_TABLE_NAME + "\" set \"col1\" = '6', col2 = 'text', col3 = 'text', col4 = NULL " +
|
||||||
|
"where ID = 5 and COL1 = 6 and \"COL2\" = 'text' " +
|
||||||
|
"and COL3 = Unsupported Type and COL4 IS NULL and \"COL5\" IS NULL and COL6 IS NULL and COL7 IS NULL and COL9 IS NULL and COL10 IS NULL and COL12 IS NULL "
|
||||||
|
+
|
||||||
|
"and COL8 = TO_TIMESTAMP('2019-05-14 02:28:32') and col11 = " + SPATIAL_DATA + ";";
|
||||||
|
|
||||||
|
LogMinerDmlEntry record = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
boolean pass = record.getCommandType().equals(Envelope.Operation.UPDATE)
|
||||||
|
&& record.getOldValues().size() == record.getNewValues().size()
|
||||||
|
&& record.getNewValues().containsAll(record.getOldValues());
|
||||||
|
assertThat(pass);
|
||||||
|
assertThat(record.getOldValues().get(4).getColumnData()).isNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldParseSpecialCharacters() throws Exception {
|
||||||
|
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
|
||||||
|
String dml = "insert into \"" + FULL_TABLE_NAME + "\"(\"ID\",\"COL1\",\"COL2\",\"COL3\",\"COL4\",\"COL5\",\"COL6\",\"COL8\"," +
|
||||||
|
"\"COL9\",\"COL10\") values ('5','4','\\','\\test',NULL,NULL,NULL,NULL,EMPTY_BLOB(),EMPTY_CLOB());";
|
||||||
|
antlrDmlParser.parse(dml, tables);
|
||||||
|
assertThat(antlrDmlParser.getDmlEntry()).isNotNull();
|
||||||
|
|
||||||
|
LogMinerDmlEntry result = sqlDmlParser.parse(dml, tables, "1");
|
||||||
|
assertThat(result).isNotNull();
|
||||||
|
LogMinerColumnValue value = result.getNewValues().get(2);
|
||||||
|
assertThat(value.getColumnData().toString()).contains("\\");
|
||||||
|
|
||||||
|
dml = "delete from \"" + FULL_TABLE_NAME +
|
||||||
|
"\" where id = 6 and col1 = 2 and col2 = 'te\\xt' and col3 = 'tExt\\' and col4 is null and col5 is null " +
|
||||||
|
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col12 is null";
|
||||||
|
antlrDmlParser.parse(dml, tables);
|
||||||
|
assertThat(antlrDmlParser.getDmlEntry()).isNotNull();
|
||||||
|
|
||||||
|
result = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
assertThat(result).isNotNull();
|
||||||
|
value = result.getOldValues().get(3);
|
||||||
|
assertThat(value.getColumnData().toString()).contains("\\");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldParseStrangeDml() throws Exception {
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
String dml = null;
|
||||||
|
LogMinerDmlEntry result = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
assertThat(result).isNull();
|
||||||
|
dml = "select * from test;null;";
|
||||||
|
result = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
assertThat(result).isNull();
|
||||||
|
dml = "full dummy mess";
|
||||||
|
result = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
assertThat(result).isNull();
|
||||||
|
dml = "delete from non_exiting_table " +
|
||||||
|
" where id = 6 and col1 = 2 and col2 = 'te\\xt' and col3 = 'tExt\\' and col4 is null and col5 is null " +
|
||||||
|
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col12 is null";
|
||||||
|
result = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
assertThat(result).isNull();
|
||||||
|
|
||||||
|
Update update = mock(Update.class);
|
||||||
|
Mockito.when(update.getTables()).thenReturn(new ArrayList<>());
|
||||||
|
dml = "update \"" + FULL_TABLE_NAME + "\" set col1 = 3 " +
|
||||||
|
" where id = 6 and col1 = 2 and col2 = 'te\\xt' and col3 = 'tExt\\' and col4 is null and col5 is null " +
|
||||||
|
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col12 is null and col20 is null";
|
||||||
|
result = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
assertThat(result.getOldValues().size()).isEqualTo(12);
|
||||||
|
assertThat(result.getOldValues().size() == 12).isTrue();
|
||||||
|
|
||||||
|
dml = "update \"" + FULL_TABLE_NAME + "\" set col1 = 3 " +
|
||||||
|
" where id = 6 and col1 = 2 and col2 = 'te\\xt' and col30 = 'tExt\\' and col4 is null and col5 is null " +
|
||||||
|
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col21 is null";
|
||||||
|
result = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
assertThat(result.getNewValues().size() == 12).isTrue();
|
||||||
|
|
||||||
|
dml = "update table1, \"" + FULL_TABLE_NAME + "\" set col1 = 3 " +
|
||||||
|
" where id = 6 and col1 = 2 and col2 = 'te\\xt' and col3 = 'tExt\\' and col4 is null and col5 is null " +
|
||||||
|
" and col6 is null and col8 is null and col9 is null and col10 is null and col11 is null and col12 is null and col20 is null";
|
||||||
|
result = sqlDmlParser.parse(dml, tables, "");
|
||||||
|
assertThat(result).isNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void verifyUpdate(LogMinerDmlEntry record, boolean checkGeometry, boolean checkOldValues, int oldValuesNumber) {
|
||||||
|
// validate
|
||||||
|
assertThat(record.getCommandType()).isEqualTo(Envelope.Operation.UPDATE);
|
||||||
|
List<LogMinerColumnValue> newValues = record.getNewValues();
|
||||||
|
assertThat(newValues.size()).isEqualTo(12);
|
||||||
|
String concatenatedNames = newValues.stream().map(LogMinerColumnValue::getColumnName).collect(Collectors.joining());
|
||||||
|
assertThat("IDCOL1COL2COL3COL4COL5COL6COL8COL9COL10COL11COL12".equals(concatenatedNames));
|
||||||
|
for (LogMinerColumnValue newValue : newValues) {
|
||||||
|
String columnName = newValue.getColumnName();
|
||||||
|
switch (columnName) {
|
||||||
|
case "COL1":
|
||||||
|
assertThat(newValue.getColumnData()).isEqualTo(BigDecimal.valueOf(9, 0));
|
||||||
|
break;
|
||||||
|
case "COL2":
|
||||||
|
assertThat(newValue.getColumnData()).isEqualTo("diFFerent");
|
||||||
|
break;
|
||||||
|
case "COL3":
|
||||||
|
assertThat(newValue.getColumnData()).isEqualTo("anotheR");
|
||||||
|
break;
|
||||||
|
case "COL4":
|
||||||
|
assertThat(newValue.getColumnData()).isEqualTo("123");
|
||||||
|
break;
|
||||||
|
case "COL6":
|
||||||
|
// todo, which one is expected value format
|
||||||
|
assertThat(newValue.getColumnData()).isEqualTo(5.2F);
|
||||||
|
// assertThat(((Struct)newValue.getColumnData()).get("scale")).isEqualTo(1);
|
||||||
|
// assertThat(((byte[])((Struct)newValue.getColumnData()).get("value"))[0]).isEqualTo((byte) 52);
|
||||||
|
break;
|
||||||
|
case "COL8":
|
||||||
|
assertThat(newValue.getColumnData()).isInstanceOf(Long.class);
|
||||||
|
// todo: DBZ-137 value didn't account for values to be GMT.
|
||||||
|
assertThat(newValue.getColumnData()).isEqualTo(1557800912302000L /* 1557826112302000L */);
|
||||||
|
break;
|
||||||
|
case "COL10":
|
||||||
|
assertThat(newValue.getColumnData()).isInstanceOf(String.class);
|
||||||
|
assertThat(newValue.getColumnData().toString().contains("clob_")).isTrue();
|
||||||
|
break;
|
||||||
|
case "COL11":
|
||||||
|
if (checkGeometry) {
|
||||||
|
assertThat(newValue.getColumnData()).isInstanceOf(String.class);
|
||||||
|
assertThat(newValue.getColumnData().toString().contains("SDO_GEOMETRY")).isTrue();
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
assertThat(newValue.getColumnData()).isNull();
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case "COL12":
|
||||||
|
assertThat(newValue.getColumnData()).isInstanceOf(Boolean.class);
|
||||||
|
assertThat(newValue.getColumnData()).isEqualTo(true);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
List<LogMinerColumnValue> oldValues = record.getOldValues();
|
||||||
|
if (!checkOldValues) {
|
||||||
|
assertThat(oldValues.size()).isEqualTo(0);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
assertThat(oldValues.size()).isEqualTo(oldValuesNumber);
|
||||||
|
concatenatedNames = oldValues.stream().map(LogMinerColumnValue::getColumnName).collect(Collectors.joining());
|
||||||
|
assertThat("IDCOL1COL2COL3COL4COL6COL8COL11COL12".equals(concatenatedNames));
|
||||||
|
for (LogMinerColumnValue oldValue : oldValues) {
|
||||||
|
String columnName = oldValue.getColumnName();
|
||||||
|
switch (columnName) {
|
||||||
|
case "COL1":
|
||||||
|
assertThat(oldValue.getColumnData()).isEqualTo(BigDecimal.valueOf(6, 0));
|
||||||
|
break;
|
||||||
|
case "COL2":
|
||||||
|
assertThat(oldValue.getColumnData()).isEqualTo("text");
|
||||||
|
break;
|
||||||
|
case "COL3":
|
||||||
|
assertThat(oldValue.getColumnData()).isEqualTo("text");
|
||||||
|
break;
|
||||||
|
case "COL4":
|
||||||
|
assertThat(oldValue.getColumnData()).isNull();
|
||||||
|
break;
|
||||||
|
case "COL5":
|
||||||
|
assertThat(oldValue.getColumnData()).isNull();
|
||||||
|
break;
|
||||||
|
case "COL6":
|
||||||
|
assertThat(oldValue.getColumnData()).isNull();
|
||||||
|
break;
|
||||||
|
case "COL8":
|
||||||
|
assertThat(oldValue.getColumnData()).isInstanceOf(Long.class);
|
||||||
|
// todo: DBZ-137 value didn't account for values to be GMT.
|
||||||
|
assertThat(oldValue.getColumnData()).isEqualTo(1557800912000000L /* 1557826112000000L */);
|
||||||
|
break;
|
||||||
|
case "COL11":
|
||||||
|
if (checkGeometry) {
|
||||||
|
assertThat(oldValue.getColumnData()).isInstanceOf(String.class);
|
||||||
|
assertThat(oldValue.getColumnData().toString().contains("SDO_GEOMETRY")).isTrue();
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
assertThat(oldValue.getColumnData()).isNull();
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case "ID":
|
||||||
|
assertThat(oldValue.getColumnData()).isEqualTo(new BigDecimal(5));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void verifyInsert(LogMinerDmlEntry record) {
|
||||||
|
List<LogMinerColumnValue> oldValues = record.getOldValues();
|
||||||
|
assertThat(oldValues.size()).isEqualTo(0);
|
||||||
|
|
||||||
|
assertThat(record.getCommandType()).isEqualTo(Envelope.Operation.CREATE);
|
||||||
|
|
||||||
|
List<LogMinerColumnValue> newValues = record.getNewValues();
|
||||||
|
assertThat(newValues.size()).isEqualTo(12);
|
||||||
|
|
||||||
|
Iterator<LogMinerColumnValue> iterator = newValues.iterator();
|
||||||
|
assertThat(iterator.next().getColumnData()).isEqualTo(new BigDecimal(5));
|
||||||
|
assertThat(iterator.next().getColumnData()).isEqualTo(BigDecimal.valueOf(4, 0));
|
||||||
|
assertThat(iterator.next().getColumnData()).isEqualTo("tExt");
|
||||||
|
assertThat(iterator.next().getColumnData()).isEqualTo("text");
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
// todo handle LOBS
|
||||||
|
// assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
// assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private void verifyDelete(LogMinerDmlEntry record, boolean checkOldValues) {
|
||||||
|
assertThat(record.getCommandType()).isEqualTo(Envelope.Operation.DELETE);
|
||||||
|
List<LogMinerColumnValue> newValues = record.getNewValues();
|
||||||
|
assertThat(newValues.size()).isEqualTo(0);
|
||||||
|
|
||||||
|
List<LogMinerColumnValue> oldValues = record.getOldValues();
|
||||||
|
if (!checkOldValues) {
|
||||||
|
assertThat(oldValues.size()).isEqualTo(0);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
// todo: DBZ-137 should not include COL7 but column is included
|
||||||
|
assertThat(oldValues.size()).isEqualTo(12);
|
||||||
|
String concatenatedColumnNames = oldValues.stream().map(LogMinerColumnValue::getColumnName).collect(Collectors.joining());
|
||||||
|
assertThat("IDCOL1COL2COL3COL4COL5COL6COL8COL9COL10COL11COL12".equals(concatenatedColumnNames));
|
||||||
|
|
||||||
|
Iterator<LogMinerColumnValue> iterator = oldValues.iterator();
|
||||||
|
assertThat(iterator.next().getColumnData()).isEqualTo(new BigDecimal(6));
|
||||||
|
assertThat(iterator.next().getColumnData()).isEqualTo(BigDecimal.valueOf(2, 0));
|
||||||
|
assertThat(iterator.next().getColumnData()).isEqualTo("text");
|
||||||
|
assertThat(iterator.next().getColumnData()).isEqualTo("tExt");
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
assertThat(iterator.next().getColumnData()).isNull();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,176 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import static org.fest.assertions.Assertions.assertThat;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.times;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
|
||||||
|
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||||
|
public class RowMapperTest {
|
||||||
|
private ResultSet rs;
|
||||||
|
private TransactionalBufferMetrics metrics;
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void before() {
|
||||||
|
rs = mock(ResultSet.class);
|
||||||
|
metrics = mock(TransactionalBufferMetrics.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testChangeTime() throws SQLException {
|
||||||
|
Mockito.when(rs.getTimestamp(4)).thenReturn(new Timestamp(1000L));
|
||||||
|
Timestamp time = RowMapper.getChangeTime(metrics, rs);
|
||||||
|
assertThat(time.getTime() == 1000L).isTrue();
|
||||||
|
Mockito.when(rs.getTimestamp(4)).thenThrow(SQLException.class);
|
||||||
|
time = RowMapper.getChangeTime(metrics, rs);
|
||||||
|
assertThat(time.getTime() == new Timestamp(Instant.now().getEpochSecond()).getTime()).isTrue();
|
||||||
|
verify(rs, times(2)).getTimestamp(4);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testOperationCode() throws SQLException {
|
||||||
|
Mockito.when(rs.getInt(3)).thenReturn(100);
|
||||||
|
int operation = RowMapper.getOperationCode(metrics, rs);
|
||||||
|
assertThat(operation == 100).isTrue();
|
||||||
|
verify(rs).getInt(3);
|
||||||
|
Mockito.when(rs.getInt(3)).thenThrow(SQLException.class);
|
||||||
|
operation = RowMapper.getOperationCode(metrics, rs);
|
||||||
|
assertThat(operation == 0).isTrue();
|
||||||
|
verify(rs, times(2)).getInt(3);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testTableName() throws SQLException {
|
||||||
|
Mockito.when(rs.getString(7)).thenReturn("table_name");
|
||||||
|
String tableName = RowMapper.getTableName(metrics, rs);
|
||||||
|
assertThat(tableName.equals("table_name")).isTrue();
|
||||||
|
verify(rs).getString(7);
|
||||||
|
Mockito.when(rs.getString(7)).thenThrow(SQLException.class);
|
||||||
|
tableName = RowMapper.getTableName(metrics, rs);
|
||||||
|
assertThat(tableName.equals("")).isTrue();
|
||||||
|
verify(rs, times(2)).getString(7);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSeqOwner() throws SQLException {
|
||||||
|
Mockito.when(rs.getString(8)).thenReturn("owner");
|
||||||
|
String owner = RowMapper.getSegOwner(metrics, rs);
|
||||||
|
assertThat(owner.equals("owner")).isTrue();
|
||||||
|
verify(rs).getString(8);
|
||||||
|
Mockito.when(rs.getString(8)).thenThrow(SQLException.class);
|
||||||
|
owner = RowMapper.getSegOwner(metrics, rs);
|
||||||
|
assertThat(owner.equals("")).isTrue();
|
||||||
|
verify(rs, times(2)).getString(8);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetScn() throws SQLException {
|
||||||
|
Mockito.when(rs.getBigDecimal(1)).thenReturn(new BigDecimal(1));
|
||||||
|
BigDecimal scn = RowMapper.getScn(metrics, rs);
|
||||||
|
assertThat(scn.equals(new BigDecimal(1))).isTrue();
|
||||||
|
verify(rs).getBigDecimal(1);
|
||||||
|
Mockito.when(rs.getBigDecimal(1)).thenThrow(SQLException.class);
|
||||||
|
scn = RowMapper.getScn(metrics, rs);
|
||||||
|
assertThat(scn.equals(new BigDecimal(-1))).isTrue();
|
||||||
|
verify(rs, times(2)).getBigDecimal(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetTransactionId() throws SQLException {
|
||||||
|
Mockito.when(rs.getBytes(5)).thenReturn("tr_id".getBytes());
|
||||||
|
String transactionId = RowMapper.getTransactionId(metrics, rs);
|
||||||
|
assertThat(transactionId.equals("74725F6964")).isTrue();
|
||||||
|
verify(rs).getBytes(5);
|
||||||
|
Mockito.when(rs.getBytes(5)).thenThrow(SQLException.class);
|
||||||
|
transactionId = RowMapper.getTransactionId(metrics, rs);
|
||||||
|
assertThat(transactionId.equals("")).isTrue();
|
||||||
|
verify(rs, times(2)).getBytes(5);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSqlRedo() throws SQLException {
|
||||||
|
Mockito.when(rs.getInt(6)).thenReturn(0);
|
||||||
|
Mockito.when(rs.getString(2)).thenReturn("short_sql");
|
||||||
|
String sql = RowMapper.getSqlRedo(metrics, rs);
|
||||||
|
assertThat(sql.equals("short_sql")).isTrue();
|
||||||
|
verify(rs).getInt(6);
|
||||||
|
verify(rs).getString(2);
|
||||||
|
|
||||||
|
Mockito.when(rs.getInt(6)).thenReturn(1).thenReturn(0);
|
||||||
|
Mockito.when(rs.getString(2)).thenReturn("long").thenReturn("_sql");
|
||||||
|
sql = RowMapper.getSqlRedo(metrics, rs);
|
||||||
|
assertThat(sql.equals("long_sql")).isTrue();
|
||||||
|
verify(rs, times(3)).getInt(6);
|
||||||
|
verify(rs, times(3)).getString(2);
|
||||||
|
|
||||||
|
// test super large DML
|
||||||
|
char[] chars = new char[4000];
|
||||||
|
Arrays.fill(chars, 'a');
|
||||||
|
Mockito.when(rs.getString(2)).thenReturn(new String(chars));
|
||||||
|
Mockito.when(rs.getInt(6)).thenReturn(1);
|
||||||
|
sql = RowMapper.getSqlRedo(metrics, rs);
|
||||||
|
assertThat(sql.length() == 40_000).isTrue();
|
||||||
|
verify(rs, times(13)).getInt(6);
|
||||||
|
verify(rs, times(13)).getString(2);
|
||||||
|
|
||||||
|
Mockito.when(rs.getInt(6)).thenReturn(0);
|
||||||
|
Mockito.when(rs.getString(2)).thenReturn(null);
|
||||||
|
sql = RowMapper.getSqlRedo(metrics, rs);
|
||||||
|
assertThat(sql == null).isTrue();
|
||||||
|
verify(rs, times(13)).getInt(6);
|
||||||
|
verify(rs, times(14)).getString(2);
|
||||||
|
|
||||||
|
Mockito.when(rs.getInt(6)).thenReturn(0);
|
||||||
|
Mockito.when(rs.getString(2)).thenThrow(SQLException.class);
|
||||||
|
sql = RowMapper.getSqlRedo(metrics, rs);
|
||||||
|
assertThat(sql.equals("")).isTrue();
|
||||||
|
verify(rs, times(13)).getInt(6);
|
||||||
|
verify(rs, times(15)).getString(2);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetTableId() throws SQLException {
|
||||||
|
Mockito.when(rs.getString(8)).thenReturn("schema");
|
||||||
|
Mockito.when(rs.getString(7)).thenReturn("table");
|
||||||
|
TableId tableId = RowMapper.getTableId("catalog", rs);
|
||||||
|
assertThat(tableId.toString().equals("CATALOG.SCHEMA.TABLE")).isTrue();
|
||||||
|
verify(rs).getString(8);
|
||||||
|
Mockito.when(rs.getString(8)).thenThrow(SQLException.class);
|
||||||
|
|
||||||
|
tableId = null;
|
||||||
|
try {
|
||||||
|
tableId = RowMapper.getTableId("catalog", rs);
|
||||||
|
assertThat(1 == 2).isTrue();
|
||||||
|
}
|
||||||
|
catch (SQLException e) {
|
||||||
|
assertThat(tableId).isNull();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,69 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import static org.fest.assertions.Assertions.assertThat;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||||
|
import io.debezium.connector.oracle.OracleDatabaseSchema;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||||
|
import io.debezium.relational.TableId;
|
||||||
|
|
||||||
|
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||||
|
public class SqlUtilsTest {
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testStatements() {
|
||||||
|
String result = SqlUtils.getAddLogFileStatement("ADD", "FILENAME");
|
||||||
|
String expected = "BEGIN sys.dbms_logmnr.add_logfile(LOGFILENAME => 'FILENAME', OPTIONS => ADD);END;";
|
||||||
|
assertThat(expected.equals(result)).isTrue();
|
||||||
|
|
||||||
|
OracleDatabaseSchema schema = mock(OracleDatabaseSchema.class);
|
||||||
|
TableId table1 = new TableId("catalog", "schema", "table1");
|
||||||
|
TableId table2 = new TableId("catalog", "schema", "table2");
|
||||||
|
Set<TableId> tables = new HashSet<>();
|
||||||
|
Mockito.when(schema.tableIds()).thenReturn(tables);
|
||||||
|
result = SqlUtils.queryLogMinerContents("DATABASE", "SCHEMA", schema);
|
||||||
|
expected = "SELECT SCN, SQL_REDO, OPERATION_CODE, TIMESTAMP, XID, CSF, TABLE_NAME, SEG_OWNER, OPERATION, USERNAME " +
|
||||||
|
"FROM V$LOGMNR_CONTENTS WHERE OPERATION_CODE in (1,2,3,5) AND SEG_OWNER = 'DATABASE' AND SCN >= ? AND SCN < ? " +
|
||||||
|
"OR (OPERATION_CODE IN (5,7,34,36) AND USERNAME NOT IN ('SYS','SYSTEM','SCHEMA'))ORDER BY SCN";
|
||||||
|
assertThat(expected.equals(result)).isTrue();
|
||||||
|
|
||||||
|
tables.add(table1);
|
||||||
|
tables.add(table2);
|
||||||
|
result = SqlUtils.queryLogMinerContents("DATABASE", "SCHEMA", schema);
|
||||||
|
expected = "SELECT SCN, SQL_REDO, OPERATION_CODE, TIMESTAMP, XID, CSF, TABLE_NAME, SEG_OWNER, OPERATION, USERNAME " +
|
||||||
|
"FROM V$LOGMNR_CONTENTS WHERE OPERATION_CODE in (1,2,3,5) " +
|
||||||
|
"AND SEG_OWNER = 'DATABASE' AND table_name IN ('table1','table2') AND SEG_NAME IN ('table1','table2') " +
|
||||||
|
"AND SCN >= ? AND SCN < ? OR (OPERATION_CODE IN (5,7,34,36) AND USERNAME NOT IN ('SYS','SYSTEM','SCHEMA'))ORDER BY SCN";
|
||||||
|
assertThat(expected.equals(result)).isTrue();
|
||||||
|
|
||||||
|
result = SqlUtils.getStartLogMinerStatement(10L, 20L, OracleConnectorConfig.LogMiningStrategy.ONLINE_CATALOG, true);
|
||||||
|
expected = "BEGIN sys.dbms_logmnr.start_logmnr(startScn => '10', endScn => '20', " +
|
||||||
|
"OPTIONS => DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG + DBMS_LOGMNR.CONTINUOUS_MINE + DBMS_LOGMNR.NO_ROWID_IN_STMT);END;";
|
||||||
|
assertThat(expected.equals(result)).isTrue();
|
||||||
|
|
||||||
|
result = SqlUtils.getStartLogMinerStatement(10L, 20L, OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO, false);
|
||||||
|
expected = "BEGIN sys.dbms_logmnr.start_logmnr(startScn => '10', endScn => '20', " +
|
||||||
|
"OPTIONS => DBMS_LOGMNR.DICT_FROM_REDO_LOGS + DBMS_LOGMNR.DDL_DICT_TRACKING + DBMS_LOGMNR.NO_ROWID_IN_STMT);END;";
|
||||||
|
assertThat(expected.equals(result)).isTrue();
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,150 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import static org.fest.assertions.Assertions.assertThat;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
|
import io.debezium.connector.common.CdcSourceTaskContext;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIs;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||||
|
|
||||||
|
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||||
|
public class TransactionalBufferMetricsTest {
|
||||||
|
|
||||||
|
private TransactionalBufferMetrics metrics;
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void before() {
|
||||||
|
CdcSourceTaskContext taskContext = mock(CdcSourceTaskContext.class);
|
||||||
|
Mockito.when(taskContext.getConnectorName()).thenReturn("connector name");
|
||||||
|
Mockito.when(taskContext.getConnectorType()).thenReturn("connector type");
|
||||||
|
metrics = new TransactionalBufferMetrics(taskContext);
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void after() {
|
||||||
|
metrics.reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@SkipWhenAdapterNameIs(value = SkipWhenAdapterNameIs.AdapterName.LOGMINER, reason = "Lag calculations fail")
|
||||||
|
public void testLagMetrics() {
|
||||||
|
// no time difference between connector and database
|
||||||
|
long lag = metrics.getLagFromSource();
|
||||||
|
assertThat(lag == 0).isTrue();
|
||||||
|
Instant dbEventTime = Instant.now().minusMillis(2000);
|
||||||
|
metrics.calculateLagMetrics(dbEventTime);
|
||||||
|
lag = metrics.getLagFromSource();
|
||||||
|
assertThat(lag == 2000).isTrue();
|
||||||
|
assertThat(metrics.getMaxLagFromSource() == 2000).isTrue();
|
||||||
|
assertThat(metrics.getMinLagFromSource() == 0).isTrue();
|
||||||
|
assertThat(metrics.getAverageLagFromSource() == 2000).isTrue();
|
||||||
|
|
||||||
|
// not realistic scenario
|
||||||
|
dbEventTime = Instant.now().plusMillis(2000);
|
||||||
|
metrics.calculateLagMetrics(dbEventTime);
|
||||||
|
lag = metrics.getLagFromSource();
|
||||||
|
assertThat(lag == -2000).isTrue();
|
||||||
|
assertThat(metrics.getMaxLagFromSource() == 2000).isTrue();
|
||||||
|
assertThat(metrics.getMinLagFromSource() == -2000).isTrue();
|
||||||
|
assertThat(metrics.getAverageLagFromSource() == 0).isTrue();
|
||||||
|
|
||||||
|
metrics.reset();
|
||||||
|
|
||||||
|
// ##########################
|
||||||
|
// the database time is ahead
|
||||||
|
metrics.setTimeDifference(new AtomicLong(-1000));
|
||||||
|
dbEventTime = Instant.now().minusMillis(2000);
|
||||||
|
metrics.calculateLagMetrics(dbEventTime);
|
||||||
|
lag = metrics.getLagFromSource();
|
||||||
|
assertThat(lag == 3000).isTrue();
|
||||||
|
assertThat(metrics.getMaxLagFromSource() == 3000).isTrue();
|
||||||
|
assertThat(metrics.getMinLagFromSource() == 0).isTrue();
|
||||||
|
assertThat(metrics.getAverageLagFromSource() == 3000).isTrue();
|
||||||
|
|
||||||
|
dbEventTime = Instant.now().minusMillis(3000);
|
||||||
|
metrics.calculateLagMetrics(dbEventTime);
|
||||||
|
lag = metrics.getLagFromSource();
|
||||||
|
assertThat(lag == 4000).isTrue();
|
||||||
|
assertThat(metrics.getMaxLagFromSource() == 4000).isTrue();
|
||||||
|
assertThat(metrics.getMinLagFromSource() == 0).isTrue();
|
||||||
|
assertThat(metrics.getAverageLagFromSource() == 3500).isTrue();
|
||||||
|
|
||||||
|
metrics.reset();
|
||||||
|
|
||||||
|
// ##########################
|
||||||
|
// the database time is behind
|
||||||
|
metrics.setTimeDifference(new AtomicLong(1000));
|
||||||
|
dbEventTime = Instant.now().minusMillis(2000);
|
||||||
|
metrics.calculateLagMetrics(dbEventTime);
|
||||||
|
lag = metrics.getLagFromSource();
|
||||||
|
assertThat(lag == 1000).isTrue();
|
||||||
|
assertThat(metrics.getMaxLagFromSource() == 1000).isTrue();
|
||||||
|
assertThat(metrics.getMinLagFromSource() == 0).isTrue();
|
||||||
|
assertThat(metrics.getAverageLagFromSource() == 1000).isTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testOtherMetrics() {
|
||||||
|
metrics.incrementScnFreezeCounter();
|
||||||
|
assertThat(metrics.getScnFreezeCounter() == 1).isTrue();
|
||||||
|
|
||||||
|
metrics.incrementErrorCounter();
|
||||||
|
assertThat(metrics.getErrorCounter() == 1).isTrue();
|
||||||
|
|
||||||
|
metrics.incrementWarningCounter();
|
||||||
|
assertThat(metrics.getWarningCounter() == 1).isTrue();
|
||||||
|
|
||||||
|
metrics.incrementCommittedDmlCounter(5_000);
|
||||||
|
for (int i = 0; i < 1000; i++) {
|
||||||
|
metrics.incrementCapturedDmlCounter();
|
||||||
|
metrics.incrementCommittedTransactions();
|
||||||
|
}
|
||||||
|
assertThat(metrics.getCapturedDmlCount() == 1000).isTrue();
|
||||||
|
assertThat(metrics.getCapturedDmlThroughput() > 10_000).isTrue();
|
||||||
|
assertThat(metrics.getNumberOfCommittedTransactions() == 1000).isTrue();
|
||||||
|
assertThat(metrics.getCommitThroughput() >= 1_000).isTrue();
|
||||||
|
|
||||||
|
metrics.incrementRolledBackTransactions();
|
||||||
|
assertThat(metrics.getNumberOfRolledBackTransactions() == 1).isTrue();
|
||||||
|
|
||||||
|
metrics.setActiveTransactions(5);
|
||||||
|
assertThat(metrics.getNumberOfActiveTransactions() == 5).isTrue();
|
||||||
|
|
||||||
|
metrics.addRolledBackTransactionId("rolledback id");
|
||||||
|
assertThat(metrics.getNumberOfRolledBackTransactions() == 1).isTrue();
|
||||||
|
assertThat(metrics.getRolledBackTransactionIds().contains("rolledback id")).isTrue();
|
||||||
|
|
||||||
|
metrics.addAbandonedTransactionId("abandoned id");
|
||||||
|
assertThat(metrics.getAbandonedTransactionIds().size() == 1).isTrue();
|
||||||
|
assertThat(metrics.getAbandonedTransactionIds().contains("abandoned id")).isTrue();
|
||||||
|
|
||||||
|
metrics.setOldestScn(10L);
|
||||||
|
assertThat(metrics.getOldestScn() == 10L).isTrue();
|
||||||
|
|
||||||
|
metrics.setCommittedScn(10L);
|
||||||
|
assertThat(metrics.getCommittedScn() == 10L).isTrue();
|
||||||
|
|
||||||
|
assertThat(metrics.toString().contains("capturedDmlCounter=1000")).isTrue();
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,361 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import static io.debezium.config.CommonConnectorConfig.DEFAULT_MAX_BATCH_SIZE;
|
||||||
|
import static io.debezium.config.CommonConnectorConfig.DEFAULT_MAX_QUEUE_SIZE;
|
||||||
|
import static junit.framework.TestCase.assertNotSame;
|
||||||
|
import static junit.framework.TestCase.assertSame;
|
||||||
|
import static junit.framework.TestCase.assertTrue;
|
||||||
|
import static org.fest.assertions.Assertions.assertThat;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.time.temporal.ChronoUnit;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
|
||||||
|
import io.debezium.config.Configuration;
|
||||||
|
import io.debezium.connector.base.ChangeEventQueue;
|
||||||
|
import io.debezium.connector.oracle.OracleConnector;
|
||||||
|
import io.debezium.connector.oracle.OracleConnectorConfig;
|
||||||
|
import io.debezium.connector.oracle.OracleOffsetContext;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||||
|
import io.debezium.connector.oracle.xstream.LcrPosition;
|
||||||
|
import io.debezium.pipeline.DataChangeEvent;
|
||||||
|
import io.debezium.pipeline.ErrorHandler;
|
||||||
|
import io.debezium.pipeline.txmetadata.TransactionContext;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Andrey Pustovetov
|
||||||
|
*/
|
||||||
|
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||||
|
public class TransactionalBufferTest {
|
||||||
|
|
||||||
|
private static final String SERVER_NAME = "serverX";
|
||||||
|
private static final String TRANSACTION_ID = "transaction";
|
||||||
|
private static final String OTHER_TRANSACTION_ID = "other_transaction";
|
||||||
|
private static final String SQL_ONE = "update table";
|
||||||
|
private static final String SQL_TWO = "insert into table";
|
||||||
|
private static final String MESSAGE = "OK";
|
||||||
|
private static final BigDecimal SCN = BigDecimal.ONE;
|
||||||
|
private static final BigDecimal OTHER_SCN = BigDecimal.TEN;
|
||||||
|
private static final BigDecimal LARGEST_SCN = BigDecimal.valueOf(100L);
|
||||||
|
private static final Timestamp TIMESTAMP = new Timestamp(System.currentTimeMillis());
|
||||||
|
private static final Configuration config = new Configuration() {
|
||||||
|
@Override
|
||||||
|
public Set<String> keys() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getString(String key) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
private static final OracleConnectorConfig connectorConfig = new OracleConnectorConfig(config);
|
||||||
|
private static OracleOffsetContext offsetContext;
|
||||||
|
|
||||||
|
private ErrorHandler errorHandler;
|
||||||
|
private TransactionalBuffer transactionalBuffer;
|
||||||
|
private TransactionalBufferMetrics metrics;
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void before() {
|
||||||
|
ChangeEventQueue<DataChangeEvent> queue = new ChangeEventQueue.Builder<DataChangeEvent>()
|
||||||
|
.pollInterval(Duration.of(DEFAULT_MAX_QUEUE_SIZE, ChronoUnit.MILLIS))
|
||||||
|
.maxBatchSize(DEFAULT_MAX_BATCH_SIZE)
|
||||||
|
.maxQueueSize(DEFAULT_MAX_QUEUE_SIZE)
|
||||||
|
.build();
|
||||||
|
errorHandler = new ErrorHandler(OracleConnector.class, SERVER_NAME, queue);
|
||||||
|
metrics = mock(TransactionalBufferMetrics.class);
|
||||||
|
transactionalBuffer = new TransactionalBuffer(SERVER_NAME, errorHandler, metrics,
|
||||||
|
DEFAULT_MAX_QUEUE_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void after() throws InterruptedException {
|
||||||
|
transactionalBuffer.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testIsEmpty() {
|
||||||
|
assertThat(transactionalBuffer.isEmpty()).isEqualTo(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testIsNotEmptyWhenTransactionIsRegistered() {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
assertThat(transactionalBuffer.isEmpty()).isEqualTo(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testIsNotEmptyWhenTransactionIsCommitting() {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> Thread.sleep(1000));
|
||||||
|
offsetContext = new OracleOffsetContext(connectorConfig, SCN.longValue(), SCN.longValue(), (LcrPosition) null, false, true, new TransactionContext());
|
||||||
|
transactionalBuffer.commit(TRANSACTION_ID, SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||||
|
assertThat(transactionalBuffer.isEmpty()).isEqualTo(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testIsEmptyWhenTransactionIsCommitted() throws InterruptedException {
|
||||||
|
CountDownLatch commitLatch = new CountDownLatch(1);
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> commitLatch.countDown());
|
||||||
|
offsetContext = new OracleOffsetContext(connectorConfig, SCN.longValue(), SCN.longValue(), (LcrPosition) null, false, true, new TransactionContext());
|
||||||
|
transactionalBuffer.commit(TRANSACTION_ID, SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||||
|
commitLatch.await();
|
||||||
|
Thread.sleep(1000);
|
||||||
|
assertThat(transactionalBuffer.isEmpty()).isEqualTo(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testIsEmptyWhenTransactionIsRolledBack() {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||||
|
assertThat(transactionalBuffer.isEmpty()).isEqualTo(true);
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(SCN);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNonEmptyFirstTransactionIsRolledBack() {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "insert", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||||
|
assertThat(transactionalBuffer.isEmpty()).isEqualTo(false);
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||||
|
assertThat(transactionalBuffer.getRolledBackTransactionIds().contains(TRANSACTION_ID)).isTrue();
|
||||||
|
assertThat(transactionalBuffer.getRolledBackTransactionIds().contains(OTHER_TRANSACTION_ID)).isFalse();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNonEmptySecondTransactionIsRolledBack() {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.rollback(OTHER_TRANSACTION_ID, "");
|
||||||
|
assertThat(transactionalBuffer.isEmpty()).isEqualTo(false);
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||||
|
assertThat(transactionalBuffer.getRolledBackTransactionIds().contains(TRANSACTION_ID)).isFalse();
|
||||||
|
assertThat(transactionalBuffer.getRolledBackTransactionIds().contains(OTHER_TRANSACTION_ID)).isTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCalculateScnWhenTransactionIsCommitted() throws InterruptedException {
|
||||||
|
CountDownLatch commitLatch = new CountDownLatch(1);
|
||||||
|
AtomicReference<BigDecimal> smallestScnContainer = new AtomicReference<>();
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
smallestScnContainer.set(smallestScn);
|
||||||
|
commitLatch.countDown();
|
||||||
|
});
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(SCN); // before commit
|
||||||
|
offsetContext = new OracleOffsetContext(connectorConfig, SCN.longValue(), SCN.longValue(), null, false, true, new TransactionContext());
|
||||||
|
transactionalBuffer.commit(TRANSACTION_ID, SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||||
|
commitLatch.await();
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(SCN); // after commit
|
||||||
|
|
||||||
|
assertThat(smallestScnContainer.get()).isNull();
|
||||||
|
|
||||||
|
assertThat(transactionalBuffer.getRolledBackTransactionIds().isEmpty()).isTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCalculateScnWhenFirstTransactionIsCommitted() throws InterruptedException {
|
||||||
|
CountDownLatch commitLatch = new CountDownLatch(1);
|
||||||
|
AtomicReference<BigDecimal> smallestScnContainer = new AtomicReference<>();
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
smallestScnContainer.set(smallestScn);
|
||||||
|
commitLatch.countDown();
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN); // before commit
|
||||||
|
offsetContext = new OracleOffsetContext(connectorConfig, SCN.longValue(), SCN.longValue(), null, false, true, new TransactionContext());
|
||||||
|
transactionalBuffer.commit(TRANSACTION_ID, SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||||
|
commitLatch.await();
|
||||||
|
// after commit, it stays the same because OTHER_TRANSACTION_ID is not committed yet
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||||
|
|
||||||
|
assertThat(smallestScnContainer.get()).isEqualTo(OTHER_SCN);
|
||||||
|
assertThat(transactionalBuffer.getRolledBackTransactionIds().isEmpty()).isTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCalculateScnWhenSecondTransactionIsCommitted() throws InterruptedException {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
CountDownLatch commitLatch = new CountDownLatch(1);
|
||||||
|
AtomicReference<BigDecimal> smallestScnContainer = new AtomicReference<>();
|
||||||
|
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
smallestScnContainer.set(smallestScn);
|
||||||
|
commitLatch.countDown();
|
||||||
|
});
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN); // before commit
|
||||||
|
offsetContext = new OracleOffsetContext(connectorConfig, OTHER_SCN.longValue(), OTHER_SCN.longValue(), null, false, true, new TransactionContext());
|
||||||
|
transactionalBuffer.commit(OTHER_TRANSACTION_ID, OTHER_SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||||
|
commitLatch.await();
|
||||||
|
assertThat(smallestScnContainer.get()).isEqualTo(SCN);
|
||||||
|
// after committing OTHER_TRANSACTION_ID
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||||
|
assertThat(transactionalBuffer.getRolledBackTransactionIds().isEmpty()).isTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testResetLargestScn() {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN); // before commit
|
||||||
|
offsetContext = new OracleOffsetContext(connectorConfig, OTHER_SCN.longValue(), OTHER_SCN.longValue(), null, false, true, new TransactionContext());
|
||||||
|
transactionalBuffer.commit(OTHER_TRANSACTION_ID, OTHER_SCN, offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN); // after commit
|
||||||
|
|
||||||
|
transactionalBuffer.resetLargestScn(null);
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(BigDecimal.ZERO);
|
||||||
|
transactionalBuffer.resetLargestScn(OTHER_SCN.longValue());
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testAbandoningOneTransaction() {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.abandonLongTransactions(SCN.longValue());
|
||||||
|
assertThat(transactionalBuffer.isEmpty()).isEqualTo(true);
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(BigDecimal.ZERO);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testAbandoningTransactionHavingAnotherOne() {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), "", (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.abandonLongTransactions(SCN.longValue());
|
||||||
|
assertThat(transactionalBuffer.isEmpty()).isEqualTo(false);
|
||||||
|
assertThat(transactionalBuffer.getLargestScn()).isEqualTo(OTHER_SCN);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testTransactionDump() {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), SQL_ONE, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), SQL_ONE, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, OTHER_SCN, Instant.now(), SQL_TWO, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
assertThat(transactionalBuffer.toString()).contains(SQL_ONE);
|
||||||
|
assertThat(transactionalBuffer.toString()).contains(SQL_TWO);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDuplicatedRedoSql() {
|
||||||
|
|
||||||
|
assertThat(transactionalBuffer.getLargestScn().equals(BigDecimal.ZERO));
|
||||||
|
|
||||||
|
final String insertIntoATable = "insert into a table";
|
||||||
|
final String anotherInsertIntoATable = "another insert into a table";
|
||||||
|
final String duplicatedInsertIntoATable = "duplicated insert into a table";
|
||||||
|
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), insertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, OTHER_SCN, Instant.now(), anotherInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
assertThat(transactionalBuffer.getLargestScn().equals(OTHER_SCN));
|
||||||
|
assertThat(transactionalBuffer.toString().contains(insertIntoATable));
|
||||||
|
assertThat(transactionalBuffer.toString().contains(anotherInsertIntoATable));
|
||||||
|
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||||
|
|
||||||
|
// duplications are OK in different transactions
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), duplicatedInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(OTHER_TRANSACTION_ID, SCN, Instant.now(), duplicatedInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
assertThat(transactionalBuffer.toString().indexOf(duplicatedInsertIntoATable) != transactionalBuffer.toString().lastIndexOf(duplicatedInsertIntoATable));
|
||||||
|
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||||
|
transactionalBuffer.rollback(OTHER_TRANSACTION_ID, "");
|
||||||
|
|
||||||
|
// duplications are NOT OK in a transactions for different SCNs if they are sequential
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), duplicatedInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, OTHER_SCN, Instant.now(), duplicatedInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
assertThat(transactionalBuffer.toString().indexOf(duplicatedInsertIntoATable) == transactionalBuffer.toString().lastIndexOf(duplicatedInsertIntoATable));
|
||||||
|
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||||
|
|
||||||
|
// duplications are OK in a transactions for different SCNs if they are NOT sequential
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), duplicatedInsertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, OTHER_SCN, Instant.now(), insertIntoATable, (timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, LARGEST_SCN, Instant.now(), duplicatedInsertIntoATable,
|
||||||
|
(timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
});
|
||||||
|
assertThat(transactionalBuffer.toString().indexOf(duplicatedInsertIntoATable) != transactionalBuffer.toString().lastIndexOf(duplicatedInsertIntoATable));
|
||||||
|
transactionalBuffer.rollback(TRANSACTION_ID, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCommitQueueOverflowProcessedOnCaller() throws InterruptedException {
|
||||||
|
Thread mainThread = Thread.currentThread();
|
||||||
|
int commitQueueCapacity = 10;
|
||||||
|
transactionalBuffer = new TransactionalBuffer(SERVER_NAME, errorHandler, metrics,
|
||||||
|
commitQueueCapacity);
|
||||||
|
int transactionToCommitCount = commitQueueCapacity + 1;
|
||||||
|
CountDownLatch countDownLatch = new CountDownLatch(transactionToCommitCount + 1);
|
||||||
|
for (int i = 0; i <= commitQueueCapacity; i++) {
|
||||||
|
commitTransaction((timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
assertNotSame(mainThread, Thread.currentThread());
|
||||||
|
TimeUnit.MILLISECONDS.sleep(100);
|
||||||
|
countDownLatch.countDown();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Commit one more over the capacity. This should process in the test thread, applying backpressure
|
||||||
|
// to the caller
|
||||||
|
commitTransaction((timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
assertSame(mainThread, Thread.currentThread());
|
||||||
|
countDownLatch.countDown();
|
||||||
|
});
|
||||||
|
|
||||||
|
TimeUnit.SECONDS.sleep(2);
|
||||||
|
|
||||||
|
// Commit one more over the capacity. After delay, the executor had time to recover and empty its queue
|
||||||
|
// This should go back to processing in the executor thread
|
||||||
|
commitTransaction((timestamp, smallestScn, commitScn, counter) -> {
|
||||||
|
assertNotSame(mainThread, Thread.currentThread());
|
||||||
|
countDownLatch.countDown();
|
||||||
|
});
|
||||||
|
|
||||||
|
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void commitTransaction(TransactionalBuffer.CommitCallback commitCallback) {
|
||||||
|
transactionalBuffer.registerCommitCallback(TRANSACTION_ID, SCN, Instant.now(), "", commitCallback);
|
||||||
|
offsetContext = new OracleOffsetContext(connectorConfig, SCN.longValue(), SCN.longValue(), null, false, true, new TransactionContext());
|
||||||
|
transactionalBuffer.commit(TRANSACTION_ID, SCN.add(BigDecimal.ONE), offsetContext, TIMESTAMP, () -> true, MESSAGE);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,94 @@
|
|||||||
|
/*
|
||||||
|
* Copyright Debezium Authors.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*/
|
||||||
|
package io.debezium.connector.oracle.logminer;
|
||||||
|
|
||||||
|
import static org.fest.assertions.Assertions.assertThat;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.sql.Types;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TestRule;
|
||||||
|
|
||||||
|
import io.debezium.connector.oracle.antlr.OracleDdlParser;
|
||||||
|
import io.debezium.connector.oracle.jsqlparser.SimpleDmlParser;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipTestDependingOnAdapterNameRule;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot;
|
||||||
|
import io.debezium.connector.oracle.junit.SkipWhenAdapterNameIsNot.AdapterName;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValue;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueImpl;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerColumnValueWrapper;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntry;
|
||||||
|
import io.debezium.connector.oracle.logminer.valueholder.LogMinerDmlEntryImpl;
|
||||||
|
import io.debezium.data.Envelope;
|
||||||
|
import io.debezium.relational.Tables;
|
||||||
|
import io.debezium.util.IoUtil;
|
||||||
|
|
||||||
|
@SkipWhenAdapterNameIsNot(value = AdapterName.LOGMINER)
|
||||||
|
public class ValueHolderTest {
|
||||||
|
private static final String TABLE_NAME = "TEST";
|
||||||
|
private static final String CATALOG_NAME = "CATALOG";
|
||||||
|
private static final String SCHEMA_NAME = "DEBEZIUM";
|
||||||
|
private OracleDdlParser ddlParser;
|
||||||
|
private SimpleDmlParser sqlDmlParser;
|
||||||
|
private Tables tables;
|
||||||
|
private static final String FULL_TABLE_NAME = SCHEMA_NAME + "\".\"" + TABLE_NAME;
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TestRule skipRule = new SkipTestDependingOnAdapterNameRule();
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() {
|
||||||
|
OracleChangeRecordValueConverter converters = new OracleChangeRecordValueConverter(null);
|
||||||
|
ddlParser = new OracleDdlParser(true, CATALOG_NAME, SCHEMA_NAME);
|
||||||
|
sqlDmlParser = new SimpleDmlParser(CATALOG_NAME, SCHEMA_NAME, converters);
|
||||||
|
tables = new Tables();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testValueHolders() throws Exception {
|
||||||
|
LogMinerColumnValue column1 = new LogMinerColumnValueImpl("COLUMN1", Types.NUMERIC);
|
||||||
|
assertThat(column1.equals(column1)).isTrue();
|
||||||
|
assertThat(column1.equals(null)).isFalse();
|
||||||
|
assertThat(new LogMinerColumnValueWrapper(column1).isProcessed()).isFalse();
|
||||||
|
|
||||||
|
column1.setColumnData(new BigDecimal(5));
|
||||||
|
LogMinerColumnValue column2 = new LogMinerColumnValueImpl("COLUMN2", Types.VARCHAR);
|
||||||
|
column2.setColumnData("Text");
|
||||||
|
List<LogMinerColumnValue> newValues = new ArrayList<>();
|
||||||
|
newValues.add(column1);
|
||||||
|
newValues.add(column2);
|
||||||
|
LogMinerDmlEntryImpl dmlEntryExpected = new LogMinerDmlEntryImpl(Envelope.Operation.CREATE, newValues, Collections.emptyList());
|
||||||
|
dmlEntryExpected.setTransactionId("transaction_id");
|
||||||
|
dmlEntryExpected.setObjectName(TABLE_NAME);
|
||||||
|
dmlEntryExpected.setObjectOwner(SCHEMA_NAME);
|
||||||
|
dmlEntryExpected.setScn(BigDecimal.ONE);
|
||||||
|
dmlEntryExpected.setSourceTime(new Timestamp(1000));
|
||||||
|
|
||||||
|
String createStatement = IoUtil.read(IoUtil.getResourceAsStream("ddl/create_small_table.sql", null, getClass(), null, null));
|
||||||
|
ddlParser.parse(createStatement, tables);
|
||||||
|
|
||||||
|
String dml = "insert into \"" + FULL_TABLE_NAME + "\" (\"column1\",\"column2\") values ('5','Text');";
|
||||||
|
LogMinerDmlEntry dmlEntryParsed = sqlDmlParser.parse(dml, tables, "1");
|
||||||
|
|
||||||
|
assertThat(dmlEntryParsed.equals(dmlEntryExpected)).isTrue();
|
||||||
|
assertThat(dmlEntryExpected.getCommandType() == Envelope.Operation.CREATE).isTrue();
|
||||||
|
assertThat(dmlEntryExpected.getScn().equals(BigDecimal.ONE)).isTrue();
|
||||||
|
assertThat(dmlEntryExpected.getSourceTime().equals(new Timestamp(1000))).isTrue();
|
||||||
|
assertThat(dmlEntryExpected.getTransactionId().equals("transaction_id")).isTrue();
|
||||||
|
assertThat(dmlEntryExpected.getObjectOwner().equals(SCHEMA_NAME)).isTrue();
|
||||||
|
assertThat(dmlEntryExpected.getObjectName().equals(TABLE_NAME)).isTrue();
|
||||||
|
|
||||||
|
assertThat(dmlEntryExpected.equals(null)).isFalse();
|
||||||
|
assertThat(dmlEntryExpected.equals(dmlEntryExpected)).isTrue();
|
||||||
|
}
|
||||||
|
}
|
@ -26,6 +26,7 @@ public class TestHelper {
|
|||||||
public static final Path DB_HISTORY_PATH = Testing.Files.createTestingPath("file-db-history-connect.txt").toAbsolutePath();
|
public static final Path DB_HISTORY_PATH = Testing.Files.createTestingPath("file-db-history-connect.txt").toAbsolutePath();
|
||||||
|
|
||||||
public static final String CONNECTOR_USER = "c##xstrm";
|
public static final String CONNECTOR_USER = "c##xstrm";
|
||||||
|
public static final String CONNECTOR_USER_LOGMINER = "c##dbzuser";
|
||||||
|
|
||||||
public static final String CONNECTOR_NAME = "oracle";
|
public static final String CONNECTOR_NAME = "oracle";
|
||||||
|
|
||||||
@ -46,13 +47,36 @@ public class TestHelper {
|
|||||||
*/
|
*/
|
||||||
public static final String TYPE_SCALE_PARAMETER_KEY = "__debezium.source.column.scale";
|
public static final String TYPE_SCALE_PARAMETER_KEY = "__debezium.source.column.scale";
|
||||||
|
|
||||||
|
// todo: unify logminer and xstream user accounts
|
||||||
|
public static final String CONNECTOR_USER_PASS = "xs";
|
||||||
|
public static final String CONNECTOR_USER_PASS_LOGMINER = "dbz";
|
||||||
|
public static final String HOST = "localhost";
|
||||||
|
public static final String SCHEMA_USER = "debezium";
|
||||||
|
public static final String SCHEMA_PASS = "dbz";
|
||||||
|
public static final String DATABASE = "ORCLPDB1";
|
||||||
|
public static final String DATABASE_CDB = "ORCLCDB";
|
||||||
|
|
||||||
|
public static String getConnectorUserName() {
|
||||||
|
if (TestHelper.adapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
|
||||||
|
return CONNECTOR_USER_LOGMINER;
|
||||||
|
}
|
||||||
|
return CONNECTOR_USER;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getConnectorUserPassword() {
|
||||||
|
if (TestHelper.adapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
|
||||||
|
return CONNECTOR_USER_PASS_LOGMINER;
|
||||||
|
}
|
||||||
|
return CONNECTOR_USER_PASS;
|
||||||
|
}
|
||||||
|
|
||||||
private static JdbcConfiguration defaultJdbcConfig() {
|
private static JdbcConfiguration defaultJdbcConfig() {
|
||||||
return JdbcConfiguration.copy(Configuration.fromSystemProperties("database."))
|
return JdbcConfiguration.copy(Configuration.fromSystemProperties("database."))
|
||||||
.withDefault(JdbcConfiguration.HOSTNAME, "localhost")
|
.withDefault(JdbcConfiguration.HOSTNAME, HOST)
|
||||||
.withDefault(JdbcConfiguration.PORT, 1521)
|
.withDefault(JdbcConfiguration.PORT, 1521)
|
||||||
.withDefault(JdbcConfiguration.USER, CONNECTOR_USER)
|
.withDefault(JdbcConfiguration.USER, getConnectorUserName())
|
||||||
.withDefault(JdbcConfiguration.PASSWORD, "xs")
|
.withDefault(JdbcConfiguration.PASSWORD, getConnectorUserPassword())
|
||||||
.withDefault(JdbcConfiguration.DATABASE, "ORCLCDB")
|
.withDefault(JdbcConfiguration.DATABASE, DATABASE_CDB)
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -71,6 +95,7 @@ public static Configuration.Builder defaultConfig() {
|
|||||||
.with(OracleConnectorConfig.PDB_NAME, "ORCLPDB1")
|
.with(OracleConnectorConfig.PDB_NAME, "ORCLPDB1")
|
||||||
.with(OracleConnectorConfig.XSTREAM_SERVER_NAME, "dbzxout")
|
.with(OracleConnectorConfig.XSTREAM_SERVER_NAME, "dbzxout")
|
||||||
.with(OracleConnectorConfig.DATABASE_HISTORY, FileDatabaseHistory.class)
|
.with(OracleConnectorConfig.DATABASE_HISTORY, FileDatabaseHistory.class)
|
||||||
|
.with(OracleConnectorConfig.SCHEMA_NAME, SCHEMA_USER)
|
||||||
.with(FileDatabaseHistory.FILE_PATH, DB_HISTORY_PATH)
|
.with(FileDatabaseHistory.FILE_PATH, DB_HISTORY_PATH)
|
||||||
.with(OracleConnectorConfig.INCLUDE_SCHEMA_CHANGES, false);
|
.with(OracleConnectorConfig.INCLUDE_SCHEMA_CHANGES, false);
|
||||||
}
|
}
|
||||||
@ -79,7 +104,7 @@ public static OracleConnection defaultConnection() {
|
|||||||
Configuration config = defaultConfig().build();
|
Configuration config = defaultConfig().build();
|
||||||
Configuration jdbcConfig = config.subset("database.", true);
|
Configuration jdbcConfig = config.subset("database.", true);
|
||||||
|
|
||||||
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());
|
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory(), TestHelper.class::getClassLoader);
|
||||||
|
|
||||||
String pdbName = new OracleConnectorConfig(config).getPdbName();
|
String pdbName = new OracleConnectorConfig(config).getPdbName();
|
||||||
|
|
||||||
@ -90,16 +115,29 @@ public static OracleConnection defaultConnection() {
|
|||||||
return jdbcConnection;
|
return jdbcConnection;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Database level connection.
|
||||||
|
* this is PDB level connector with LogMiner adapter
|
||||||
|
* @return OracleConnection
|
||||||
|
*/
|
||||||
|
public static OracleConnection logMinerPdbConnection() {
|
||||||
|
Configuration jdbcConfig = testJdbcConfig().edit()
|
||||||
|
.with(OracleConnectorConfig.CONNECTOR_ADAPTER, "LogMiner")
|
||||||
|
.with(OracleConnectorConfig.DRIVER_TYPE, "thin")
|
||||||
|
.build();
|
||||||
|
return new OracleConnection(jdbcConfig, new OracleConnectionFactory(), TestHelper.class::getClassLoader);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns a JDBC configuration for the test data schema and user (NOT the XStream user).
|
* Returns a JDBC configuration for the test data schema and user (NOT the XStream user).
|
||||||
*/
|
*/
|
||||||
private static JdbcConfiguration testJdbcConfig() {
|
private static JdbcConfiguration testJdbcConfig() {
|
||||||
return JdbcConfiguration.copy(Configuration.fromSystemProperties("database."))
|
return JdbcConfiguration.copy(Configuration.fromSystemProperties("database."))
|
||||||
.withDefault(JdbcConfiguration.HOSTNAME, "localhost")
|
.withDefault(JdbcConfiguration.HOSTNAME, HOST)
|
||||||
.withDefault(JdbcConfiguration.PORT, 1521)
|
.withDefault(JdbcConfiguration.PORT, 1521)
|
||||||
.withDefault(JdbcConfiguration.USER, "debezium")
|
.withDefault(JdbcConfiguration.USER, SCHEMA_USER)
|
||||||
.withDefault(JdbcConfiguration.PASSWORD, "dbz")
|
.withDefault(JdbcConfiguration.PASSWORD, SCHEMA_PASS)
|
||||||
.withDefault(JdbcConfiguration.DATABASE, "ORCLPDB1")
|
.withDefault(JdbcConfiguration.DATABASE, DATABASE)
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -108,11 +146,11 @@ private static JdbcConfiguration testJdbcConfig() {
|
|||||||
*/
|
*/
|
||||||
private static JdbcConfiguration adminJdbcConfig() {
|
private static JdbcConfiguration adminJdbcConfig() {
|
||||||
return JdbcConfiguration.copy(Configuration.fromSystemProperties("database.admin."))
|
return JdbcConfiguration.copy(Configuration.fromSystemProperties("database.admin."))
|
||||||
.withDefault(JdbcConfiguration.HOSTNAME, "localhost")
|
.withDefault(JdbcConfiguration.HOSTNAME, HOST)
|
||||||
.withDefault(JdbcConfiguration.PORT, 1521)
|
.withDefault(JdbcConfiguration.PORT, 1521)
|
||||||
.withDefault(JdbcConfiguration.USER, "sys as sysdba")
|
.withDefault(JdbcConfiguration.USER, "sys as sysdba")
|
||||||
.withDefault(JdbcConfiguration.PASSWORD, "top_secret")
|
.withDefault(JdbcConfiguration.PASSWORD, "top_secret")
|
||||||
.withDefault(JdbcConfiguration.DATABASE, "ORCLPDB1")
|
.withDefault(JdbcConfiguration.DATABASE, DATABASE)
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -140,7 +178,7 @@ public static OracleConnection testConnection() {
|
|||||||
Configuration config = testConfig().build();
|
Configuration config = testConfig().build();
|
||||||
Configuration jdbcConfig = config.subset("database.", true);
|
Configuration jdbcConfig = config.subset("database.", true);
|
||||||
|
|
||||||
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());
|
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory(), TestHelper.class::getClassLoader);
|
||||||
try {
|
try {
|
||||||
jdbcConnection.setAutoCommit(false);
|
jdbcConnection.setAutoCommit(false);
|
||||||
}
|
}
|
||||||
@ -161,7 +199,7 @@ public static OracleConnection adminConnection() {
|
|||||||
Configuration config = adminConfig().build();
|
Configuration config = adminConfig().build();
|
||||||
Configuration jdbcConfig = config.subset("database.", true);
|
Configuration jdbcConfig = config.subset("database.", true);
|
||||||
|
|
||||||
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());
|
OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory(), TestHelper.class::getClassLoader);
|
||||||
try {
|
try {
|
||||||
jdbcConnection.setAutoCommit(false);
|
jdbcConnection.setAutoCommit(false);
|
||||||
}
|
}
|
||||||
@ -192,4 +230,9 @@ public static void dropTable(OracleConnection connection, String table) {
|
|||||||
public static int defaultMessageConsumerPollTimeout() {
|
public static int defaultMessageConsumerPollTimeout() {
|
||||||
return 120;
|
return 120;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static OracleConnectorConfig.ConnectorAdapter adapter() {
|
||||||
|
final String s = System.getProperty(OracleConnectorConfig.CONNECTOR_ADAPTER.name());
|
||||||
|
return (s == null || s.length() == 0) ? OracleConnectorConfig.ConnectorAdapter.XSTREAM : OracleConnectorConfig.ConnectorAdapter.parse(s);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,4 @@
|
|||||||
|
CREATE TABLE TEST
|
||||||
|
(COLUMN1 NUMBER(19) NOT NULL,
|
||||||
|
COLUMN2 VARCHAR2(255)
|
||||||
|
PRIMARY KEY (COLUMN1));
|
@ -1,13 +1,16 @@
|
|||||||
create table DEBEZIUM
|
create table TEST
|
||||||
(id number(19) not null,
|
(id number(19) not null,
|
||||||
col1 number(4,2),
|
col1 numeric(4,2),
|
||||||
col2 varchar2(255) default 'debezium' not null ,
|
col2 varchar2(255) default 'debezium' not null ,
|
||||||
col3 nvarchar2(255) not null,
|
col3 nvarchar2(255) not null,
|
||||||
col4 char(4),
|
col4 char(4),
|
||||||
col5 nchar(4),
|
col5 nchar(4),
|
||||||
col6 float(126),
|
col6 float(126),
|
||||||
col7 date,
|
-- todo: dbz-137 this field was removed, re-add it?
|
||||||
|
-- col7 date,
|
||||||
col8 timestamp,
|
col8 timestamp,
|
||||||
col9 blob,
|
col9 blob,
|
||||||
col10 clob,
|
col10 clob,
|
||||||
|
col11 "mdsys"."sdo_geometry",
|
||||||
|
col12 number(1,0),
|
||||||
primary key (id));
|
primary key (id));
|
||||||
|
@ -13,3 +13,5 @@ log4j.logger.io.debezium.embedded.EmbeddedEngine$EmbeddedConfig=WARN
|
|||||||
#log4j.logger.io.debezium.embedded.EmbeddedEngine=DEBUG
|
#log4j.logger.io.debezium.embedded.EmbeddedEngine=DEBUG
|
||||||
log4j.logger.io.debezium.core=DEBUG
|
log4j.logger.io.debezium.core=DEBUG
|
||||||
log4j.logger.io.debezium.connector.oracle=DEBUG
|
log4j.logger.io.debezium.connector.oracle=DEBUG
|
||||||
|
log4j.logger.io.debezium.connector.oracle.logminer.LogMinerQueryResultProcessor=TRACE
|
||||||
|
log4j.logger.io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSource=TRACE
|
Loading…
Reference in New Issue
Block a user