DBZ-777 PostgreSQL connector rewrite to the standard framework

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/LegacyV1PostgresSourceInfoStructMaker.java

@@ -11,6 +11,8 @@
 
 import io.debezium.config.CommonConnectorConfig;
 import io.debezium.connector.LegacyV1AbstractSourceInfoStructMaker;
+import io.debezium.connector.SnapshotRecord;
+import io.debezium.time.Conversions;
 
 public class LegacyV1PostgresSourceInfoStructMaker extends LegacyV1AbstractSourceInfoStructMaker<SourceInfo> {
 
@@ -51,8 +53,22 @@ public Struct struct(SourceInfo sourceInfo) {
         result.put(SourceInfo.DATABASE_NAME_KEY, sourceInfo.database());
         result.put(SourceInfo.SCHEMA_NAME_KEY, sourceInfo.schemaName());
         result.put(SourceInfo.TABLE_NAME_KEY, sourceInfo.tableName());
-        // use the offset information without the snapshot part (see below)
-        sourceInfo.offset().forEach(result::put);
+        if (sourceInfo.timestamp() != null) {
+            result.put(SourceInfo.TIMESTAMP_USEC_KEY, Conversions.toEpochMicros(sourceInfo.timestamp()));
+        }
+        if (sourceInfo.txId() != null) {
+            result.put(SourceInfo.TXID_KEY, sourceInfo.txId());
+        }
+        if (sourceInfo.lsn() != null) {
+            result.put(SourceInfo.LSN_KEY, sourceInfo.lsn());
+        }
+        if (sourceInfo.xmin() != null) {
+            result.put(SourceInfo.XMIN_KEY, sourceInfo.xmin());
+        }
+        if (sourceInfo.isSnapshot()) {
+            result.put(SourceInfo.SNAPSHOT_KEY, true);
+            result.put(SourceInfo.LAST_SNAPSHOT_RECORD_KEY, sourceInfo.snapshot() == SnapshotRecord.LAST);
+        }
         return result;
     }
 }

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresChangeEventSourceFactory.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright Debezium Authors.
+ *
+ * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
+ */
+package io.debezium.connector.postgresql;
+
+import io.debezium.connector.postgresql.connection.PostgresConnection;
+import io.debezium.connector.postgresql.connection.ReplicationConnection;
+import io.debezium.connector.postgresql.spi.SlotCreationResult;
+import io.debezium.connector.postgresql.spi.Snapshotter;
+import io.debezium.pipeline.ErrorHandler;
+import io.debezium.pipeline.EventDispatcher;
+import io.debezium.pipeline.source.spi.ChangeEventSourceFactory;
+import io.debezium.pipeline.source.spi.SnapshotChangeEventSource;
+import io.debezium.pipeline.source.spi.SnapshotProgressListener;
+import io.debezium.pipeline.source.spi.StreamingChangeEventSource;
+import io.debezium.pipeline.spi.OffsetContext;
+import io.debezium.relational.TableId;
+import io.debezium.util.Clock;
+
+public class PostgresChangeEventSourceFactory implements ChangeEventSourceFactory {
+
+    private final PostgresConnectorConfig configuration;
+    private final PostgresConnection jdbcConnection;
+    private final ErrorHandler errorHandler;
+    private final EventDispatcher<TableId> dispatcher;
+    private final Clock clock;
+    private final PostgresSchema schema;
+    private final PostgresTaskContext taskContext;
+    private final Snapshotter snapshotter;
+    private final ReplicationConnection replicationConnection;
+    private final SlotCreationResult slotCreatedInfo;
+
+    public PostgresChangeEventSourceFactory(PostgresConnectorConfig configuration, Snapshotter snapshotter, PostgresConnection jdbcConnection,
+            ErrorHandler errorHandler, EventDispatcher<TableId> dispatcher, Clock clock, PostgresSchema schema, PostgresTaskContext taskContext,
+            ReplicationConnection replicationConnection, SlotCreationResult slotCreatedInfo) {
+        this.configuration = configuration;
+        this.jdbcConnection = jdbcConnection;
+        this.errorHandler = errorHandler;
+        this.dispatcher = dispatcher;
+        this.clock = clock;
+        this.schema = schema;
+        this.taskContext = taskContext;
+        this.snapshotter = snapshotter;
+        this.replicationConnection = replicationConnection;
+        this.slotCreatedInfo = slotCreatedInfo;
+    }
+
+    @Override
+    public SnapshotChangeEventSource getSnapshotChangeEventSource(OffsetContext offsetContext, SnapshotProgressListener snapshotProgressListener) {
+        return new PostgresSnapshotChangeEventSource(
+                configuration,
+                snapshotter,
+                (PostgresOffsetContext) offsetContext,
+                jdbcConnection,
+                schema,
+                dispatcher,
+                clock,
+                snapshotProgressListener,
+                slotCreatedInfo
+            );
+    }
+
+    @Override
+    public StreamingChangeEventSource getStreamingChangeEventSource(OffsetContext offsetContext) {
+        return new PostgresStreamingChangeEventSource(
+                configuration,
+                snapshotter,
+                (PostgresOffsetContext) offsetContext,
+                jdbcConnection,
+                dispatcher,
+                errorHandler,
+                clock,
+                schema,
+                taskContext,
+                replicationConnection
+        );
+    }
+}

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresChangeRecordEmitter.java

@@ -0,0 +1,318 @@
+/*
+ * Copyright Debezium Authors.
+ *
+ * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
+ */
+package io.debezium.connector.postgresql;
+
+import java.sql.SQLException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.apache.kafka.connect.errors.ConnectException;
+import org.postgresql.jdbc.PgConnection;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.debezium.connector.postgresql.connection.PostgresConnection;
+import io.debezium.connector.postgresql.connection.ReplicationMessage;
+import io.debezium.data.Envelope.Operation;
+import io.debezium.pipeline.spi.ChangeRecordEmitter;
+import io.debezium.pipeline.spi.OffsetContext;
+import io.debezium.relational.Column;
+import io.debezium.relational.ColumnEditor;
+import io.debezium.relational.RelationalChangeRecordEmitter;
+import io.debezium.relational.Table;
+import io.debezium.relational.TableEditor;
+import io.debezium.relational.TableId;
+import io.debezium.relational.TableSchema;
+import io.debezium.schema.DataCollectionSchema;
+import io.debezium.util.Clock;
+import io.debezium.util.Strings;
+
+/**
+ * Emits change data based on a logical decoding event coming as protobuf or JSON message.
+ *
+ * @author Horia Chiorean (hchiorea@redhat.com), Jiri Pechanec
+ */
+public class PostgresChangeRecordEmitter extends RelationalChangeRecordEmitter {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(PostgresChangeRecordEmitter.class);
+
+    private final ReplicationMessage message;
+    private final PostgresSchema schema;
+    private final PostgresConnectorConfig connectorConfig;
+    private final PostgresConnection connection;
+
+    public PostgresChangeRecordEmitter(OffsetContext offset, Clock clock, PostgresConnectorConfig connectorConfig, PostgresSchema schema, PostgresConnection connection, ReplicationMessage message) {
+        super(offset, clock);
+
+        this.schema = schema;
+        this.message = message;
+        this.connectorConfig = connectorConfig;
+        this.connection = connection;
+    }
+
+    @Override
+    protected Operation getOperation() {
+        switch (message.getOperation()) {
+        case INSERT:
+            return Operation.CREATE;
+        case UPDATE:
+            return Operation.UPDATE;
+        case DELETE:
+            return Operation.DELETE;
+        default:
+            throw new IllegalArgumentException("Received event of unexpected command type: " + message.getOperation());
+        }
+    }
+
+    @Override
+    protected Object[] getOldColumnValues() {
+        final TableId tableId = PostgresSchema.parse(message.getTable());
+        Objects.requireNonNull(tableId);
+
+        try {
+            switch (getOperation()) {
+                case CREATE:
+                    return null;
+                case UPDATE:
+                    return columnValues(message.getOldTupleList(), tableId, true, message.hasTypeMetadata());
+                default:
+                    return columnValues(message.getOldTupleList(), tableId, true, message.hasTypeMetadata());
+            }
+        }
+        catch (SQLException e) {
+            throw new ConnectException(e);
+        }
+    }
+
+    @Override
+    protected Object[] getNewColumnValues() {
+        final TableId tableId = PostgresSchema.parse(message.getTable());
+        Objects.requireNonNull(tableId);
+
+        try {
+            switch (getOperation()) {
+                case CREATE:
+                    return columnValues(message.getNewTupleList(), tableId, true, message.hasTypeMetadata());
+                case UPDATE:
+                    return columnValues(message.getNewTupleList(), tableId, true, message.hasTypeMetadata());
+                default:
+                    return null;
+            }
+        }
+        catch (SQLException e) {
+            throw new ConnectException(e);
+        }
+    }
+
+    @Override
+    public DataCollectionSchema synchronizeTableSchema(DataCollectionSchema tableSchema) {
+        final boolean metadataInMessage = message.hasTypeMetadata();
+        final TableId tableId = (TableId) tableSchema.id();
+        final Table table = schema.tableFor(tableId);
+        final List<ReplicationMessage.Column> columns = getOperation() == Operation.DELETE ? message.getOldTupleList() : message.getNewTupleList();
+        // check if we need to refresh our local schema due to DB schema changes for this table
+        if (schemaChanged(columns, table, metadataInMessage)) {
+            // Refresh the schema so we get information about primary keys
+            refreshTableFromDatabase(tableId);
+            // Update the schema with metadata coming from decoder message
+            if (metadataInMessage) {
+                schema.refresh(tableFromFromMessage(columns, schema.tableFor(tableId)));
+            }
+        }
+        return schema.schemaFor(tableId);
+    }
+
+    private Object[] columnValues(List<ReplicationMessage.Column> columns, TableId tableId, boolean refreshSchemaIfChanged, boolean metadataInMessage)
+            throws SQLException {
+        if (columns == null || columns.isEmpty()) {
+            return null;
+        }
+        final Table table = schema.tableFor(tableId);
+        Objects.requireNonNull(table);
+
+        // based on the schema columns, create the values on the same position as the columns
+        List<Column> schemaColumns = table.columns();
+        // JSON does not deliver a list of all columns for REPLICA IDENTITY DEFAULT
+        Object[] values = new Object[columns.size() < schemaColumns.size() ? schemaColumns.size() : columns.size()];
+
+        for (ReplicationMessage.Column column: columns) {
+            //DBZ-298 Quoted column names will be sent like that in messages, but stored unquoted in the column names
+            final String columnName = Strings.unquoteIdentifierPart(column.getName());
+            final Column tableColumn = table.columnWithName(columnName);
+            if (tableColumn == null) {
+                LOGGER.warn(
+                        "Internal schema is out-of-sync with incoming decoder events; column {} will be omitted from the change event.",
+                        column.getName());
+                continue;
+            }
+            int position = tableColumn.position() - 1;
+            if (position < 0 || position >= values.length) {
+                LOGGER.warn(
+                        "Internal schema is out-of-sync with incoming decoder events; column {} will be omitted from the change event.",
+                        column.getName());
+                continue;
+            }
+            values[position] = column.getValue(() -> (PgConnection) connection.connection(), connectorConfig.includeUnknownDatatypes());
+        }
+
+        return values;
+    }
+
+    private Optional<DataCollectionSchema> newTable(TableId tableId) {
+        refreshTableFromDatabase(tableId);
+        final TableSchema tableSchema = schema.schemaFor(tableId);
+        if (tableSchema == null) {
+            LOGGER.warn("cannot load schema for table '{}'", tableId);
+            return Optional.empty();
+        }
+        else {
+            LOGGER.debug("refreshed DB schema to include table '{}'", tableId);
+            return Optional.of(tableSchema);
+        }
+    }
+
+    private void refreshTableFromDatabase(TableId tableId) {
+        try {
+            schema.refresh(connection, tableId, connectorConfig.skipRefreshSchemaOnMissingToastableData());
+        }
+        catch (SQLException e) {
+            throw new ConnectException("Database error while refresing table schema");
+        }
+    }
+
+    static Optional<DataCollectionSchema> updateSchema(TableId tableId, ChangeRecordEmitter changeRecordEmitter) {
+        LOGGER.debug("Schema for table '{}' is missing", tableId);
+        return ((PostgresChangeRecordEmitter) changeRecordEmitter).newTable(tableId);
+    }
+
+    private boolean schemaChanged(List<ReplicationMessage.Column> columns, Table table, boolean metadataInMessage) {
+        int tableColumnCount = table.columns().size();
+        int replicationColumnCount = columns.size();
+
+        boolean msgHasMissingColumns = tableColumnCount > replicationColumnCount;
+
+        if (msgHasMissingColumns && connectorConfig.skipRefreshSchemaOnMissingToastableData()) {
+            // if we are ignoring missing toastable data for the purpose of schema sync, we need to modify the
+            // hasMissingColumns boolean to account for this. If there are untoasted columns missing from the replication
+            // message, we'll still have missing columns and thus require a schema refresh. However, we can /possibly/
+            // avoid the refresh if there are only toastable columns missing from the message.
+            msgHasMissingColumns = hasMissingUntoastedColumns(table, columns);
+        }
+
+        boolean msgHasAdditionalColumns = tableColumnCount < replicationColumnCount;
+
+        if (msgHasMissingColumns || msgHasAdditionalColumns) {
+            // the table metadata has less or more columns than the event, which means the table structure has changed,
+            // so we need to trigger a refresh...
+            LOGGER.info("Different column count {} present in the server message as schema in memory contains {}; refreshing table schema",
+                        replicationColumnCount,
+                        tableColumnCount);
+            return true;
+        }
+
+        // go through the list of columns from the message to figure out if any of them are new or have changed their type based
+        // on what we have in the table metadata....
+        return columns.stream().filter(message -> {
+            String columnName = message.getName();
+            Column column = table.columnWithName(columnName);
+            if (column == null) {
+                LOGGER.info("found new column '{}' present in the server message which is not part of the table metadata; refreshing table schema", columnName);
+                return true;
+            }
+            else {
+                final int localType = column.nativeType();
+                final int incomingType = message.getType().getOid();
+                if (localType != incomingType) {
+                    LOGGER.info("detected new type for column '{}', old type was {} ({}), new type is {} ({}); refreshing table schema", columnName, localType, column.typeName(),
+                                incomingType, message.getType().getName());
+                    return true;
+                }
+                if (metadataInMessage) {
+                    final int localLength = column.length();
+                    final int incomingLength = message.getTypeMetadata().getLength();
+                    if (localLength != incomingLength) {
+                        LOGGER.info("detected new length for column '{}', old length was {}, new length is {}; refreshing table schema", columnName, localLength,
+                                    incomingLength);
+                        return true;
+                    }
+                    final int localScale = column.scale().get();
+                    final int incomingScale = message.getTypeMetadata().getScale();
+                    if (localScale != incomingScale) {
+                        LOGGER.info("detected new scale for column '{}', old scale was {}, new scale is {}; refreshing table schema", columnName, localScale,
+                                    incomingScale);
+                        return true;
+                    }
+                    final boolean localOptional = column.isOptional();
+                    final boolean incomingOptional = message.isOptional();
+                    if (localOptional != incomingOptional) {
+                        LOGGER.info("detected new optional status for column '{}', old value was {}, new value is {}; refreshing table schema", columnName, localOptional, incomingOptional);
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }).findFirst().isPresent();
+    }
+
+    private boolean hasMissingUntoastedColumns(Table table, List<ReplicationMessage.Column> columns) {
+        List<String> msgColumnNames = columns.stream()
+                .map(ReplicationMessage.Column::getName)
+                .collect(Collectors.toList());
+
+        // Compute list of table columns not present in the replication message
+        List<String> missingColumnNames = table.columns()
+                .stream()
+                .filter(c -> !msgColumnNames.contains(c.name()))
+                .map(Column::name)
+                .collect(Collectors.toList());
+
+        List<String> toastableColumns = schema.getToastableColumnsForTableId(table.id());
+
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug("msg columns: '{}' --- missing columns: '{}' --- toastableColumns: '{}",
+                    String.join(",", msgColumnNames),
+                    String.join(",", missingColumnNames),
+                    String.join(",", toastableColumns));
+        }
+        // Return `true` if we have some columns not in the replication message that are not toastable or that we do
+        // not recognize
+        return !toastableColumns.containsAll(missingColumnNames);
+    }
+
+    private Table tableFromFromMessage(List<ReplicationMessage.Column> columns, Table table) {
+        final TableEditor combinedTable = table.edit()
+            .setColumns(columns.stream()
+                .map(column -> {
+                    final PostgresType type = column.getType();
+                    final ColumnEditor columnEditor = Column.editor()
+                            .name(column.getName())
+                            .jdbcType(type.getJdbcId())
+                            .type(type.getName())
+                            .optional(column.isOptional())
+                            .nativeType(type.getOid());
+                    columnEditor.length(column.getTypeMetadata().getLength());
+                    columnEditor.scale(column.getTypeMetadata().getScale());
+                    return columnEditor.create();
+                })
+                .collect(Collectors.toList())
+            );
+        final List<String> pkCandidates = table.filterColumnNames(c -> table.isPrimaryKeyColumn(c.name()));
+        final Iterator<String> itPkCandidates = pkCandidates.iterator();
+        while (itPkCandidates.hasNext()) {
+            final String candidateName = itPkCandidates.next();
+            if (!combinedTable.hasUniqueValues() && combinedTable.columnWithName(candidateName) == null) {
+                LOGGER.error("Potentional inconsistency in key for message {}", columns);
+                itPkCandidates.remove();
+            }
+        }
+        combinedTable.setPrimaryKeyNames(pkCandidates);
+        return combinedTable.create();
+    }
+
+}

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresConnectorConfig.java

@@ -7,6 +7,8 @@
 package io.debezium.connector.postgresql;
 
 import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -39,6 +41,8 @@
 import io.debezium.jdbc.JdbcConfiguration;
 import io.debezium.jdbc.TemporalPrecisionMode;
 import io.debezium.relational.RelationalDatabaseConnectorConfig;
+import io.debezium.relational.TableId;
+import io.debezium.relational.Tables.TableFilter;
 
 /**
  * The configuration properties for the {@link PostgresConnector}
@@ -586,32 +590,6 @@ public static SchemaRefreshMode parse(String value) {
                                                         .withImportance(Importance.MEDIUM)
                                                         .withDescription("A name of class to that creates SSL Sockets. Use org.postgresql.ssl.NonValidatingFactory to disable SSL validation in development environments");
 
-    /**
-     * A comma-separated list of regular expressions that match schema names to be monitored.
-     * May not be used with {@link #SCHEMA_BLACKLIST}.
-     */
-    public static final Field SCHEMA_WHITELIST = Field.create("schema.whitelist")
-                                                      .withDisplayName("Schemas")
-                                                      .withType(Type.LIST)
-                                                      .withWidth(Width.LONG)
-                                                      .withImportance(Importance.HIGH)
-                                                      .withDependents(TABLE_WHITELIST_NAME)
-                                                      .withDescription("The schemas for which events should be captured");
-
-    /**
-     * A comma-separated list of regular expressions that match schema names to be excluded from monitoring.
-     * May not be used with {@link #SCHEMA_WHITELIST}.
-     */
-    public static final Field SCHEMA_BLACKLIST = Field.create("schema.blacklist")
-                                                      .withDisplayName("Exclude Schemas")
-                                                      .withType(Type.STRING)
-                                                      .withWidth(Width.LONG)
-                                                      .withImportance(Importance.MEDIUM)
-                                                      .withValidation(PostgresConnectorConfig::validateSchemaBlacklist)
-                                                      .withInvisibleRecommender()
-                                                      .withDescription("");
-
-
     /**
      * A comma-separated list of regular expressions that match the fully-qualified names of tables to be monitored.
      * Fully-qualified names for tables are of the form {@code <schemaName>.<tableName>} or
@@ -792,8 +770,8 @@ protected PostgresConnectorConfig(Configuration config) {
         super(
                 config,
                 config.getString(RelationalDatabaseConnectorConfig.SERVER_NAME),
-                null, // TODO whitelist handling implemented locally here for the time being
-                null,
+                new SystemTablesPredicate(),
+                x -> x.schema() + "." + x.table(),
                 DEFAULT_SNAPSHOT_FETCH_SIZE
         );
 
@@ -924,16 +902,6 @@ protected static ConfigDef configDef() {
         return config;
     }
 
-    private static int validateSchemaBlacklist(Configuration config, Field field, Field.ValidationOutput problems) {
-        String whitelist = config.getString(SCHEMA_WHITELIST);
-        String blacklist = config.getString(SCHEMA_BLACKLIST);
-        if (whitelist != null && blacklist != null) {
-            problems.accept(SCHEMA_BLACKLIST, blacklist, "Schema whitelist is already specified");
-            return 1;
-        }
-        return 0;
-    }
-
     private static int validateTableBlacklist(Configuration config, Field field, Field.ValidationOutput problems) {
         String whitelist = config.getString(TABLE_WHITELIST);
         String blacklist = config.getString(TABLE_BLACKLIST);
@@ -948,4 +916,12 @@ private static int validateTableBlacklist(Configuration config, Field field, Fie
     public String getContextName() {
         return Module.contextName();
     }
-}
+
+    private static class SystemTablesPredicate implements TableFilter {
+        protected static final List<String> SYSTEM_SCHEMAS = Arrays.asList("pg_catalog", "information_schema");
+
+        @Override
+        public boolean isIncluded(TableId t) {
+            return !SYSTEM_SCHEMAS.contains(t.schema().toLowerCase());
+        }
+    }}

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresConnectorTask.java

@@ -10,14 +10,9 @@
 import java.sql.SQLException;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
-import io.debezium.connector.postgresql.snapshot.SnapshotterWrapper;
-import io.debezium.connector.postgresql.spi.Snapshotter;
-import io.debezium.connector.postgresql.spi.SlotState;
-import io.debezium.relational.TableId;
-import io.debezium.schema.TopicSelector;
 import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.source.SourceRecord;
 import org.slf4j.Logger;
@@ -28,6 +23,17 @@
 import io.debezium.connector.base.ChangeEventQueue;
 import io.debezium.connector.common.BaseSourceTask;
 import io.debezium.connector.postgresql.connection.PostgresConnection;
+import io.debezium.connector.postgresql.connection.ReplicationConnection;
+import io.debezium.connector.postgresql.spi.SlotCreationResult;
+import io.debezium.connector.postgresql.spi.SlotState;
+import io.debezium.connector.postgresql.spi.Snapshotter;
+import io.debezium.pipeline.ChangeEventSourceCoordinator;
+import io.debezium.pipeline.DataChangeEvent;
+import io.debezium.pipeline.ErrorHandler;
+import io.debezium.pipeline.EventDispatcher;
+import io.debezium.relational.TableId;
+import io.debezium.schema.TopicSelector;
+import io.debezium.util.Clock;
 import io.debezium.util.LoggingContext;
 
 /**
@@ -37,142 +43,199 @@
  */
 public class PostgresConnectorTask extends BaseSourceTask {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(PostgresConnectorTask.class);
     private static final String CONTEXT_NAME = "postgres-connector-task";
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-    private final AtomicBoolean running = new AtomicBoolean(false);
 
-    private PostgresTaskContext taskContext;
-    private RecordsProducer producer;
+    private static enum State {
+        RUNNING, STOPPED;
+    }
 
-    /**
-     * In case of wal2json, all records of one TX will be sent with the same LSN. This is the last LSN that was
-     * completely processed, i.e. we've seen all events originating from that TX.
-     */
-    private volatile Long lastCompletelyProcessedLsn;
+    private final AtomicReference<State> state = new AtomicReference<State>(State.STOPPED);
 
-    /**
-     * A queue with change events filled by the snapshot and streaming producers, consumed
-     * by Kafka Connect via this task.
-     */
-    private ChangeEventQueue<ChangeEvent> changeEventQueue;
+    private volatile PostgresTaskContext taskContext;
+    private volatile ChangeEventQueue<DataChangeEvent> queue;
+    private volatile PostgresConnection jdbcConnection;
+    private volatile ChangeEventSourceCoordinator coordinator;
+    private volatile ErrorHandler errorHandler;
+    private volatile PostgresSchema schema;
+    private volatile Map<String, ?> lastOffset;
 
     @Override
     public void start(Configuration config) {
-        if (running.get()) {
-            // already running
+        if (!state.compareAndSet(State.STOPPED, State.RUNNING)) {
+            LOGGER.info("Connector has already been started");
             return;
         }
 
-        PostgresConnectorConfig connectorConfig = new PostgresConnectorConfig(config);
-
-        TypeRegistry typeRegistry;
-        Charset databaseCharset;
-
-        try (final PostgresConnection connection = new PostgresConnection(connectorConfig.jdbcConfig())) {
-            typeRegistry = connection.getTypeRegistry();
-            databaseCharset = connection.getDatabaseCharset();
-        }
-
-        Snapshotter snapshotter = connectorConfig.getSnapshotter();
+        final PostgresConnectorConfig connectorConfig = new PostgresConnectorConfig(config);
+        final TopicSelector<TableId> topicSelector = PostgresTopicSelector.create(connectorConfig);
+        final Snapshotter snapshotter = connectorConfig.getSnapshotter();
         if (snapshotter == null) {
-            logger.error("Unable to load snapshotter, if using custom snapshot mode, double check your settings");
+            LOGGER.error("Unable to load snapshotter, if using custom snapshot mode, double check your settings");
             throw new ConnectException("Unable to load snapshotter, if using custom snapshot mode, double check your settings");
         }
 
-        // create the task context and schema...
-        TopicSelector<TableId> topicSelector = PostgresTopicSelector.create(connectorConfig);
-        PostgresSchema schema = new PostgresSchema(connectorConfig, typeRegistry, databaseCharset, topicSelector);
-        this.taskContext = new PostgresTaskContext(connectorConfig, schema, topicSelector);
 
-        SourceInfo sourceInfo = new SourceInfo(connectorConfig);
-        Map<String, Object> existingOffset = context.offsetStorageReader().offset(sourceInfo.partition());
+        jdbcConnection = new PostgresConnection(connectorConfig.jdbcConfig());
+        final TypeRegistry typeRegistry = jdbcConnection.getTypeRegistry();
+        final Charset databaseCharset = jdbcConnection.getDatabaseCharset();
+
+        schema = new PostgresSchema(connectorConfig, typeRegistry, databaseCharset, topicSelector);
+        this.taskContext = new PostgresTaskContext(connectorConfig, schema, topicSelector);
+        final PostgresOffsetContext previousOffset = (PostgresOffsetContext) getPreviousOffset(new PostgresOffsetContext.Loader(connectorConfig));
+        final Clock clock = Clock.system();
+
+        final SourceInfo sourceInfo = new SourceInfo(connectorConfig);
         LoggingContext.PreviousContext previousContext = taskContext.configureLoggingContext(CONTEXT_NAME);
         try {
             //Print out the server information
             SlotState slotInfo = null;
             try (PostgresConnection connection = taskContext.createConnection()) {
-                if (logger.isInfoEnabled()) {
-                    logger.info(connection.serverInfo().toString());
+                if (LOGGER.isInfoEnabled()) {
+                    LOGGER.info(connection.serverInfo().toString());
                 }
                 slotInfo = connection.getReplicationSlotState(connectorConfig.slotName(), connectorConfig.plugin().getPostgresPluginName());
             }
             catch (SQLException e) {
-                logger.warn("unable to load info of replication slot, debezium will try to create the slot");
+                LOGGER.warn("unable to load info of replication slot, debezium will try to create the slot");
             }
 
-            SnapshotterWrapper snapWrapper;
-            if (existingOffset == null) {
-                logger.info("No previous offset found");
+            if (previousOffset == null) {
+                LOGGER.info("No previous offset found");
                 // if we have no initial offset, indicate that to Snapshotter by passing null
-                snapWrapper = new SnapshotterWrapper(snapshotter, connectorConfig, null, slotInfo);
+                snapshotter.init(connectorConfig, null, slotInfo);
             }
             else {
-                logger.info("Found previous offset {}", sourceInfo);
-                sourceInfo.load(existingOffset);
-                snapWrapper = new SnapshotterWrapper(snapshotter, connectorConfig, sourceInfo.asOffsetState(), slotInfo);
+                LOGGER.info("Found previous offset {}", sourceInfo);
+                snapshotter.init(connectorConfig, previousOffset.asOffsetState(), slotInfo);
             }
 
-            createRecordProducer(taskContext, sourceInfo, snapWrapper);
+            ReplicationConnection replicationConnection = null;
+            SlotCreationResult slotCreatedInfo = null;
+            if (snapshotter.shouldStream()) {
+                boolean shouldExport = snapshotter.exportSnapshot();
+                try {
+                    replicationConnection = taskContext.createReplicationConnection(shouldExport);
+                    // we need to create the slot before we start streaming if it doesn't exist
+                    // otherwise we can't stream back changes happening while the snapshot is taking place
+                    if (slotInfo == null) {
+                        slotCreatedInfo = replicationConnection.createReplicationSlot().orElse(null);
+                    }
+                    else {
+                        slotCreatedInfo = null;
+                    }
+                }
+                catch (SQLException ex) {
+                    throw new ConnectException(ex);
+                }
+            }
 
-            changeEventQueue = new ChangeEventQueue.Builder<ChangeEvent>()
+            queue = new ChangeEventQueue.Builder<DataChangeEvent>()
                 .pollInterval(connectorConfig.getPollInterval())
                 .maxBatchSize(connectorConfig.getMaxBatchSize())
                 .maxQueueSize(connectorConfig.getMaxQueueSize())
                 .loggingContextSupplier(() -> taskContext.configureLoggingContext(CONTEXT_NAME))
                 .build();
 
-            producer.start(changeEventQueue::enqueue, changeEventQueue::producerFailure);
-            running.compareAndSet(false, true);
+            errorHandler = new ErrorHandler(PostgresConnector.class, connectorConfig.getLogicalName(), queue, this::cleanupResources);
+
+            final EventDispatcher<TableId> dispatcher = new EventDispatcher<>(
+                    connectorConfig,
+                    topicSelector,
+                    schema,
+                    queue,
+                    connectorConfig.getTableFilters().dataCollectionFilter(),
+                    DataChangeEvent::new);
+                    dispatcher.setInconsistentSchemaHandler(PostgresChangeRecordEmitter::updateSchema);
+
+            coordinator = new ChangeEventSourceCoordinator(
+                    previousOffset,
+                    errorHandler,
+                    PostgresConnector.class,
+                    connectorConfig.getLogicalName(),
+                    new PostgresChangeEventSourceFactory(
+                            connectorConfig,
+                            snapshotter,
+                            jdbcConnection,
+                            errorHandler,
+                            dispatcher,
+                            clock,
+                            schema,
+                            taskContext,
+                            replicationConnection,
+                            slotCreatedInfo
+                    ),
+                    dispatcher,
+                    schema
+            );
+
+            coordinator.start(taskContext, this.queue, new PostgresEventMetadataProvider());
         }
         finally {
             previousContext.restore();
         }
     }
 
-    private void createRecordProducer(PostgresTaskContext taskContext, SourceInfo sourceInfo, SnapshotterWrapper snapshotter) {
-        Snapshotter snapInstance = snapshotter.getSnapshotter();
-        if (snapInstance.shouldSnapshot()) {
-            if (snapInstance.shouldStream()) {
-                logger.info("Taking a new snapshot of the DB and streaming logical changes once the snapshot is finished...");
-                producer = new RecordsSnapshotProducer(taskContext, sourceInfo, snapshotter);
-            }
-            else {
-                logger.info("Taking only a snapshot of the DB without streaming any changes afterwards...");
-                producer = new RecordsSnapshotProducer(taskContext, sourceInfo, snapshotter);
-            }
-        }
-        else if (snapInstance.shouldStream()) {
-            logger.info("Not attempting to take a snapshot, immediately starting to stream logical changes...");
-            producer = new RecordsStreamProducer(taskContext, sourceInfo);
-        }
-        else {
-            throw new ConnectException("Snapshotter neither is snapshotting or streaming, invalid!");
-        }
-    }
-
     @Override
     public void commit() throws InterruptedException {
-        if (running.get()) {
-            if (lastCompletelyProcessedLsn != null) {
-                producer.commit(lastCompletelyProcessedLsn);
-            }
+        if (coordinator != null) {
+            coordinator.commitOffset(lastOffset);
         }
     }
 
     @Override
     public List<SourceRecord> poll() throws InterruptedException {
-        List<ChangeEvent> events = changeEventQueue.poll();
+        final List<DataChangeEvent> records = queue.poll();
 
-        if (events.size() > 0) {
-            lastCompletelyProcessedLsn = events.get(events.size() - 1).getLastCompletelyProcessedLsn();
+        final List<SourceRecord> sourceRecords = records.stream()
+            .map(DataChangeEvent::getRecord)
+            .collect(Collectors.toList());
+
+        if (!sourceRecords.isEmpty()) {
+            this.lastOffset = sourceRecords.get(sourceRecords.size() - 1).sourceOffset();
         }
-        return events.stream().map(ChangeEvent::getRecord).collect(Collectors.toList());
+
+        return sourceRecords;
     }
 
     @Override
     public void stop() {
-        if (running.compareAndSet(true, false)) {
-            producer.stop();
+        cleanupResources();
+    }
+
+    private void cleanupResources() {
+        if (!state.compareAndSet(State.RUNNING, State.STOPPED)) {
+            LOGGER.info("Connector has already been stopped");
+            return;
+        }
+
+        try {
+            if (coordinator != null) {
+                coordinator.stop();
+            }
+        }
+        catch (InterruptedException e) {
+            Thread.interrupted();
+            LOGGER.error("Interrupted while stopping coordinator", e);
+            throw new ConnectException("Interrupted while stopping coordinator, failing the task");
+        }
+
+        try {
+            if (errorHandler != null) {
+                errorHandler.stop();
+            }
+        }
+        catch (InterruptedException e) {
+            Thread.interrupted();
+            LOGGER.error("Interrupted while stopping", e);
+        }
+
+        if (jdbcConnection != null) {
+            jdbcConnection.close();
+        }
+
+        if (schema != null) {
+            schema.close();
         }
     }
 
@@ -185,4 +248,4 @@ public String version() {
     protected Iterable<Field> getAllConfigurationFields() {
         return PostgresConnectorConfig.ALL_FIELDS;
     }
-}
+}

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresEventMetadataProvider.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright Debezium Authors.
+ *
+ * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
+ */
+package io.debezium.connector.postgresql;
+
+import java.time.Instant;
+import java.util.Map;
+
+import org.apache.kafka.connect.data.Struct;
+
+import io.debezium.data.Envelope;
+import io.debezium.pipeline.source.spi.EventMetadataProvider;
+import io.debezium.pipeline.spi.OffsetContext;
+import io.debezium.schema.DataCollectionId;
+import io.debezium.time.Conversions;
+import io.debezium.util.Collect;
+
+class PostgresEventMetadataProvider implements EventMetadataProvider {
+
+    @Override
+    public Instant getEventTimestamp(DataCollectionId source, OffsetContext offset, Object key, Struct value) {
+        if (value == null) {
+            return null;
+        }
+        final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
+        if (source == null) {
+            return null;
+        }
+        if (sourceInfo.schema().field(SourceInfo.TIMESTAMP_USEC_KEY) != null) {
+            final Long timestamp = sourceInfo.getInt64(SourceInfo.TIMESTAMP_USEC_KEY);
+            return timestamp == null ? null : Conversions.toInstantFromMicros(timestamp);
+        }
+        final Long timestamp = sourceInfo.getInt64(SourceInfo.TIMESTAMP_KEY);
+        return timestamp == null ? null : Conversions.toInstantFromMillis(timestamp);
+    }
+
+    @Override
+    public Map<String, String> getEventSourcePosition(DataCollectionId source, OffsetContext offset, Object key, Struct value) {
+        if (value == null) {
+            return null;
+        }
+        final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
+        if (source == null) {
+            return null;
+        }
+        Long xmin = sourceInfo.getInt64(SourceInfo.XMIN_KEY);
+
+        Map<String, String> r = Collect.hashMapOf(
+                SourceInfo.LSN_KEY, Long.toString(sourceInfo.getInt64(SourceInfo.LSN_KEY))
+        );
+        if (xmin != null) {
+            r.put(SourceInfo.XMIN_KEY, Long.toString(xmin));
+        }
+        return r;
+    }
+
+    @Override
+    public String getTransactionId(DataCollectionId source, OffsetContext offset, Object key, Struct value) {
+        if (value == null) {
+            return null;
+        }
+        final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
+        if (source == null) {
+            return null;
+        }
+        return Long.toString(sourceInfo.getInt64(SourceInfo.TXID_KEY));
+    }
+}

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresOffsetContext.java

@@ -0,0 +1,217 @@
+/*
+ * Copyright Debezium Authors.
+ *
+ * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
+ */
+package io.debezium.connector.postgresql;
+
+import java.sql.SQLException;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.Struct;
+import org.apache.kafka.connect.errors.ConnectException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.debezium.connector.SnapshotRecord;
+import io.debezium.connector.postgresql.connection.PostgresConnection;
+import io.debezium.connector.postgresql.connection.ReplicationConnection;
+import io.debezium.connector.postgresql.spi.OffsetState;
+import io.debezium.pipeline.spi.OffsetContext;
+import io.debezium.relational.TableId;
+import io.debezium.time.Conversions;
+import io.debezium.util.Clock;
+
+public class PostgresOffsetContext implements OffsetContext {
+    private static final Logger LOGGER = LoggerFactory.getLogger(PostgresSnapshotChangeEventSource.class);
+
+    private static final String SERVER_PARTITION_KEY = "server";
+    public static final String LAST_COMPLETELY_PROCESSED_LSN_KEY = "lsn_proc";
+
+    private final Schema sourceInfoSchema;
+    private final SourceInfo sourceInfo;
+    private final Map<String, String> partition;
+    private boolean lastSnapshotRecord;
+    private Long lastCompletelyProcessedLsn;
+
+    private PostgresOffsetContext(PostgresConnectorConfig connectorConfig, Long lsn, Long lastCompletelyProcessedLsn, Long txId, Instant time, boolean snapshot, boolean lastSnapshotRecord) {
+        partition = Collections.singletonMap(SERVER_PARTITION_KEY, connectorConfig.getLogicalName());
+        sourceInfo = new SourceInfo(connectorConfig);
+
+        this.lastCompletelyProcessedLsn = lastCompletelyProcessedLsn;
+        sourceInfo.update(lsn, time, txId, null, sourceInfo.xmin());
+        sourceInfoSchema = sourceInfo.schema();
+
+        this.lastSnapshotRecord = lastSnapshotRecord;
+        if (this.lastSnapshotRecord) {
+            postSnapshotCompletion();
+        }
+        else {
+            sourceInfo.setSnapshot(snapshot ? SnapshotRecord.TRUE : SnapshotRecord.FALSE);
+        }
+    }
+
+    @Override
+    public Map<String, ?> getPartition() {
+        return partition;
+    }
+
+    @Override
+    public Map<String, ?> getOffset() {
+        Map<String, Object> result = new HashMap<>();
+        if (sourceInfo.timestamp() != null) {
+            result.put(SourceInfo.TIMESTAMP_USEC_KEY, Conversions.toEpochMicros(sourceInfo.timestamp()));
+        }
+        if (sourceInfo.txId() != null) {
+            result.put(SourceInfo.TXID_KEY, sourceInfo.txId());
+        }
+        if (sourceInfo.lsn() != null) {
+            result.put(SourceInfo.LSN_KEY, sourceInfo.lsn());
+        }
+        if (sourceInfo.xmin() != null) {
+            result.put(SourceInfo.XMIN_KEY, sourceInfo.xmin());
+        }
+        if (sourceInfo.isSnapshot()) {
+            result.put(SourceInfo.SNAPSHOT_KEY, true);
+            result.put(SourceInfo.LAST_SNAPSHOT_RECORD_KEY, lastSnapshotRecord);
+        }
+        if (lastCompletelyProcessedLsn != null) {
+            result.put(LAST_COMPLETELY_PROCESSED_LSN_KEY, lastCompletelyProcessedLsn);
+        }
+        return result;
+    }
+
+    @Override
+    public Schema getSourceInfoSchema() {
+        return sourceInfoSchema;
+    }
+
+    @Override
+    public Struct getSourceInfo() {
+        return sourceInfo.struct();
+    }
+
+    @Override
+    public boolean isSnapshotRunning() {
+        return sourceInfo.isSnapshot();
+    }
+
+    @Override
+    public void preSnapshotStart() {
+        sourceInfo.setSnapshot(SnapshotRecord.TRUE);
+        lastSnapshotRecord = false;
+    }
+
+    @Override
+    public void preSnapshotCompletion() {
+        lastSnapshotRecord = true;
+    }
+
+    @Override
+    public void postSnapshotCompletion() {
+        sourceInfo.setSnapshot(SnapshotRecord.FALSE);
+    }
+
+    public void updateSnapshotPosition(Instant timestamp, TableId tableId) {
+        sourceInfo.update(timestamp, tableId);
+    }
+
+    public void updateWalPosition(Long lsn, Long lastCompletelyProcessedLsn, Instant commitTime, Long txId, TableId tableId, Long xmin) {
+        this.lastCompletelyProcessedLsn = lastCompletelyProcessedLsn;
+        sourceInfo.update(lsn, commitTime, txId, tableId, xmin);
+    }
+
+    boolean hasLastKnownPosition() {
+        return sourceInfo.lsn() != null;
+    }
+
+    Long lsn() {
+        return sourceInfo.lsn();
+    }
+
+    Long xmin() {
+        return sourceInfo.xmin();
+    }
+
+    public static class Loader implements OffsetContext.Loader {
+
+        private final PostgresConnectorConfig connectorConfig;
+
+        public Loader(PostgresConnectorConfig connectorConfig) {
+            this.connectorConfig = connectorConfig;
+        }
+
+        @Override
+        public Map<String, ?> getPartition() {
+            return Collections.singletonMap(SERVER_PARTITION_KEY, connectorConfig.getLogicalName());
+        }
+
+        private Long readOptionalLong(Map<String, ?> offset, String key) {
+            final Object obj = offset.get(key);
+            return (obj == null) ? null : ((Number) obj).longValue();
+        }
+
+        @SuppressWarnings("unchecked")
+        @Override
+        public OffsetContext load(Map<String, ?> offset) {
+            final Long lsn = readOptionalLong(offset, SourceInfo.LSN_KEY);
+            final Long lastCompletelyProcessedLsn = readOptionalLong(offset, LAST_COMPLETELY_PROCESSED_LSN_KEY);
+            final Long txId = readOptionalLong(offset, SourceInfo.TXID_KEY);
+
+            final Instant useconds = Conversions.toInstantFromMicros((Long) offset.get(SourceInfo.TIMESTAMP_USEC_KEY));
+            final boolean snapshot = (boolean) ((Map<String, Object>) offset).getOrDefault(SourceInfo.SNAPSHOT_KEY, Boolean.FALSE);
+            final boolean lastSnapshotRecord = (boolean) ((Map<String, Object>) offset).getOrDefault(SourceInfo.LAST_SNAPSHOT_RECORD_KEY, Boolean.FALSE);
+            return new PostgresOffsetContext(connectorConfig, lsn, lastCompletelyProcessedLsn, txId, useconds, snapshot, lastSnapshotRecord); 
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "PostgresOffsetContext [sourceInfo=" + sourceInfo
+                + ", partition=" + partition
+                + ", lastSnapshotRecord=" + lastSnapshotRecord + "]";
+    }
+
+    public static PostgresOffsetContext initialContext(PostgresConnectorConfig connectorConfig, PostgresConnection jdbcConnection, Clock clock) {
+        try {
+            LOGGER.info("Creating initial offset context");
+            final long lsn = jdbcConnection.currentXLogLocation();
+            final long txId = jdbcConnection.currentTransactionId().longValue();
+            LOGGER.info("Read xlogStart at '{}' from transaction '{}'", ReplicationConnection.format(lsn), txId);
+            return new PostgresOffsetContext(
+                    connectorConfig,
+                    lsn,
+                    null,
+                    txId,
+                    clock.currentTimeAsInstant(),
+                    false,
+                    false);
+        }
+        catch (SQLException e) {
+            throw new ConnectException("Database processing error", e);
+        }
+    }
+
+    public OffsetState asOffsetState() {
+        return new OffsetState(
+                sourceInfo.lsn(),
+                sourceInfo.txId(),
+                sourceInfo.xmin(),
+                sourceInfo.timestamp(),
+                sourceInfo.isSnapshot());
+    }
+
+    @Override
+    public void markLastSnapshotRecord() {
+        sourceInfo.setSnapshot(SnapshotRecord.LAST);
+    }
+
+    @Override
+    public void event(TableId tableId, Instant instant) {
+        sourceInfo.update(instant, tableId);
+    }
+}

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresSnapshotChangeEventSource.java

@@ -0,0 +1,251 @@
+/*
+ * Copyright Debezium Authors.
+ *
+ * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
+ */
+package io.debezium.connector.postgresql;
+
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.time.Duration;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.postgresql.util.PGmoney;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.debezium.connector.postgresql.connection.PostgresConnection;
+import io.debezium.connector.postgresql.connection.ReplicationConnection;
+import io.debezium.connector.postgresql.spi.SlotCreationResult;
+import io.debezium.connector.postgresql.spi.Snapshotter;
+import io.debezium.data.SpecialValueDecimal;
+import io.debezium.pipeline.EventDispatcher;
+import io.debezium.pipeline.source.spi.SnapshotProgressListener;
+import io.debezium.pipeline.spi.OffsetContext;
+import io.debezium.relational.Column;
+import io.debezium.relational.RelationalSnapshotChangeEventSource;
+import io.debezium.relational.Table;
+import io.debezium.relational.TableId;
+import io.debezium.schema.SchemaChangeEvent;
+import io.debezium.schema.SchemaChangeEvent.SchemaChangeEventType;
+import io.debezium.util.Clock;
+
+public class PostgresSnapshotChangeEventSource extends RelationalSnapshotChangeEventSource {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(PostgresSnapshotChangeEventSource.class);
+
+    private final PostgresConnectorConfig connectorConfig;
+    private final PostgresConnection jdbcConnection;
+    private final PostgresSchema schema;
+    private final Snapshotter snapshotter;
+    private final SlotCreationResult slotCreatedInfo;
+
+    public PostgresSnapshotChangeEventSource(PostgresConnectorConfig connectorConfig, Snapshotter snapshotter, PostgresOffsetContext previousOffset, PostgresConnection jdbcConnection, PostgresSchema schema, EventDispatcher<TableId> dispatcher, Clock clock, SnapshotProgressListener snapshotProgressListener, SlotCreationResult slotCreatedInfo) {
+        super(connectorConfig, previousOffset, jdbcConnection, dispatcher, clock, snapshotProgressListener);
+        this.connectorConfig = connectorConfig;
+        this.jdbcConnection = jdbcConnection;
+        this.schema = schema;
+        this.snapshotter = snapshotter;
+        this.slotCreatedInfo = slotCreatedInfo;
+    }
+
+    @Override
+    protected SnapshottingTask getSnapshottingTask(OffsetContext previousOffset) {
+        boolean snapshotSchema = true;
+        boolean snapshotData = true;
+
+        snapshotData = snapshotter.shouldSnapshot();
+        if (snapshotData) {
+            LOGGER.info("According to the connector configuration data will be snapshotted");
+        }
+        else {
+            LOGGER.info("According to the connector configuration no snapshot will be executed");
+            snapshotSchema = false;
+        }
+
+        return new SnapshottingTask(snapshotSchema, snapshotData);
+    }
+
+    @Override
+    protected SnapshotContext prepare(ChangeEventSourceContext context) throws Exception {
+        return new PostgresSnapshotContext(connectorConfig.databaseName());
+    }
+
+    @Override
+    protected void connectionCreated(SnapshotContext snapshotContext) throws Exception {
+        LOGGER.info("Setting isolation level");
+        String transactionStatement = snapshotter.snapshotTransactionIsolationLevelStatement(slotCreatedInfo);
+        LOGGER.info("Opening transaction with statement {}", transactionStatement);
+        jdbcConnection.executeWithoutCommitting(transactionStatement);
+        schema.refresh(jdbcConnection, false);
+    }
+
+    @Override
+    protected Set<TableId> getAllTableIds(SnapshotContext ctx) throws Exception {
+        return jdbcConnection.readTableNames(ctx.catalogName, null, null, new String[] {"TABLE"});
+    }
+
+    @Override
+    protected void lockTablesForSchemaSnapshot(ChangeEventSourceContext sourceContext, SnapshotContext snapshotContext) throws SQLException, InterruptedException {
+        final Duration lockTimeout = connectorConfig.snapshotLockTimeout();
+        final Optional<String> lockStatement = snapshotter.snapshotTableLockingStatement(lockTimeout, schema.tableIds());
+
+        if (lockStatement.isPresent()) {
+            LOGGER.info("Waiting a maximum of '{}' seconds for each table lock", lockTimeout.getSeconds());
+            jdbcConnection.executeWithoutCommitting(lockStatement.get());
+            //now that we have the locks, refresh the schema
+            schema.refresh(jdbcConnection, false);
+        }
+        else {
+            // if we are not in an exported snapshot, this may result in some inconsistencies.
+            // Let the user know
+            if (!snapshotter.exportSnapshot()) {
+                LOGGER.warn("Step 2: skipping locking each table, this may result in inconsistent schema!");
+            }
+            else {
+                LOGGER.info("Step 2: skipping locking each table in an exported snapshot");
+            }
+        }
+    }
+
+    @Override
+    protected void releaseSchemaSnapshotLocks(SnapshotContext snapshotContext) throws SQLException {
+    }
+
+    @Override
+    protected void determineSnapshotOffset(SnapshotContext ctx) throws Exception {
+        PostgresOffsetContext offset = (PostgresOffsetContext) ctx.offset;
+        final long xlogStart = getTransactionStartLsn();
+        final long txId = jdbcConnection.currentTransactionId().longValue();
+        LOGGER.info("Read xlogStart at '{}' from transaction '{}'", ReplicationConnection.format(xlogStart), txId);
+        if (offset == null) {
+            offset = PostgresOffsetContext.initialContext(connectorConfig, jdbcConnection, getClock());
+            ctx.offset = offset;
+        }
+
+        // use the old xmin, as we don't want to update it if in xmin recovery
+        offset.updateWalPosition(xlogStart, null, clock.currentTime(), txId, null, offset.xmin());
+    }
+
+    private long getTransactionStartLsn() throws SQLException {
+        if (snapshotter.exportSnapshot() && slotCreatedInfo != null) {
+            // When performing an exported snapshot based on a newly created replication slot, the txLogStart position
+            // should be based on the replication slot snapshot transaction point.  This is crucial so that if any
+            // SQL operations occur mid-snapshot that they'll be properly captured when streaming begins; otherwise
+            // they'll be lost.
+            return slotCreatedInfo.startLsn();
+        }
+        return jdbcConnection.currentXLogLocation();
+    }
+
+    @Override
+    protected void readTableStructure(ChangeEventSourceContext sourceContext, SnapshotContext snapshotContext) throws SQLException, InterruptedException {
+        Set<String> schemas = snapshotContext.capturedTables.stream()
+                .map(TableId::schema)
+                .collect(Collectors.toSet());
+
+        // reading info only for the schemas we're interested in as per the set of captured tables;
+        // while the passed table name filter alone would skip all non-included tables, reading the schema
+        // would take much longer that way
+        for (String schema : schemas) {
+            if (!sourceContext.isRunning()) {
+                throw new InterruptedException("Interrupted while reading structure of schema " + schema);
+            }
+
+            LOGGER.info("Reading structure of schema '{}'", snapshotContext.catalogName);
+            jdbcConnection.readSchema(
+                    snapshotContext.tables,
+                    snapshotContext.catalogName,
+                    schema,
+                    connectorConfig.getTableFilters().dataCollectionFilter(),
+                    null,
+                    false
+            );
+        }
+        schema.refresh(jdbcConnection, false);
+    }
+
+    @Override
+    protected SchemaChangeEvent getCreateTableEvent(SnapshotContext snapshotContext, Table table) throws SQLException {
+        return new SchemaChangeEvent(
+                snapshotContext.offset.getPartition(),
+                snapshotContext.offset.getOffset(),
+                snapshotContext.catalogName,
+                table.id().schema(),
+                null,
+                table,
+                SchemaChangeEventType.CREATE,
+                true);
+    }
+
+    @Override
+    protected void complete(SnapshotContext snapshotContext) {
+    }
+
+    @Override
+    protected Optional<String> getSnapshotSelect(SnapshotContext snapshotContext, TableId tableId) {
+        return snapshotter.buildSnapshotQuery(tableId);
+    }
+
+    @Override
+    protected Object getColumnValue(ResultSet rs, int columnIndex, Column column) throws SQLException {
+        try {
+            final ResultSetMetaData metaData = rs.getMetaData();
+            final String columnTypeName = metaData.getColumnTypeName(columnIndex);
+            final PostgresType type = schema.getTypeRegistry().get(columnTypeName);
+
+            LOGGER.trace("Type of incoming data is: {}", type.getOid());
+            LOGGER.trace("ColumnTypeName is: {}", columnTypeName);
+            LOGGER.trace("Type is: {}", type);
+
+            if (type.isArrayType()) {
+                return rs.getArray(columnIndex);
+            }
+
+            switch (type.getOid()) {
+                case PgOid.MONEY:
+                    //TODO author=Horia Chiorean date=14/11/2016 description=workaround for https://github.com/pgjdbc/pgjdbc/issues/100
+                    return new PGmoney(rs.getString(columnIndex)).val;
+                case PgOid.BIT:
+                    return rs.getString(columnIndex);
+                case PgOid.NUMERIC:
+                    final String s = rs.getString(columnIndex);
+                    if (s == null) {
+                        return s;
+                    }
+
+                    Optional<SpecialValueDecimal> value = PostgresValueConverter.toSpecialValue(s);
+                    return value.isPresent() ? value.get() : new SpecialValueDecimal(rs.getBigDecimal(columnIndex));
+                case PgOid.TIME:
+                    // To handle time 24:00:00 supported by TIME columns, read the column as a string.
+                case PgOid.TIMETZ:
+                    // In order to guarantee that we resolve TIMETZ columns with proper microsecond precision,
+                    // read the column as a string instead and then re-parse inside the converter.
+                    return rs.getString(columnIndex);
+                default:
+                    Object x = rs.getObject(columnIndex);
+                    if(x != null) {
+                        LOGGER.trace("rs getobject returns class: {}; rs getObject value is: {}", x.getClass(), x);
+                    }
+                    return x;
+            }
+        }
+        catch (SQLException e) {
+            // not a known type
+            return super.getColumnValue(rs, columnIndex, column);
+        }
+    }
+
+    /**
+     * Mutable context which is populated in the course of snapshotting.
+     */
+    private static class PostgresSnapshotContext extends SnapshotContext {
+
+        public PostgresSnapshotContext(String catalogName) throws SQLException {
+            super(catalogName);
+        }
+    }
+}

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresSourceInfoStructMaker.java

@@ -41,17 +41,15 @@ public Struct struct(SourceInfo sourceInfo) {
         Struct result = super.commonStruct(sourceInfo);
         result.put(SourceInfo.SCHEMA_NAME_KEY, sourceInfo.schemaName());
         result.put(SourceInfo.TABLE_NAME_KEY, sourceInfo.tableName());
-        // use the offset information without the snapshot part and usec part
-        sourceInfo.offset().forEach((k, v) -> {
-            if (SourceInfo.TIMESTAMP_USEC_KEY.equals(k) ||
-                    SourceInfo.LAST_SNAPSHOT_RECORD_KEY.equals(k) ||
-                    SourceInfo.SNAPSHOT_KEY.equals(k)) {
-                return;
-            }
-            else {
-                result.put(k, v);
-            }
-        });
+        if (sourceInfo.txId() != null) {
+            result.put(SourceInfo.TXID_KEY, sourceInfo.txId());
+        }
+        if (sourceInfo.lsn() != null) {
+            result.put(SourceInfo.LSN_KEY, sourceInfo.lsn());
+        }
+        if (sourceInfo.xmin() != null) {
+            result.put(SourceInfo.XMIN_KEY, sourceInfo.xmin());
+        }
         return result;
     }
 }

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresStreamingChangeEventSource.java

@@ -3,21 +3,15 @@
  *
  * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
  */
-
 package io.debezium.connector.postgresql;
 
-import java.io.IOException;
 import java.sql.SQLException;
 import java.time.Instant;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 import org.apache.kafka.connect.data.Schema;
@@ -26,279 +20,172 @@
 import org.apache.kafka.connect.source.SourceRecord;
 import org.postgresql.jdbc.PgConnection;
 import org.postgresql.replication.LogSequenceNumber;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import io.debezium.annotation.ThreadSafe;
 import io.debezium.connector.postgresql.connection.PostgresConnection;
 import io.debezium.connector.postgresql.connection.ReplicationConnection;
 import io.debezium.connector.postgresql.connection.ReplicationMessage;
 import io.debezium.connector.postgresql.connection.ReplicationStream;
+import io.debezium.connector.postgresql.spi.Snapshotter;
 import io.debezium.data.Envelope;
 import io.debezium.function.BlockingConsumer;
-import io.debezium.function.Predicates;
-import io.debezium.heartbeat.Heartbeat;
+import io.debezium.pipeline.ErrorHandler;
+import io.debezium.pipeline.EventDispatcher;
+import io.debezium.pipeline.source.spi.StreamingChangeEventSource;
 import io.debezium.relational.Column;
 import io.debezium.relational.ColumnEditor;
 import io.debezium.relational.Table;
 import io.debezium.relational.TableEditor;
 import io.debezium.relational.TableId;
 import io.debezium.relational.TableSchema;
+import io.debezium.schema.TopicSelector;
 import io.debezium.util.Clock;
-import io.debezium.util.LoggingContext;
 import io.debezium.util.Metronome;
 import io.debezium.util.Strings;
-import io.debezium.util.Threads;
 
 /**
- * A {@link RecordsProducer} which creates {@link SourceRecord records} from a
- * Postgres streaming replication connection and {@link ReplicationMessage
- * messages}.
- * <p>
- * See the <a href=
- * "https://jdbc.postgresql.org/documentation/head/replication.html">Physical
- * and Logical replication API</a> to learn more about the underlying Postgres
- * APIs.
  *
- * @author Horia Chiorean (hchiorea@redhat.com)
+ * @author Horia Chiorean (hchiorea@redhat.com), Jiri Pechanec
  */
-@ThreadSafe
-public class RecordsStreamProducer extends RecordsProducer {
+public class PostgresStreamingChangeEventSource implements StreamingChangeEventSource {
 
-    private static final String CONTEXT_NAME = "records-stream-producer";
+    private static final Logger LOGGER = LoggerFactory.getLogger(PostgresStreamingChangeEventSource.class);
 
-    private final ExecutorService executorService;
-    private ReplicationConnection replicationConnection;
-    private final AtomicReference<ReplicationStream> replicationStream;
-    private final AtomicBoolean cleanupExecuted = new AtomicBoolean();
-    private final PostgresConnection metadataConnection;
+    private final PostgresConnection connection;
+    private final EventDispatcher<TableId> dispatcher;
+    private final ErrorHandler errorHandler;
+    private final Clock clock;
+    private final PostgresSchema schema;
+    private final PostgresOffsetContext offsetContext;
+    private final PostgresConnectorConfig connectorConfig;
+    private final PostgresTaskContext taskContext;
+    private final ReplicationConnection replicationConnection;
+    private final AtomicReference<ReplicationStream> replicationStream = new AtomicReference<>();
     private Long lastCompletelyProcessedLsn;
-
-    /**
-     * Serves as a message box between Kafka Connect main loop thread and stream producer thread.
-     * Kafka Connect thread sends the message with the value of LSN of the last transaction that was
-     * fully sent to Kafka and appropriate offsets committed.
-     * <p>
-     * Stream producer thread receives the LSN that is certain that was delivered to Kafka topic
-     * and flushes the LSN to PostgreSQL database server so the associated WAL segment can be released.
-     */
-    private final AtomicLong lastCommittedLsn = new AtomicLong(-1);
+    private final Snapshotter snapshotter;
     private final Metronome pauseNoMessage;
 
-    private final Heartbeat heartbeat;
-
-    @FunctionalInterface
-    public static interface PgConnectionSupplier {
-        PgConnection get() throws SQLException;
-    }
-
-    /**
-     * Creates new producer instance for the given task context
-     *
-     * @param taskContext a {@link PostgresTaskContext}, never null
-     * @param sourceInfo a {@link SourceInfo} instance to track stored offsets
-     * @param replicationConnection a {@link ReplicationConnection} that is used to perform actual replication
-     */
-    public RecordsStreamProducer(PostgresTaskContext taskContext,
-                                 SourceInfo sourceInfo,
-                                 ReplicationConnection replicationConnection) {
-        super(taskContext, sourceInfo);
-        executorService = Threads.newSingleThreadExecutor(PostgresConnector.class, taskContext.config().getLogicalName(), CONTEXT_NAME);
-        this.replicationStream = new AtomicReference<>();
-        this.replicationConnection = replicationConnection;
-
-        heartbeat = Heartbeat.create(taskContext.config().getConfig(), taskContext.topicSelector().getHeartbeatTopic(),
-                taskContext.config().getLogicalName());
+    public PostgresStreamingChangeEventSource(PostgresConnectorConfig connectorConfig, Snapshotter snapshotter, PostgresOffsetContext offsetContext, PostgresConnection connection, EventDispatcher<TableId> dispatcher, ErrorHandler errorHandler, Clock clock, PostgresSchema schema, PostgresTaskContext taskContext, ReplicationConnection replicationConnection) {
+        this.connectorConfig = connectorConfig;
+        this.connection = connection;
+        this.dispatcher = dispatcher;
+        this.errorHandler = errorHandler;
+        this.clock = clock;
+        this.schema = schema;
+        this.offsetContext = (offsetContext != null) ? offsetContext :
+            PostgresOffsetContext.initialContext(connectorConfig, connection, clock);
         pauseNoMessage = Metronome.sleeper(taskContext.getConfig().getPollInterval(), Clock.SYSTEM);
-        metadataConnection = taskContext.createConnection();
-    }
-
-    // this maybe should only be used for testing?
-    public RecordsStreamProducer(PostgresTaskContext taskContext,
-                                 SourceInfo sourceInfo) {
-        this(taskContext, sourceInfo, null);
-        try {
-            this.replicationConnection = taskContext.createReplicationConnection(false);
-        }
-        catch (SQLException e) {
-            throw new ConnectException(e);
-        }
+        this.taskContext = taskContext;
+        this.snapshotter = snapshotter;
+        this.replicationConnection = replicationConnection;
     }
 
     @Override
-    protected synchronized void start(BlockingConsumer<ChangeEvent> eventConsumer, Consumer<Throwable> failureConsumer)  {
-        LoggingContext.PreviousContext previousContext = taskContext.configureLoggingContext(CONTEXT_NAME);
+    public void execute(ChangeEventSourceContext context) throws InterruptedException {
+        if (!snapshotter.shouldStream()) {
+            LOGGER.info("Streaming is not enabled in currect configuration");
+            return;
+        }
+
         try {
-            if (executorService.isShutdown()) {
-                logger.info("Streaming will not start, stop already requested");
-                return;
-            }
-            if (sourceInfo.hasLastKnownPosition()) {
+            if (offsetContext.hasLastKnownPosition()) {
                 // start streaming from the last recorded position in the offset
-                Long lsn = sourceInfo.lsn();
-                if (logger.isDebugEnabled()) {
-                    logger.debug("retrieved latest position from stored offset '{}'", ReplicationConnection.format(lsn));
+                Long lsn = offsetContext.lsn();
+                if (LOGGER.isDebugEnabled()) {
+                    LOGGER.debug("retrieved latest position from stored offset '{}'", ReplicationConnection.format(lsn));
                 }
                 replicationStream.compareAndSet(null, replicationConnection.startStreaming(lsn));
             } else {
-                logger.info("no previous LSN found in Kafka, streaming from the latest xlogpos or flushed LSN...");
+                LOGGER.info("no previous LSN found in Kafka, streaming from the latest xlogpos or flushed LSN...");
                 replicationStream.compareAndSet(null, replicationConnection.startStreaming());
             }
 
-            // for large databases with many tables, we can timeout the slot while refreshing schema
-            // so we need to start a background thread that just responds to keep alive
-            replicationStream.get().startKeepAlive(Threads.newSingleThreadExecutor(PostgresConnector.class, taskContext.config().getLogicalName(), CONTEXT_NAME + "-keep-alive"));
             // refresh the schema so we have a latest view of the DB tables
-            taskContext.refreshSchema(metadataConnection, true);
-            taskContext.schema().assureNonEmptySchema();
+            taskContext.refreshSchema(connection, true);
 
-            this.lastCompletelyProcessedLsn = sourceInfo.lsn();
+            this.lastCompletelyProcessedLsn = offsetContext.lsn();
 
-            // the new thread will inherit it's parent MDC
-            executorService.submit(() -> streamChanges(eventConsumer, failureConsumer));
-        } catch (Throwable t) {
-            throw new ConnectException(t.getCause() != null ? t.getCause() : t);
-        } finally {
-            previousContext.restore();
-        }
-    }
-
-    private void streamChanges(BlockingConsumer<ChangeEvent> consumer, Consumer<Throwable> failureConsumer) {
-        ReplicationStream stream = this.replicationStream.get();
-        // once we are streaming changes, we stop the keep alive, as the read loop
-        // will ensure that happens
-        stream.stopKeepAlive();
-        // run while we haven't been requested to stop
-        while (!Thread.currentThread().isInterrupted()) {
-            try {
-                flushLatestCommittedLsn(stream);
-                // this will block until a message is available
-                if (!stream.readPending(x -> process(x, stream.lastReceivedLsn(), consumer))) {
-                    if (lastCompletelyProcessedLsn != null) {
-                        heartbeat.heartbeat(sourceInfo.partition(), sourceInfo.offset(),
-                                r -> consumer.accept(new ChangeEvent(r, lastCompletelyProcessedLsn)));
+            final ReplicationStream stream = this.replicationStream.get();
+            while (context.isRunning()) {
+                stream.readPending(message -> {
+                    final Long lsn = stream.lastReceivedLsn();
+                    if (message == null) {
+                        LOGGER.trace("Received empty message");
+                        lastCompletelyProcessedLsn = lsn;
+                        pauseNoMessage.pause();
+                        return;
                     }
-                    pauseNoMessage.pause();
-                }
-            }
-            catch (SQLException e) {
-                Throwable cause = e.getCause();
-                if (cause != null && (cause instanceof IOException)) {
-                    //TODO author=Horia Chiorean date=08/11/2016 description=this is because we can't safely close the stream atm
-                    logger.warn("Closing replication stream due to db connection IO exception...");
-                } else {
-                    logger.error("unexpected exception while streaming logical changes", e);
-                }
-                failureConsumer.accept(e);
-                throw new ConnectException(e);
-            }
-            catch (InterruptedException e) {
-                logger.info("Interrupted from sleep, producer termination requested");
-                Thread.currentThread().interrupt();
-            }
-            catch (Throwable e) {
-                logger.error("unexpected exception while streaming logical changes", e);
-                failureConsumer.accept(e);
-                throw new ConnectException(e);
+                    if (message.isLastEventForLsn()) {
+                        lastCompletelyProcessedLsn = lsn;
+                    }
+
+                    final TableId tableId = PostgresSchema.parse(message.getTable());
+                    Objects.requireNonNull(tableId);
+
+                    offsetContext.updateWalPosition(lsn, lastCompletelyProcessedLsn, message.getCommitTime(), message.getTransactionId(), tableId, taskContext.getSlotXmin(connection));
+                    dispatcher
+                        .dispatchDataChangeEvent(
+                                tableId,
+                                new PostgresChangeRecordEmitter(
+                                        offsetContext,
+                                        clock,
+                                        connectorConfig,
+                                        schema,
+                                        connection,
+                                        message
+                                )
+                        );
+                });
             }
         }
-    }
-
-    private void flushLatestCommittedLsn(ReplicationStream stream) throws SQLException {
-        final long newLsn = lastCommittedLsn.getAndSet(-1);
-        if (newLsn != -1) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("Flushing LSN to server: {}", LogSequenceNumber.valueOf(newLsn));
-            }
-            // tell the server the point up to which we've processed data, so it can be free to recycle WAL segments
-            stream.flushLsn(newLsn);
-        }
-    }
-
-    @Override
-    protected synchronized void commit(long lsn)  {
-        LoggingContext.PreviousContext previousContext = taskContext.configureLoggingContext(CONTEXT_NAME);
-        try {
-            if (logger.isDebugEnabled()) {
-                logger.debug("Flushing of LSN '{}' requested", LogSequenceNumber.valueOf(lsn));
-            }
-            lastCommittedLsn.set(lsn);
+        catch (Throwable e) {
+            errorHandler.setProducerThrowable(e);
         }
         finally {
-            previousContext.restore();
-        }
-    }
-
-    @Override
-    protected synchronized void stop() {
-        LoggingContext.PreviousContext previousContext = taskContext.configureLoggingContext(CONTEXT_NAME);
-
-        try {
-            if (!cleanupExecuted.compareAndSet(false, true)) {
-                logger.debug("already stopped....");
-                return;
-            }
-
-            ReplicationStream stream = this.replicationStream.get();
-            // if we have a stream, ensure that it has been stopped
-            if (stream != null) {
-                try {
-                    flushLatestCommittedLsn(stream);
-                }
-                catch (SQLException e) {
-                    logger.error("Failed to execute the final LSN flush", e);
-                }
-                stream.stopKeepAlive();
-            }
-
-            closeConnections();
-        } finally {
-            replicationStream.set(null);
-            executorService.shutdownNow();
-            previousContext.restore();
-        }
-    }
-
-    private void closeConnections() {
-        Exception closingException = null;
-
-        try {
             if (replicationConnection != null) {
-                logger.debug("stopping streaming...");
+                LOGGER.debug("stopping streaming...");
                 //TODO author=Horia Chiorean date=08/11/2016 description=Ideally we'd close the stream, but it's not reliable atm (see javadoc)
                 //replicationStream.close();
                 // close the connection - this should also disconnect the current stream even if it's blocking
-                replicationConnection.close();
-            }
-        }
-        catch(Exception e) {
-            closingException = e;
-        }
-        finally {
-            try {
-                if (metadataConnection != null) {
-                    metadataConnection.close();
+                try {
+                    replicationConnection.close();
+                }
+                catch (Exception e) {
                 }
             }
-            catch(Exception e) {
-                ConnectException rethrown = new ConnectException(e);
-                if (closingException != null) {
-                    rethrown.addSuppressed(closingException);
+        }
+    }
+
+    @Override
+    public void commitOffset(Map<String, ?> offset) {
+        try {
+            ReplicationStream replicationStream = this.replicationStream.get();
+            final Long lsn = (Long) offset.get(PostgresOffsetContext.LAST_COMPLETELY_PROCESSED_LSN_KEY);
+
+            if (replicationStream != null && lsn != null) {
+                if (LOGGER.isDebugEnabled()) {
+                    LOGGER.debug("Flushing LSN to server: {}", LogSequenceNumber.valueOf(lsn));
                 }
-
-                throw rethrown;
+                // tell the server the point up to which we've processed data, so it can be free to recycle WAL segments
+                replicationStream.flushLsn(lsn);
             }
-
-            if (closingException != null) {
-                throw new ConnectException(closingException);
+            else {
+                LOGGER.debug("Streaming has already stopped, ignoring commit callback...");
             }
         }
+        catch (SQLException e) {
+            throw new ConnectException(e);
+        }
     }
 
     private void process(ReplicationMessage message, Long lsn, BlockingConsumer<ChangeEvent> consumer) throws SQLException, InterruptedException {
         // in some cases we can get null if PG gives us back a message earlier than the latest reported flushed LSN.
         // WAL2JSON can also send empty changes for DDL, materialized views, etc. and the heartbeat still needs to fire.
         if (message == null) {
-            logger.trace("Received empty message");
+            LOGGER.trace("Received empty message");
             lastCompletelyProcessedLsn = lsn;
+            // TODO heartbeat
             return;
         }
         if (message.isLastEventForLsn()) {
@@ -311,9 +198,9 @@ private void process(ReplicationMessage message, Long lsn, BlockingConsumer<Chan
         // update the source info with the coordinates for this message
         Instant commitTime = message.getCommitTime();
         long txId = message.getTransactionId();
-        sourceInfo.update(lsn, commitTime, txId, tableId, taskContext.getSlotXmin(metadataConnection));
-        if (logger.isDebugEnabled()) {
-            logger.debug("received new message at position {}\n{}", ReplicationConnection.format(lsn), message);
+        offsetContext.updateWalPosition(lsn, null, commitTime, txId, tableId, taskContext.getSlotXmin(connection));
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug("received new message at position {}\n{}", ReplicationConnection.format(lsn), message);
         }
 
         TableSchema tableSchema = tableSchemaFor(tableId);
@@ -322,12 +209,12 @@ private void process(ReplicationMessage message, Long lsn, BlockingConsumer<Chan
             ReplicationMessage.Operation operation = message.getOperation();
             switch (operation) {
                 case INSERT: {
-                    Object[] row = columnValues(message.getNewTupleList(), tableId, message.shouldSchemaBeSynchronized(), message.hasTypeMetadata());
+                    Object[] row = columnValues(message.getNewTupleList(), tableId, true, message.hasTypeMetadata());
                     generateCreateRecord(tableId, row, consumer);
                     break;
                 }
                 case UPDATE: {
-                    Object[] newRow = columnValues(message.getNewTupleList(), tableId, message.shouldSchemaBeSynchronized(), message.hasTypeMetadata());
+                    Object[] newRow = columnValues(message.getNewTupleList(), tableId, true, message.hasTypeMetadata());
                     Object[] oldRow = columnValues(message.getOldTupleList(), tableId, false, message.hasTypeMetadata());
                     generateUpdateRecord(tableId, oldRow, newRow, consumer);
                     break;
@@ -338,36 +225,40 @@ private void process(ReplicationMessage message, Long lsn, BlockingConsumer<Chan
                     break;
                 }
                 default: {
-                   logger.warn("unknown message operation: {}", operation);
+                   LOGGER.warn("unknown message operation: {}", operation);
                 }
             }
         }
+
+        if (message.isLastEventForLsn()) {
+            // TODO heartbeat
+        }
     }
 
     protected void generateCreateRecord(TableId tableId, Object[] rowData, BlockingConsumer<ChangeEvent> recordConsumer) throws InterruptedException {
         if (rowData == null || rowData.length == 0) {
-            logger.warn("no new values found for table '{}' from update message at '{}'; skipping record", tableId, sourceInfo);
+            LOGGER.warn("no new values found for table '{}' from update message at '{}'; skipping record", tableId, offsetContext);
             return;
         }
-        TableSchema tableSchema = schema().schemaFor(tableId);
+        TableSchema tableSchema = schema.schemaFor(tableId);
         assert tableSchema != null;
         Object key = tableSchema.keyFromColumnData(rowData);
-        logger.trace("key value is: {}", key);
+        LOGGER.trace("key value is: {}", key);
         Struct value = tableSchema.valueFromColumnData(rowData);
         if (value == null) {
-            logger.warn("no values found for table '{}' from create message at '{}'; skipping record", tableId, sourceInfo);
+            LOGGER.warn("no values found for table '{}' from create message at '{}'; skipping record", tableId, offsetContext);
             return;
         }
         Schema keySchema = tableSchema.keySchema();
-        Map<String, ?> partition = sourceInfo.partition();
-        Map<String, ?> offset = sourceInfo.offset();
+        Map<String, ?> partition = offsetContext.getPartition();
+        Map<String, ?> offset = offsetContext.getOffset();
         String topicName = topicSelector().topicNameFor(tableId);
         Envelope envelope = tableSchema.getEnvelopeSchema();
 
         SourceRecord record = new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(),
-                                               envelope.create(value, sourceInfo.struct(), clock().currentTimeInMillis()));
-        if (logger.isDebugEnabled()) {
-            logger.debug("sending create event '{}' to topic '{}'", record, topicName);
+                                               envelope.create(value, offsetContext.getSourceInfo(), clock.currentTimeInMillis()));
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug("sending create event '{}' to topic '{}'", record, topicName);
         }
         recordConsumer.accept(new ChangeEvent(record, lastCompletelyProcessedLsn));
     }
@@ -375,14 +266,14 @@ protected void generateCreateRecord(TableId tableId, Object[] rowData, BlockingC
     protected void generateUpdateRecord(TableId tableId, Object[] oldRowData, Object[] newRowData,
                                         BlockingConsumer<ChangeEvent> recordConsumer) throws InterruptedException {
         if (newRowData == null || newRowData.length == 0) {
-            logger.warn("no values found for table '{}' from update message at '{}'; skipping record" , tableId, sourceInfo);
+            LOGGER.warn("no values found for table '{}' from update message at '{}'; skipping record" , tableId, offsetContext);
             return;
         }
         Schema oldKeySchema = null;
         Struct oldValue = null;
         Object oldKey = null;
 
-        TableSchema tableSchema = schema().schemaFor(tableId);
+        TableSchema tableSchema = schema.schemaFor(tableId);
         assert tableSchema != null;
 
         if (oldRowData != null && oldRowData.length > 0) {
@@ -395,11 +286,11 @@ protected void generateUpdateRecord(TableId tableId, Object[] oldRowData, Object
         Struct newValue = tableSchema.valueFromColumnData(newRowData);
 
         Schema newKeySchema = tableSchema.keySchema();
-        Map<String, ?> partition = sourceInfo.partition();
-        Map<String, ?> offset = sourceInfo.offset();
+        Map<String, ?> partition =  offsetContext.getPartition();
+        Map<String, ?> offset = offsetContext.getOffset();
         String topicName = topicSelector().topicNameFor(tableId);
         Envelope envelope = tableSchema.getEnvelopeSchema();
-        Struct source = sourceInfo.struct();
+        Struct source = offsetContext.getSourceInfo();
 
         if (oldKey != null && !Objects.equals(oldKey, newKey)) {
             // the primary key has changed, so we need to send a DELETE followed by a CREATE
@@ -408,10 +299,10 @@ protected void generateUpdateRecord(TableId tableId, Object[] oldRowData, Object
             ChangeEvent changeEvent = new ChangeEvent(
                     new SourceRecord(
                             partition, offset, topicName, null, oldKeySchema, oldKey, envelope.schema(),
-                            envelope.delete(oldValue, source, clock().currentTimeInMillis())),
+                            envelope.delete(oldValue, source, clock.currentTimeInMillis())),
                     lastCompletelyProcessedLsn);
-            if (logger.isDebugEnabled()) {
-                logger.debug("sending delete event '{}' to topic '{}'", changeEvent.getRecord(), topicName);
+            if (LOGGER.isDebugEnabled()) {
+                LOGGER.debug("sending delete event '{}' to topic '{}'", changeEvent.getRecord(), topicName);
             }
             recordConsumer.accept(changeEvent);
 
@@ -420,8 +311,8 @@ protected void generateUpdateRecord(TableId tableId, Object[] oldRowData, Object
                 changeEvent = new ChangeEvent(
                         new SourceRecord(partition, offset, topicName, null, oldKeySchema, oldKey, null, null),
                         lastCompletelyProcessedLsn);
-                if (logger.isDebugEnabled()) {
-                    logger.debug("sending tombstone event '{}' to topic '{}'", changeEvent.getRecord(), topicName);
+                if (LOGGER.isDebugEnabled()) {
+                    LOGGER.debug("sending tombstone event '{}' to topic '{}'", changeEvent.getRecord(), topicName);
                 }
                 recordConsumer.accept(changeEvent);
             }
@@ -430,36 +321,36 @@ protected void generateUpdateRecord(TableId tableId, Object[] oldRowData, Object
             changeEvent = new ChangeEvent(
                     new SourceRecord(
                             partition, offset, topicName, null, newKeySchema, newKey, envelope.schema(),
-                            envelope.create(newValue, source, clock().currentTimeInMillis())),
+                            envelope.create(newValue, source, clock.currentTimeInMillis())),
                     lastCompletelyProcessedLsn);
-            if (logger.isDebugEnabled()) {
-                logger.debug("sending create event '{}' to topic '{}'", changeEvent.getRecord(), topicName);
+            if (LOGGER.isDebugEnabled()) {
+                LOGGER.debug("sending create event '{}' to topic '{}'", changeEvent.getRecord(), topicName);
             }
             recordConsumer.accept(changeEvent);
         } else {
             SourceRecord record = new SourceRecord(partition, offset, topicName, null,
                                                    newKeySchema, newKey, envelope.schema(),
-                                                   envelope.update(oldValue, newValue, source, clock().currentTimeInMillis()));
+                                                   envelope.update(oldValue, newValue, source, clock.currentTimeInMillis()));
             recordConsumer.accept(new ChangeEvent(record, lastCompletelyProcessedLsn));
         }
     }
 
     protected void generateDeleteRecord(TableId tableId, Object[] oldRowData, BlockingConsumer<ChangeEvent> recordConsumer) throws InterruptedException {
         if (oldRowData == null || oldRowData.length == 0) {
-            logger.warn("no values found for table '{}' from delete message at '{}'; skipping record" , tableId, sourceInfo);
+            LOGGER.warn("no values found for table '{}' from delete message at '{}'; skipping record" , tableId, offsetContext);
             return;
         }
-        TableSchema tableSchema = schema().schemaFor(tableId);
+        TableSchema tableSchema = schema.schemaFor(tableId);
         assert tableSchema != null;
         Object key = tableSchema.keyFromColumnData(oldRowData);
         Struct value = tableSchema.valueFromColumnData(oldRowData);
         if (value == null) {
-            logger.warn("ignoring delete message for table '{}' because it does not have a primary key defined and replica identity for the table is not FULL", tableId);
+            LOGGER.warn("ignoring delete message for table '{}' because it does not have a primary key defined and replica identity for the table is not FULL", tableId);
             return;
         }
         Schema keySchema = tableSchema.keySchema();
-        Map<String, ?> partition = sourceInfo.partition();
-        Map<String, ?> offset = sourceInfo.offset();
+        Map<String, ?> partition =  offsetContext.getPartition();
+        Map<String, ?> offset = offsetContext.getOffset();
         String topicName = topicSelector().topicNameFor(tableId);
         Envelope envelope = tableSchema.getEnvelopeSchema();
 
@@ -468,10 +359,10 @@ protected void generateDeleteRecord(TableId tableId, Object[] oldRowData, Blocki
                 new SourceRecord(
                         partition, offset, topicName, null,
                         keySchema, key, envelope.schema(),
-                        envelope.delete(value, sourceInfo.struct(), clock().currentTimeInMillis())),
+                        envelope.delete(value, offsetContext.getSourceInfo(), clock.currentTimeInMillis())),
                 lastCompletelyProcessedLsn);
-        if (logger.isDebugEnabled()) {
-            logger.debug("sending delete event '{}' to topic '{}'", changeEvent.getRecord(), topicName);
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug("sending delete event '{}' to topic '{}'", changeEvent.getRecord(), topicName);
         }
         recordConsumer.accept(changeEvent);
 
@@ -480,8 +371,8 @@ protected void generateDeleteRecord(TableId tableId, Object[] oldRowData, Blocki
             changeEvent = new ChangeEvent(
                     new SourceRecord(partition, offset, topicName, null, keySchema, key, null, null),
                     lastCompletelyProcessedLsn);
-            if (logger.isDebugEnabled()) {
-                logger.debug("sending tombstone event '{}' to topic '{}'", changeEvent.getRecord(), topicName);
+            if (LOGGER.isDebugEnabled()) {
+                LOGGER.debug("sending tombstone event '{}' to topic '{}'", changeEvent.getRecord(), topicName);
             }
             recordConsumer.accept(changeEvent);
         }
@@ -492,40 +383,40 @@ private Object[] columnValues(List<ReplicationMessage.Column> columns, TableId t
         if (columns == null || columns.isEmpty()) {
             return null;
         }
-        Table table = schema().tableFor(tableId);
+        Table table = schema.tableFor(tableId);
         assert table != null;
 
         // check if we need to refresh our local schema due to DB schema changes for this table
         if (refreshSchemaIfChanged && schemaChanged(columns, table, metadataInMessage)) {
-            // Refresh the schema so we get information about primary keys
-            schema().refresh(metadataConnection, tableId, taskContext.config().skipRefreshSchemaOnMissingToastableData());
-            // Update the schema with metadata coming from decoder message
-            if (metadataInMessage) {
-                schema().refresh(tableFromFromMessage(columns, schema().tableFor(tableId)));
+            try (final PostgresConnection connection = taskContext.createConnection()) {
+                // Refresh the schema so we get information about primary keys
+                schema.refresh(connection, tableId, taskContext.config().skipRefreshSchemaOnMissingToastableData());
+                // Update the schema with metadata coming from decoder message
+                if (metadataInMessage) {
+                    schema.refresh(tableFromFromMessage(columns, schema.tableFor(tableId)));
+                }
+                table = schema.tableFor(tableId);
             }
-            table = schema().tableFor(tableId);
         }
 
         // based on the schema columns, create the values on the same position as the columns
         List<Column> schemaColumns = table.columns();
-        // based on the replication message without toasted columns for now
-        List<ReplicationMessage.Column> columnsWithoutToasted = columns.stream().filter(Predicates.not(ReplicationMessage.Column::isToastedColumn)).collect(Collectors.toList());
         // JSON does not deliver a list of all columns for REPLICA IDENTITY DEFAULT
-        Object[] values = new Object[columnsWithoutToasted.size() < schemaColumns.size() ? schemaColumns.size() : columnsWithoutToasted.size()];
+        Object[] values = new Object[columns.size() < schemaColumns.size() ? schemaColumns.size() : columns.size()];
 
-        for (ReplicationMessage.Column column : columnsWithoutToasted) {
+        for (ReplicationMessage.Column column : columns) {
             //DBZ-298 Quoted column names will be sent like that in messages, but stored unquoted in the column names
             final String columnName = Strings.unquoteIdentifierPart(column.getName());
             final Column tableColumn = table.columnWithName(columnName);
             if (tableColumn == null) {
-                logger.warn(
+                LOGGER.warn(
                         "Internal schema is out-of-sync with incoming decoder events; column {} will be omitted from the change event.",
                         column.getName());
                 continue;
             }
             int position = tableColumn.position() - 1;
             if (position < 0 || position >= values.length) {
-                logger.warn(
+                LOGGER.warn(
                         "Internal schema is out-of-sync with incoming decoder events; column {} will be omitted from the change event.",
                         column.getName());
                 continue;
@@ -555,7 +446,7 @@ private boolean schemaChanged(List<ReplicationMessage.Column> columns, Table tab
         if (msgHasMissingColumns || msgHasAdditionalColumns) {
             // the table metadata has less or more columns than the event, which means the table structure has changed,
             // so we need to trigger a refresh...
-            logger.info("Different column count {} present in the server message as schema in memory contains {}; refreshing table schema",
+            LOGGER.info("Different column count {} present in the server message as schema in memory contains {}; refreshing table schema",
                         replicationColumnCount,
                         tableColumnCount);
             return true;
@@ -567,14 +458,14 @@ private boolean schemaChanged(List<ReplicationMessage.Column> columns, Table tab
             String columnName = message.getName();
             Column column = table.columnWithName(columnName);
             if (column == null) {
-                logger.info("found new column '{}' present in the server message which is not part of the table metadata; refreshing table schema", columnName);
+                LOGGER.info("found new column '{}' present in the server message which is not part of the table metadata; refreshing table schema", columnName);
                 return true;
             }
             else {
                 final int localType = column.nativeType();
                 final int incomingType = message.getType().getOid();
                 if (localType != incomingType) {
-                    logger.info("detected new type for column '{}', old type was {} ({}), new type is {} ({}); refreshing table schema", columnName, localType, column.typeName(),
+                    LOGGER.info("detected new type for column '{}', old type was {} ({}), new type is {} ({}); refreshing table schema", columnName, localType, column.typeName(),
                                 incomingType, message.getType().getName());
                     return true;
                 }
@@ -582,21 +473,21 @@ private boolean schemaChanged(List<ReplicationMessage.Column> columns, Table tab
                     final int localLength = column.length();
                     final int incomingLength = message.getTypeMetadata().getLength();
                     if (localLength != incomingLength) {
-                        logger.info("detected new length for column '{}', old length was {}, new length is {}; refreshing table schema", columnName, localLength,
+                        LOGGER.info("detected new length for column '{}', old length was {}, new length is {}; refreshing table schema", columnName, localLength,
                                     incomingLength);
                         return true;
                     }
                     final int localScale = column.scale().get();
                     final int incomingScale = message.getTypeMetadata().getScale();
                     if (localScale != incomingScale) {
-                        logger.info("detected new scale for column '{}', old scale was {}, new scale is {}; refreshing table schema", columnName, localScale,
+                        LOGGER.info("detected new scale for column '{}', old scale was {}, new scale is {}; refreshing table schema", columnName, localScale,
                                     incomingScale);
                         return true;
                     }
                     final boolean localOptional = column.isOptional();
                     final boolean incomingOptional = message.isOptional();
                     if (localOptional != incomingOptional) {
-                        logger.info("detected new optional status for column '{}', old value was {}, new value is {}; refreshing table schema", columnName, localOptional, incomingOptional);
+                        LOGGER.info("detected new optional status for column '{}', old value was {}, new value is {}; refreshing table schema", columnName, localOptional, incomingOptional);
                         return true;
                     }
                 }
@@ -617,10 +508,10 @@ private boolean hasMissingUntoastedColumns(Table table, List<ReplicationMessage.
                 .map(Column::name)
                 .collect(Collectors.toList());
 
-        List<String> toastableColumns = schema().getToastableColumnsForTableId(table.id());
+        List<String> toastableColumns = schema.getToastableColumnsForTableId(table.id());
 
-        if (logger.isDebugEnabled()) {
-            logger.debug("msg columns: '{}' --- missing columns: '{}' --- toastableColumns: '{}",
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug("msg columns: '{}' --- missing columns: '{}' --- toastableColumns: '{}",
                     String.join(",", msgColumnNames),
                     String.join(",", missingColumnNames),
                     String.join(",", toastableColumns));
@@ -631,9 +522,8 @@ private boolean hasMissingUntoastedColumns(Table table, List<ReplicationMessage.
     }
 
     private TableSchema tableSchemaFor(TableId tableId) throws SQLException {
-        PostgresSchema schema = schema();
         if (schema.isFilteredOut(tableId)) {
-            logger.debug("table '{}' is filtered out, ignoring", tableId);
+            LOGGER.debug("table '{}' is filtered out, ignoring", tableId);
             return null;
         }
         TableSchema tableSchema = schema.schemaFor(tableId);
@@ -642,19 +532,21 @@ private TableSchema tableSchemaFor(TableId tableId) throws SQLException {
         }
         // we don't have a schema registered for this table, even though the filters would allow it...
         // which means that is a newly created table; so refresh our schema to get the definition for this table
-        schema.refresh(metadataConnection, tableId, taskContext.config().skipRefreshSchemaOnMissingToastableData());
+        try (final PostgresConnection connection = taskContext.createConnection()) {
+            schema.refresh(connection, tableId, taskContext.config().skipRefreshSchemaOnMissingToastableData());
+        }
         tableSchema = schema.schemaFor(tableId);
         if (tableSchema == null) {
-            logger.warn("cannot load schema for table '{}'", tableId);
+            LOGGER.warn("cannot load schema for table '{}'", tableId);
             return null;
         } else {
-            logger.debug("refreshed DB schema to include table '{}'", tableId);
+            LOGGER.debug("refreshed DB schema to include table '{}'", tableId);
             return tableSchema;
         }
     }
 
     private synchronized PgConnection typeResolverConnection() throws SQLException {
-        return (PgConnection) metadataConnection.connection();
+        return (PgConnection) connection.connection();
     }
 
     private Table tableFromFromMessage(List<ReplicationMessage.Column> columns, Table table) {
@@ -679,7 +571,7 @@ private Table tableFromFromMessage(List<ReplicationMessage.Column> columns, Tabl
         while (itPkCandidates.hasNext()) {
             final String candidateName = itPkCandidates.next();
             if (!combinedTable.hasUniqueValues() && combinedTable.columnWithName(candidateName) == null) {
-                logger.error("Potentional inconsistency in key for message {}", columns);
+                LOGGER.error("Potentional inconsistency in key for message {}", columns);
                 itPkCandidates.remove();
             }
         }
@@ -687,8 +579,12 @@ private Table tableFromFromMessage(List<ReplicationMessage.Column> columns, Tabl
         return combinedTable.create();
     }
 
-    // test-only
-    boolean isStreamingRunning() {
-        return replicationStream.get() != null;
+    private TopicSelector<TableId> topicSelector() {
+        return taskContext.topicSelector();
+    }
+
+    @FunctionalInterface
+    public static interface PgConnectionSupplier {
+        PgConnection get() throws SQLException;
     }
 }

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/RecordsSnapshotProducer.java

@@ -1,471 +0,0 @@
-/*
- * Copyright Debezium Authors.
- *
- * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
- */
-
-package io.debezium.connector.postgresql;
-
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.time.Duration;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
-
-import org.apache.kafka.connect.data.Schema;
-import org.apache.kafka.connect.data.Struct;
-import org.apache.kafka.connect.errors.ConnectException;
-import org.apache.kafka.connect.source.SourceRecord;
-import org.postgresql.util.PGmoney;
-
-import io.debezium.annotation.ThreadSafe;
-import io.debezium.config.ConfigurationDefaults;
-import io.debezium.connector.SnapshotRecord;
-import io.debezium.connector.postgresql.connection.PostgresConnection;
-import io.debezium.connector.postgresql.connection.ReplicationConnection;
-import io.debezium.connector.postgresql.snapshot.SnapshotterWrapper;
-import io.debezium.connector.postgresql.spi.SlotCreationResult;
-import io.debezium.connector.postgresql.spi.Snapshotter;
-import io.debezium.data.Envelope;
-import io.debezium.data.SpecialValueDecimal;
-import io.debezium.function.BlockingConsumer;
-import io.debezium.heartbeat.Heartbeat;
-import io.debezium.relational.Table;
-import io.debezium.relational.TableId;
-import io.debezium.relational.TableSchema;
-import io.debezium.util.Clock;
-import io.debezium.util.LoggingContext;
-import io.debezium.util.Metronome;
-import io.debezium.util.Strings;
-import io.debezium.util.Threads;
-
-/**
- * Producer of {@link org.apache.kafka.connect.source.SourceRecord source records} from a database snapshot. Once completed,
- * this producer can optionally continue streaming records, using another {@link RecordsStreamProducer} instance.
- *
- * @author Horia Chiorean (hchiorea@redhat.com)
- */
-@ThreadSafe
-public class RecordsSnapshotProducer extends RecordsProducer {
-
-    private static final String CONTEXT_NAME = "records-snapshot-producer";
-
-    private final ExecutorService executorService;
-    private final Optional<RecordsStreamProducer> streamProducer;
-
-    private final AtomicReference<SourceRecord> currentRecord;
-    private final Snapshotter snapshotter;
-    private final SlotCreationResult slotCreatedInfo;
-
-    public RecordsSnapshotProducer(PostgresTaskContext taskContext,
-                                   SourceInfo sourceInfo,
-                                   SnapshotterWrapper snapWrapper) {
-        super(taskContext, sourceInfo);
-        executorService = Threads.newSingleThreadExecutor(PostgresConnector.class, taskContext.config().getLogicalName(), CONTEXT_NAME);
-        currentRecord = new AtomicReference<>();
-        this.snapshotter = snapWrapper.getSnapshotter();
-        if (snapshotter.shouldStream()) {
-            boolean shouldExport = snapshotter.exportSnapshot();
-            ReplicationConnection replConn;
-            try {
-                replConn = taskContext.createReplicationConnection(shouldExport);
-                // we need to create the slot before we start streaming if it doesn't exist
-                // otherwise we can't stream back changes happening while the snapshot is taking place
-                if (!snapWrapper.doesSlotExist()) {
-                    slotCreatedInfo = replConn.createReplicationSlot().orElse(null);
-                }
-                else {
-                    slotCreatedInfo = null;
-                }
-            }
-            catch (SQLException ex) {
-                throw new ConnectException(ex);
-            }
-            streamProducer = Optional.of(new RecordsStreamProducer(taskContext, sourceInfo, replConn));
-        }
-        else {
-            streamProducer = Optional.empty();
-            slotCreatedInfo = null;
-        }
-    }
-
-    @Override
-    protected void start(BlockingConsumer<ChangeEvent> eventConsumer, Consumer<Throwable> failureConsumer) {
-        // MDC should be in inherited from parent to child threads
-        LoggingContext.PreviousContext previousContext = taskContext.configureLoggingContext(CONTEXT_NAME);
-        try {
-            CompletableFuture.runAsync(this::delaySnapshotIfNeeded, executorService)
-                             .thenRunAsync(() -> this.takeSnapshot(eventConsumer), executorService)
-                             .thenRunAsync(() -> this.startStreaming(eventConsumer, failureConsumer), executorService)
-                             .exceptionally(e -> {
-                                 logger.error("unexpected exception", e.getCause() != null ? e.getCause() : e);
-                                 // always stop to clean up data
-                                 stop();
-                                 failureConsumer.accept(e);
-
-                                 return null;
-                             });
-        } finally {
-            previousContext.restore();
-        }
-    }
-
-    private void delaySnapshotIfNeeded() {
-        Duration delay = taskContext.getConfig().getSnapshotDelay();
-        if (delay.isZero() || delay.isNegative()) {
-            return;
-        }
-
-        Threads.Timer timer = Threads.timer(Clock.SYSTEM, delay);
-        Metronome metronome = Metronome.parker(ConfigurationDefaults.RETURN_CONTROL_INTERVAL, Clock.SYSTEM);
-
-        while (!timer.expired()) {
-            try {
-                logger.info("The connector will wait for {}s before proceeding", timer.remaining().getSeconds());
-                metronome.pause();
-            }
-            catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-                logger.debug("Interrupted while awaiting initial snapshot delay");
-                return;
-            }
-        }
-    }
-
-    private void startStreaming(BlockingConsumer<ChangeEvent> consumer, Consumer<Throwable> failureConsumer) {
-        try {
-            // and then start streaming if necessary
-            streamProducer.ifPresent(producer -> {
-                if (sourceInfo.lsn() != null) {
-                    if (logger.isInfoEnabled()) {
-                        logger.info("Snapshot finished, continuing streaming changes from {}", ReplicationConnection.format(sourceInfo.lsn()));
-                    }
-                }
-
-                // still starting the stream producer, also if the connector has stopped already.
-                // otherwise the executor started in its constructor wouldn't be stopped. This logic
-                // will be obsolete when moving to the new framework classes.
-                producer.start(consumer, failureConsumer);
-            });
-        } finally {
-            // always cleanup our local data
-            cleanup();
-        }
-    }
-
-    @Override
-    protected void commit(long lsn)  {
-        streamProducer.ifPresent(x -> x.commit(lsn));
-    }
-
-    @Override
-    protected void stop() {
-        try {
-            streamProducer.ifPresent(RecordsStreamProducer::stop);
-        } finally {
-            cleanup();
-        }
-    }
-
-    private void cleanup() {
-        currentRecord.set(null);
-        executorService.shutdownNow();
-    }
-
-    private void takeSnapshot(BlockingConsumer<ChangeEvent> consumer) {
-        if (executorService.isShutdown()) {
-            logger.info("Not taking snapshot as this task has been cancelled already");
-            return;
-        }
-
-        long snapshotStart = clock().currentTimeInMillis();
-        Connection jdbcConnection = null;
-        try (PostgresConnection connection = taskContext.createConnection()) {
-            jdbcConnection = connection.connection();
-
-            logger.info("Step 0: disabling autocommit");
-            connection.setAutoCommit(false);
-
-            Duration lockTimeout = taskContext.config().snapshotLockTimeout();
-            logger.info("Step 1: starting transaction and refreshing the DB schemas for database '{}' and user '{}'",
-                        connection.database(), connection.username());
-
-            String transactionStatement = snapshotter.snapshotTransactionIsolationLevelStatement(slotCreatedInfo);
-            logger.info("opening transaction with statement {}", transactionStatement);
-            connection.executeWithoutCommitting(transactionStatement);
-
-            //next refresh the schema which will load all the tables taking the filters into account
-            PostgresSchema schema = schema();
-            schema.refresh(connection, false);
-
-            Optional<String> lockStatement = snapshotter.snapshotTableLockingStatement(lockTimeout, schema.tableIds());
-            if (lockStatement.isPresent()) {
-                logger.info("Step 2: locking each of the database tables, waiting a maximum of '{}' seconds for each lock",
-                        lockTimeout.getSeconds());
-                connection.executeWithoutCommitting(lockStatement.get());
-                //now that we have the locks, refresh the schema
-                schema.refresh(connection, false);
-            }
-            else {
-                // if we are not in an exported snapshot, this may result in some inconsistencies.
-                // Let the user know
-                if (!snapshotter.exportSnapshot()) {
-                    logger.warn("Step 2: skipping locking each table, this may result in inconsistent schema!");
-                }
-                else {
-                    logger.info("Step 2: skipping locking each table in an exported snapshot");
-                }
-            }
-
-            long xlogStart = getTransactionStartLsn(connection);
-            long txId = connection.currentTransactionId().longValue();
-            if (logger.isInfoEnabled()) {
-                logger.info("\t read xlogStart at '{}' from transaction '{}'", ReplicationConnection.format(xlogStart), txId);
-            }
-
-            // and mark the start of the snapshot
-            sourceInfo.startSnapshot();
-            // use the old xmin, as we don't want to update it if in xmin recovery
-            sourceInfo.update(xlogStart, clock().currentTime(), txId, null, sourceInfo.xmin());
-
-            logger.info("Step 3: reading and exporting the contents of each table");
-            AtomicInteger rowsCounter = new AtomicInteger(0);
-
-            for(TableId tableId : schema.tableIds()) {
-                long exportStart = clock().currentTimeInMillis();
-                logger.info("\t exporting data from table '{}'", tableId);
-                try {
-                    final Optional<String> selectStatement = snapshotter.buildSnapshotQuery(tableId);
-                    if (!selectStatement.isPresent()) {
-                        logger.warn("For table '{}' the select statement was not provided, skipping table", tableId);
-                    }
-                    else {
-                        logger.info("For table '{}' using select statement: '{}'", tableId, selectStatement);
-
-                        connection.queryWithBlockingConsumer(selectStatement.get(),
-                                this::readTableStatement,
-                                rs -> readTable(tableId, rs, consumer, rowsCounter));
-                        if (logger.isInfoEnabled()) {
-                            logger.info("\t finished exporting '{}' records for '{}'; total duration '{}'", rowsCounter.get(),
-                                        tableId, Strings.duration(clock().currentTimeInMillis() - exportStart));
-                        }
-                        rowsCounter.set(0);
-                    }
-                } catch (SQLException e) {
-                    throw new ConnectException(e);
-                }
-            }
-
-            // finally commit the transaction to release all the locks...
-            logger.info("Step 4: committing transaction '{}'", txId);
-            jdbcConnection.commit();
-
-            SourceRecord currentRecord = this.currentRecord.get();
-            if (currentRecord != null) {
-                // process and send the last record after marking it as such
-                logger.info("Step 5: sending the last snapshot record");
-                sourceInfo.markLastSnapshotRecord();
-
-                // the sourceInfo element already has been baked into the record value, so
-                // update the "last_snapshot_marker" in there
-                changeSourceToLastSnapshotRecord(currentRecord);
-                this.currentRecord.set(new SourceRecord(currentRecord.sourcePartition(), sourceInfo.offset(),
-                                                        currentRecord.topic(), currentRecord.kafkaPartition(),
-                                                        currentRecord.keySchema(), currentRecord.key(),
-                                                        currentRecord.valueSchema(), currentRecord.value()));
-
-                sendCurrentRecord(consumer);
-            }
-
-            // and complete the snapshot
-            sourceInfo.completeSnapshot();
-            if (logger.isInfoEnabled()) {
-                logger.info("Snapshot completed in '{}'", Strings.duration(clock().currentTimeInMillis() - snapshotStart));
-            }
-            Heartbeat
-                .create(
-                    taskContext.config().getConfig(),
-                    taskContext.topicSelector().getHeartbeatTopic(),
-                    taskContext.config().getLogicalName()
-                )
-                .forcedBeat(
-                    sourceInfo.partition(),
-                    sourceInfo.offset(),
-                    r -> consumer.accept(new ChangeEvent(r, sourceInfo.lsn())
-                )
-            );
-
-            taskContext.schema().assureNonEmptySchema();
-        }
-        catch (SQLException e) {
-            rollbackTransaction(jdbcConnection);
-
-            throw new ConnectException(e);
-        }
-        catch(InterruptedException e)  {
-            Thread.interrupted();
-            rollbackTransaction(jdbcConnection);
-
-            if (logger.isWarnEnabled()) {
-                logger.warn("Snapshot aborted after '{}'", Strings.duration(clock().currentTimeInMillis() - snapshotStart));
-            }
-        }
-    }
-
-    private long getTransactionStartLsn(PostgresConnection connection) throws SQLException {
-        if (snapshotter.exportSnapshot() && slotCreatedInfo != null) {
-            // When performing an exported snapshot based on a newly created replication slot, the txLogStart position
-            // should be based on the replication slot snapshot transaction point.  This is crucial so that if any
-            // SQL operations occur mid-snapshot that they'll be properly captured when streaming begins; otherwise
-            // they'll be lost.
-            return slotCreatedInfo.startLsn();
-        }
-        return connection.currentXLogLocation();
-    }
-
-    private void changeSourceToLastSnapshotRecord(SourceRecord currentRecord) {
-        final Struct envelope = (Struct) currentRecord.value();
-        final Struct source = (Struct) envelope.get(Envelope.FieldName.SOURCE);
-        if (source.schema().field(SourceInfo.LAST_SNAPSHOT_RECORD_KEY) != null && source.getBoolean(SourceInfo.LAST_SNAPSHOT_RECORD_KEY) != null) {
-            source.put(SourceInfo.LAST_SNAPSHOT_RECORD_KEY, true);
-        }
-        if (SnapshotRecord.fromSource(source) == SnapshotRecord.TRUE) {
-            SnapshotRecord.LAST.toSource(source);
-        }
-    }
-
-    private void rollbackTransaction(Connection jdbcConnection) {
-        try {
-            if (jdbcConnection != null) {
-                jdbcConnection.rollback();
-            }
-        }
-        catch (SQLException se) {
-            logger.error("Cannot rollback snapshot transaction", se);
-        }
-    }
-
-    private Statement readTableStatement(Connection conn) throws SQLException {
-        int fetchSize = taskContext.config().getSnapshotFetchSize();
-        Statement statement = conn.createStatement(); // the default cursor is FORWARD_ONLY
-        statement.setFetchSize(fetchSize);
-        return statement;
-    }
-
-    private void readTable(TableId tableId, ResultSet rs,
-                           BlockingConsumer<ChangeEvent> consumer,
-                           AtomicInteger rowsCounter) throws SQLException, InterruptedException {
-        Table table = schema().tableFor(tableId);
-        assert table != null;
-        final int numColumns = table.columns().size();
-        final Object[] row = new Object[numColumns];
-        final ResultSetMetaData metaData = rs.getMetaData();
-        while (rs.next()) {
-            rowsCounter.incrementAndGet();
-            sendCurrentRecord(consumer);
-            for (int i = 0, j = 1; i != numColumns; ++i, ++j) {
-                row[i] = valueForColumn(rs, j, metaData);
-            }
-            generateReadRecord(tableId, row);
-        }
-    }
-
-    private Object valueForColumn(ResultSet rs, int colIdx, ResultSetMetaData metaData) throws SQLException {
-        try {
-            final String columnTypeName = metaData.getColumnTypeName(colIdx);
-            final PostgresType type = taskContext.schema().getTypeRegistry().get(columnTypeName);
-
-            logger.trace("Type of incoming data is: {}", type.getOid());
-            logger.trace("ColumnTypeName is: {}", columnTypeName);
-            logger.trace("Type is: {}", type);
-
-            if (type.isArrayType()) {
-                return rs.getArray(colIdx);
-            }
-
-            switch (type.getOid()) {
-                case PgOid.MONEY:
-                    //TODO author=Horia Chiorean date=14/11/2016 description=workaround for https://github.com/pgjdbc/pgjdbc/issues/100
-                    return new PGmoney(rs.getString(colIdx)).val;
-                case PgOid.BIT:
-                    return rs.getString(colIdx);
-                case PgOid.NUMERIC:
-                    final String s = rs.getString(colIdx);
-                    if (s == null) {
-                        return s;
-                    }
-
-                    Optional<SpecialValueDecimal> value = PostgresValueConverter.toSpecialValue(s);
-                    return value.isPresent() ? value.get() : new SpecialValueDecimal(rs.getBigDecimal(colIdx));
-                case PgOid.TIME:
-                    // To handle time 24:00:00 supported by TIME columns, read the column as a string.
-                case PgOid.TIMETZ:
-                    // In order to guarantee that we resolve TIMETZ columns with proper microsecond precision,
-                    // read the column as a string instead and then re-parse inside the converter.
-                    return rs.getString(colIdx);
-                default:
-                    Object x = rs.getObject(colIdx);
-                    if(x != null) {
-                        logger.trace("rs getobject returns class: {}; rs getObject value is: {}", x.getClass(), x);
-                    }
-                    return x;
-            }
-        }
-        catch (SQLException e) {
-            // not a known type
-            return rs.getObject(colIdx);
-        }
-    }
-
-    protected void generateReadRecord(TableId tableId, Object[] rowData) {
-        // Clear the existing record to prevent reprocessing stale data.
-        currentRecord.set(null);
-
-        if (rowData.length == 0) {
-            return;
-        }
-        logger.trace("tableId value is: {}", tableId);
-        TableSchema tableSchema = schema().schemaFor(tableId);
-        assert tableSchema != null;
-        Object key = tableSchema.keyFromColumnData(rowData);
-        Struct value = tableSchema.valueFromColumnData(rowData);
-
-        if (value == null) {
-            logger.trace("Read event for null key with value {}", value);
-            return;
-        }
-        Schema keySchema = tableSchema.keySchema();
-        sourceInfo.update(clock().currentTimeAsInstant(), tableId);
-        Map<String, ?> partition = sourceInfo.partition();
-        Map<String, ?> offset = sourceInfo.offset();
-        String topicName = topicSelector().topicNameFor(tableId);
-        Envelope envelope = tableSchema.getEnvelopeSchema();
-        currentRecord.set(new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(),
-                                           envelope.read(value, sourceInfo.struct(), clock().currentTimeInMillis())));
-    }
-
-    private void sendCurrentRecord(BlockingConsumer<ChangeEvent> consumer) throws InterruptedException {
-        SourceRecord record = currentRecord.get();
-        if (record == null) {
-            return;
-        }
-        if (logger.isDebugEnabled()) {
-            logger.debug("sending read event '{}'", record);
-        }
-        //send the last generated record
-        consumer.accept(new ChangeEvent(record, sourceInfo.lsn()));
-    }
-
-    // test-only
-    boolean isStreamingRunning() {
-        return streamProducer.isPresent() && streamProducer.get().isStreamingRunning();
-    }
-}

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/SourceInfo.java

@@ -7,17 +7,12 @@
 package io.debezium.connector.postgresql;
 
 import java.time.Instant;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
 
 import io.debezium.annotation.NotThreadSafe;
-import io.debezium.connector.AbstractSourceInfo;
 import io.debezium.connector.SnapshotRecord;
+import io.debezium.connector.common.BaseSourceInfo;
 import io.debezium.connector.postgresql.connection.ReplicationConnection;
-import io.debezium.connector.postgresql.spi.OffsetState;
 import io.debezium.relational.TableId;
-import io.debezium.time.Conversions;
 
 /**
  * Information about the source of information, which for normal events contains information about the transaction id and the
@@ -74,9 +69,8 @@
  * @author Horia Chiorean
  */
 @NotThreadSafe
-public final class SourceInfo extends AbstractSourceInfo {
+public final class SourceInfo extends BaseSourceInfo {
 
-    public static final String SERVER_PARTITION_KEY = "server";
     public static final String TIMESTAMP_USEC_KEY = "ts_usec";
     public static final String TXID_KEY = "txId";
     public static final String XMIN_KEY = "xmin";
@@ -84,75 +78,17 @@ public final class SourceInfo extends AbstractSourceInfo {
     public static final String LAST_SNAPSHOT_RECORD_KEY = "last_snapshot_record";
 
     private final String dbName;
-    private final Map<String, String> sourcePartition;
 
     private Long lsn;
     private Long txId;
     private Long xmin;
     private Instant timestamp;
-    private boolean snapshot = false;
-    private Boolean lastSnapshotRecord;
     private String schemaName;
     private String tableName;
 
     protected SourceInfo(PostgresConnectorConfig connectorConfig) {
         super(connectorConfig);
         this.dbName = connectorConfig.databaseName();
-        this.sourcePartition = Collections.singletonMap(SERVER_PARTITION_KEY, connectorConfig.getLogicalName());
-    }
-
-    protected void load(Map<String, Object> lastStoredOffset) {
-        this.lsn = ((Number) lastStoredOffset.get(LSN_KEY)).longValue();
-        this.txId = ((Number) lastStoredOffset.get(TXID_KEY)).longValue();
-        this.xmin = (Long) lastStoredOffset.get(XMIN_KEY);
-        this.timestamp = Conversions.toInstantFromMicros((Long) lastStoredOffset.get(TIMESTAMP_USEC_KEY));
-        this.snapshot = lastStoredOffset.containsKey(SNAPSHOT_KEY);
-        if (this.snapshot) {
-            this.lastSnapshotRecord = (Boolean) lastStoredOffset.get(LAST_SNAPSHOT_RECORD_KEY);
-        }
-    }
-
-    /**
-     * Get the Kafka Connect detail about the source "partition", which describes the portion of the source that we are
-     * consuming. Since we're streaming changes for a single database, the source partition specifies only the {@code serverName}
-     * as the value for the partition.
-     *
-     * @return the source partition information; never null
-     */
-    public Map<String, String> partition() {
-        return sourcePartition;
-    }
-
-    /**
-     * Get the Kafka Connect detail about the source "offset", which describes the position within the source where we last
-     * have last read.
-     *
-     * @return a copy of the current offset; never null
-     */
-    public Map<String, ?> offset() {
-        assert serverName() != null && dbName != null;
-        Map<String, Object> result = new HashMap<>();
-        if (timestamp != null) {
-            result.put(TIMESTAMP_USEC_KEY, Conversions.toEpochMicros(timestamp));
-        }
-        if (txId != null) {
-            result.put(TXID_KEY, txId);
-        }
-        if (lsn != null) {
-            result.put(LSN_KEY, lsn);
-        }
-        if (xmin != null) {
-            result.put(XMIN_KEY, xmin);
-        }
-        if (snapshot) {
-            result.put(SNAPSHOT_KEY, true);
-            result.put(LAST_SNAPSHOT_RECORD_KEY, lastSnapshotRecord);
-        }
-        return result;
-    }
-
-    public OffsetState asOffsetState() {
-        return new OffsetState(lsn, txId, xmin, timestamp, isSnapshotInEffect());
     }
 
     /**
@@ -194,35 +130,6 @@ protected SourceInfo update(Instant timestamp, TableId tableId) {
         return this;
     }
 
-    protected SourceInfo markLastSnapshotRecord() {
-        this.lastSnapshotRecord = true;
-        return this;
-    }
-
-    /**
-     * Determine whether a snapshot is currently in effect, meaning it was started and has not completed.
-     *
-     * @return {@code true} if a snapshot is in effect, or {@code false} otherwise
-     */
-    public boolean isSnapshotInEffect() {
-        return snapshot && (this.lastSnapshotRecord == null || !this.lastSnapshotRecord);
-    }
-
-    /**
-     * Denote that a snapshot is being (or has been) started.
-     */
-    protected void startSnapshot() {
-        this.snapshot = true;
-        this.lastSnapshotRecord = false;
-    }
-
-    /**
-     * Denote that a snapshot has completed successfully.
-     */
-    protected void completeSnapshot() {
-        this.snapshot = false;
-    }
-
     public Long lsn() {
         return this.lsn;
     }
@@ -249,19 +156,13 @@ protected Instant timestamp() {
         return timestamp;
     }
 
-    @Override
-    protected SnapshotRecord snapshot() {
-        if (snapshot) {
-            if (lastSnapshotRecord) {
-                return SnapshotRecord.LAST;
-            }
-            return SnapshotRecord.TRUE;
-        }
-        return SnapshotRecord.FALSE;
+    protected Long txId() {
+        return txId;
     }
 
-    public boolean hasLastKnownPosition() {
-        return this.lsn != null;
+    @Override
+    public SnapshotRecord snapshot() {
+        return super.snapshot();
     }
 
     @Override
@@ -281,11 +182,7 @@ public String toString() {
         if (timestamp != null) {
             sb.append(", timestamp=").append(timestamp);
         }
-        boolean snapshotInEffect = isSnapshotInEffect();
-        sb.append(", snapshot=").append(snapshotInEffect);
-        if (snapshotInEffect) {
-            sb.append(", last_snapshot_record=").append(lastSnapshotRecord);
-        }
+        sb.append(", snapshot=").append(snapshot());
         if (schemaName != null) {
             sb.append(", schema=").append(schemaName);
         }

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/ToastedReplicationMessageColumn.java

@@ -28,7 +28,7 @@ public boolean isToastedColumn() {
     }
 
     @Override
-    public Object getValue(RecordsStreamProducer.PgConnectionSupplier connection, boolean includeUnknownDatatypes) {
+    public Object getValue(PostgresStreamingChangeEventSource.PgConnectionSupplier connection, boolean includeUnknownDatatypes) {
         throw new UnsupportedOperationException("A toasted column does not supply a value");
     }
 }

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/PostgresConnection.java

@@ -484,7 +484,7 @@ private static TypeRegistry initTypeRegistry(Connection db) {
             }
         }
         catch (SQLException e) {
-            throw new ConnectException("Could not intialize type registry", e);
+            throw new ConnectException("Database connection failed during intializiation of the type registry", e);
         }
         return typeRegistryBuilder.build();
     }

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/ReplicationMessage.java

@@ -22,8 +22,8 @@
 import org.postgresql.util.PGInterval;
 import org.postgresql.util.PGmoney;
 
+import io.debezium.connector.postgresql.PostgresStreamingChangeEventSource.PgConnectionSupplier;
 import io.debezium.connector.postgresql.PostgresType;
-import io.debezium.connector.postgresql.RecordsStreamProducer.PgConnectionSupplier;
 import io.debezium.data.SpecialValueDecimal;
 
 /**

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/ReplicationMessageColumnValueResolver.java

@@ -11,8 +11,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import io.debezium.connector.postgresql.PostgresStreamingChangeEventSource.PgConnectionSupplier;
 import io.debezium.connector.postgresql.PostgresType;
-import io.debezium.connector.postgresql.RecordsStreamProducer.PgConnectionSupplier;
 import io.debezium.connector.postgresql.connection.ReplicationMessage.ColumnValue;
 
 /**

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputMessageDecoder.java

@@ -25,8 +25,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import io.debezium.connector.postgresql.PostgresStreamingChangeEventSource.PgConnectionSupplier;
 import io.debezium.connector.postgresql.PostgresType;
-import io.debezium.connector.postgresql.RecordsStreamProducer.PgConnectionSupplier;
 import io.debezium.connector.postgresql.ToastedReplicationMessageColumn;
 import io.debezium.connector.postgresql.TypeRegistry;
 import io.debezium.connector.postgresql.connection.AbstractMessageDecoder;

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputReplicationMessage.java

@@ -8,8 +8,8 @@
 import java.time.Instant;
 import java.util.List;
 
+import io.debezium.connector.postgresql.PostgresStreamingChangeEventSource.PgConnectionSupplier;
 import io.debezium.connector.postgresql.PostgresType;
-import io.debezium.connector.postgresql.RecordsStreamProducer.PgConnectionSupplier;
 import io.debezium.connector.postgresql.connection.ReplicationMessage;
 import io.debezium.connector.postgresql.connection.ReplicationMessageColumnValueResolver;
 

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgproto/PgProtoMessageDecoder.java

@@ -11,6 +11,8 @@
 
 import org.apache.kafka.connect.errors.ConnectException;
 import org.postgresql.replication.fluent.logical.ChainedLogicalStreamBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.protobuf.InvalidProtocolBufferException;
 
@@ -29,6 +31,8 @@
  */
 public class PgProtoMessageDecoder extends AbstractMessageDecoder {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(PgProtoMessageDecoder.class);
+
     @Override
     public void processMessage(final ByteBuffer buffer, ReplicationMessageProcessor processor, TypeRegistry typeRegistry) throws SQLException, InterruptedException {
         try {
@@ -39,6 +43,7 @@ public void processMessage(final ByteBuffer buffer, ReplicationMessageProcessor
             final byte[] source = buffer.array();
             final byte[] content = Arrays.copyOfRange(source, buffer.arrayOffset(), source.length);
             final RowMessage message = PgProto.RowMessage.parseFrom(content);
+            LOGGER.trace("Received protobuf message from the server {}", message);
             if (!message.getNewTypeinfoList().isEmpty() && message.getNewTupleCount() != message.getNewTypeinfoCount()) {
                 throw new ConnectException(String.format("Message from transaction {} has {} data columns but only {} of type info",
                         Integer.toUnsignedLong(message.getTransactionId()),

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgproto/PgProtoReplicationMessage.java

@@ -26,9 +26,9 @@
 import org.slf4j.LoggerFactory;
 
 import io.debezium.connector.postgresql.PgOid;
+import io.debezium.connector.postgresql.PostgresStreamingChangeEventSource.PgConnectionSupplier;
 import io.debezium.connector.postgresql.PostgresType;
 import io.debezium.connector.postgresql.PostgresValueConverter;
-import io.debezium.connector.postgresql.RecordsStreamProducer.PgConnectionSupplier;
 import io.debezium.connector.postgresql.TypeRegistry;
 import io.debezium.connector.postgresql.connection.AbstractReplicationMessageColumn;
 import io.debezium.connector.postgresql.connection.ReplicationMessage;

debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/AbstractRecordsProducerTest.java

@@ -47,8 +47,6 @@
 import org.apache.kafka.connect.data.SchemaBuilder;
 import org.apache.kafka.connect.data.Struct;
 import org.apache.kafka.connect.source.SourceRecord;
-import org.awaitility.Awaitility;
-import org.awaitility.Duration;
 import org.junit.Rule;
 import org.junit.rules.TestRule;
 import org.slf4j.Logger;
@@ -65,7 +63,7 @@
 import io.debezium.data.geometry.Geography;
 import io.debezium.data.geometry.Geometry;
 import io.debezium.data.geometry.Point;
-import io.debezium.function.BlockingConsumer;
+import io.debezium.embedded.AbstractConnectorTest;
 import io.debezium.junit.TestLogger;
 import io.debezium.relational.TableId;
 import io.debezium.time.Date;
@@ -76,14 +74,16 @@
 import io.debezium.time.Timestamp;
 import io.debezium.time.ZonedTime;
 import io.debezium.time.ZonedTimestamp;
-import io.debezium.util.VariableLatch;
+import io.debezium.util.Clock;
+import io.debezium.util.ElapsedTimeStrategy;
+import io.debezium.util.Testing;
 
 /**
  * Base class for the integration tests for the different {@link RecordsProducer} instances
  *
  * @author Horia Chiorean (hchiorea@redhat.com)
  */
-public abstract class AbstractRecordsProducerTest {
+public abstract class AbstractRecordsProducerTest extends AbstractConnectorTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(AbstractRecordsProducerTest.class);
 
@@ -975,14 +975,14 @@ protected TestConsumer testConsumer(int expectedRecordsCount, String... topicPre
          return new TestConsumer(expectedRecordsCount, topicPrefixes);
     }
 
-    protected static class TestConsumer implements BlockingConsumer<ChangeEvent> {
+    protected class TestConsumer {
         private final ConcurrentLinkedQueue<SourceRecord> records;
-        private final VariableLatch latch;
+        private int expectedRecordsCount;
         private final List<String> topicPrefixes;
         private boolean ignoreExtraRecords = false;
 
         protected TestConsumer(int expectedRecordsCount, String... topicPrefixes) {
-            this.latch = new VariableLatch(expectedRecordsCount);
+            this.expectedRecordsCount = expectedRecordsCount;
             this.records = new ConcurrentLinkedQueue<>();
             this.topicPrefixes = Arrays.stream(topicPrefixes)
                     .map(p -> TestHelper.TEST_SERVER + "." + p)
@@ -993,22 +993,33 @@ public void setIgnoreExtraRecords(boolean ignoreExtraRecords) {
             this.ignoreExtraRecords = ignoreExtraRecords;
         }
 
-        @Override
-        public void accept(ChangeEvent event) {
-            final SourceRecord record = event.getRecord();
-            if ( ignoreTopic(record.topic()) ) {
+        public void accept(SourceRecord record) {
+            if (ignoreTopic(record.topic())) {
                 return;
             }
 
-            if (latch.getCount() == 0) {
-                if (ignoreExtraRecords) {
-                    records.add(record);
-                } else {
+            if (records.size() >= expectedRecordsCount) {
+                addRecord(record);
+                if (!ignoreExtraRecords) {
                     fail("received more events than expected");
                 }
-            } else {
-                records.add(record);
-                latch.countDown();
+            }
+            else {
+                addRecord(record);
+            }
+        }
+
+        private void addRecord(SourceRecord record) {
+            records.add(record);
+            if (Testing.Debug.isEnabled()) {
+                Testing.debug("Consumed record " + records.size() + " / " + expectedRecordsCount + " ("
+                        + (expectedRecordsCount - records.size()) + " more)");
+                Testing.debug(record);
+            }
+            else if (Testing.Print.isEnabled()) {
+                Testing.print("Consumed record " + records.size() + " / " + expectedRecordsCount + " ("
+                        + (expectedRecordsCount - records.size()) + " more)");
+                Testing.print(record);
             }
         }
 
@@ -1027,8 +1038,7 @@ private boolean ignoreTopic(String topicName) {
         }
 
         protected void expects(int expectedRecordsCount) {
-            assert latch.getCount() == 0;
-            this.latch.countUp(expectedRecordsCount);
+            this.expectedRecordsCount = expectedRecordsCount;
         }
 
         protected SourceRecord remove() {
@@ -1048,13 +1058,28 @@ protected void clear() {
         }
 
         protected void await(long timeout, TimeUnit unit) throws InterruptedException {
-            if (!latch.await(timeout, unit)) {
-                fail("Consumer is still expecting " + latch.getCount() + " records, as it received only " + records.size());
+            final ElapsedTimeStrategy timer = ElapsedTimeStrategy.constant(Clock.SYSTEM, unit.toMillis(timeout));
+            timer.hasElapsed();
+            while (!timer.hasElapsed()) {
+                final SourceRecord r = consumeRecord();
+                if (r != null) {
+                    accept(r);
+                    if (records.size() == expectedRecordsCount) {
+                        break;
+                    }
+                }
+            }
+            if (records.size() != expectedRecordsCount) {
+                fail("Consumer is still expecting " + (expectedRecordsCount -records.size()) + " records, as it received only " + records.size());
             }
         }
     }
 
-    protected void waitForStreamingToStart(RecordsSnapshotProducer producer) throws InterruptedException {
-        Awaitility.await().atMost(Duration.FIVE_SECONDS).until(producer::isStreamingRunning);
+    protected void waitForStreamingToStart() throws InterruptedException {
+        waitForStreamingRunning("postgres", "test_server");
+    }
+
+    protected PostgresSchema getSchema() {
+        return null;
     }
 }

debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/OutboxEventRouterIT.java

@@ -469,10 +469,10 @@ private void startConnectorWithInitialSnapshotRecord() throws Exception {
         assertThat(recordsFromOutbox.size()).isEqualTo(1);
     }
 
-    private void startConnectorWithNoSnapshot() {
+    private void startConnectorWithNoSnapshot() throws InterruptedException {
         Configuration.Builder configBuilder = getConfigurationBuilder(SnapshotMode.NEVER);
         start(PostgresConnector.class, configBuilder.build());
-        assertConnectorIsRunning();
+        waitForStreamingRunning("postgres", TestHelper.TEST_SERVER);
     }
 
     private static Configuration.Builder getConfigurationBuilder(SnapshotMode snapshotMode) {

debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/PostgresConnectorIT.java

@@ -319,6 +319,7 @@ public void shouldConsumeMessagesFromSnapshot() throws Exception {
     @Test
     @FixFor("DBZ-997")
     public void shouldReceiveChangesForChangePKColumnDefinition() throws Exception {
+        Testing.Print.enable();
         final String slotName = "pkcolumndef" + new Random().nextInt(100);
         TestHelper.create().dropReplicationSlot(slotName);
         try {
@@ -410,6 +411,7 @@ public void shouldReceiveChangesForChangePKColumnDefinition() throws Exception {
             // where there are limited logical replication slots configured.
             stopConnector(null);
             TestHelper.create().dropReplicationSlot(slotName);
+            throw t;
         }
     }
 
@@ -536,14 +538,14 @@ public void shouldExecuteOnConnectStatements() throws Exception {
         start(PostgresConnector.class, configBuilder.build());
         assertConnectorIsRunning();
 
-        SourceRecords actualRecords = consumeRecordsByTopic(7);
+        SourceRecords actualRecords = consumeRecordsByTopic(6);
         assertKey(actualRecords.allRecordsInOrder().get(0), "pk", 1);
         assertKey(actualRecords.allRecordsInOrder().get(1), "pk", 2);
 
         // JdbcConnection#connection() is called multiple times during connector start-up,
         // so the given statements will be executed multiple times, resulting in multiple
         // records; here we're interested just in the first insert for s2.a
-        assertValueField(actualRecords.allRecordsInOrder().get(6), "after/bb", "hello; world");
+        assertValueField(actualRecords.allRecordsInOrder().get(5), "after/bb", "hello; world");
 
         stopConnector();
         TestHelper.dropDefaultReplicationSlot();

debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/PublicGeometryIT.java

@@ -13,15 +13,15 @@
 
 import java.util.List;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
 
 import org.apache.kafka.connect.source.SourceRecord;
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestRule;
 
+import io.debezium.config.Configuration;
+import io.debezium.connector.postgresql.PostgresConnectorConfig.SnapshotMode;
 import io.debezium.connector.postgresql.connection.PostgresConnection;
 import io.debezium.connector.postgresql.connection.ReplicationConnection;
 import io.debezium.connector.postgresql.junit.SkipTestDependingOnDecoderPluginNameRule;
@@ -30,7 +30,6 @@
 import io.debezium.doc.FixFor;
 import io.debezium.junit.ConditionalFail;
 import io.debezium.relational.TableId;
-import io.debezium.schema.TopicSelector;
 
 /**
  * Integration test to verify PostGIS types defined in public schema.
@@ -39,9 +38,7 @@
  */
 public class PublicGeometryIT extends AbstractRecordsProducerTest {
 
-    private RecordsStreamProducer recordsProducer;
     private TestConsumer consumer;
-    private final Consumer<Throwable> blackHole = t -> {};
 
     @Rule
     public final TestRule skip = new SkipTestDependingOnDecoderPluginNameRule();
@@ -63,26 +60,14 @@ public void before() throws Exception {
             "CREATE TABLE public.postgis_array_table (pk SERIAL, ga GEOMETRY[], gann GEOMETRY[] NOT NULL, PRIMARY KEY(pk));",
             "CREATE TABLE public.dummy_table (pk SERIAL, PRIMARY KEY(pk));"
         );
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, false)
-                .build());
-        setupRecordsProducer(config);
-    }
-
-    @After
-    public void after() throws Exception {
-        if (recordsProducer != null) {
-            recordsProducer.stop();
-        }
+        setupRecordsProducer(TestHelper.defaultConfig());
     }
 
     @Test(timeout = 30000)
     @FixFor("DBZ-1144")
     public void shouldReceiveChangesForInsertsWithPostgisTypes() throws Exception {
-        consumer = testConsumer(1, "public"); // spatial_ref_sys produces a ton of records in the postgis schema
-        consumer.setIgnoreExtraRecords(true);
-        recordsProducer.start(consumer, blackHole);
-
+        consumer = testConsumer(1, "public");
+        waitForStreamingToStart();
         // need to wait for all the spatial_ref_sys to flow through and be ignored.
         // this exceeds the normal 2s timeout.
         TestHelper.execute("INSERT INTO public.dummy_table DEFAULT VALUES;");
@@ -104,19 +89,12 @@ public void shouldReceiveChangesForInsertsWithPostgisTypes() throws Exception {
         assertInsert(INSERT_POSTGIS_ARRAY_TYPES_IN_PUBLIC_STMT, 1, schemaAndValuesForPostgisArrayTypes());
     }
 
-    private void setupRecordsProducer(PostgresConnectorConfig config) {
-        if (recordsProducer != null) {
-            recordsProducer.stop();
-        }
-
-        TopicSelector<TableId> selector = PostgresTopicSelector.create(config);
-
-        PostgresTaskContext context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                selector
+    private void setupRecordsProducer(Configuration.Builder config) {
+        start(PostgresConnector.class, config
+                .with(PostgresConnectorConfig.SNAPSHOT_MODE, SnapshotMode.NEVER)
+                .build()
         );
-        recordsProducer = new RecordsStreamProducer(context, new SourceInfo(config));
+        assertConnectorIsRunning();
     }
 
     private void assertInsert(String statement, Integer pk, List<SchemaAndValueField> expectedSchemaAndValuesByColumn) {

debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/RecordsSnapshotProducerIT.java

@@ -26,27 +26,23 @@
 import org.apache.kafka.connect.data.Struct;
 import org.apache.kafka.connect.source.SourceRecord;
 import org.fest.assertions.Assertions;
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestRule;
 
+import io.debezium.config.Configuration;
+import io.debezium.connector.postgresql.PostgresConnectorConfig.SnapshotMode;
 import io.debezium.connector.postgresql.junit.SkipTestDependingOnDatabaseVersionRule;
 import io.debezium.connector.postgresql.junit.SkipWhenDatabaseVersionLessThan;
-import io.debezium.connector.postgresql.snapshot.AlwaysSnapshotter;
-import io.debezium.connector.postgresql.snapshot.InitialOnlySnapshotter;
-import io.debezium.connector.postgresql.snapshot.SnapshotterWrapper;
-import io.debezium.connector.postgresql.spi.Snapshotter;
 import io.debezium.data.Envelope;
 import io.debezium.data.VerifyRecord;
 import io.debezium.doc.FixFor;
 import io.debezium.heartbeat.Heartbeat;
 import io.debezium.jdbc.TemporalPrecisionMode;
 import io.debezium.relational.RelationalDatabaseConnectorConfig.DecimalHandlingMode;
-import io.debezium.relational.TableId;
-import io.debezium.schema.TopicSelector;
 import io.debezium.util.Collect;
+import io.debezium.util.Testing;
 
 /**
  * Integration test for {@link RecordsSnapshotProducerIT}
@@ -58,53 +54,29 @@ public class RecordsSnapshotProducerIT extends AbstractRecordsProducerTest {
     @Rule
     public final TestRule skip = new SkipTestDependingOnDatabaseVersionRule();
 
-    private RecordsSnapshotProducer snapshotProducer;
-    private PostgresTaskContext context;
-
     @Before
     public void before() throws Exception {
         TestHelper.dropAllSchemas();
         TestHelper.executeDDL("init_postgis.ddl");
         TestHelper.executeDDL("postgres_create_tables.ddl");
         TestHelper.executeDDL("postgis_create_tables.ddl");
-
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .build());
-        TopicSelector<TableId> selector = PostgresTopicSelector.create(config);
-        context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                selector
-        );
-        // Testing.Print.enable();
-    }
-
-    @After
-    public void after() throws Exception {
-        if (snapshotProducer != null) {
-            snapshotProducer.stop();
-        }
     }
 
     @Test
     public void shouldGenerateSnapshotsForDefaultDatatypes() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .build());
-        snapshotProducer = buildNoStreamProducer(context, config);
-
-        TestConsumer consumer = testConsumer(ALL_STMTS.size(), "public", "Quoted__");
-
         //insert data for each of different supported types
         String statementsBuilder = ALL_STMTS.stream().collect(Collectors.joining(";" + System.lineSeparator())) + ";";
         TestHelper.execute(statementsBuilder);
 
         //then start the producer and validate all records are there
-        snapshotProducer.start(consumer, e -> {});
+        buildNoStreamProducer(TestHelper.defaultConfig());
+
+        TestConsumer consumer = testConsumer(ALL_STMTS.size(), "public", "Quoted__");
         consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         Map<String, List<SchemaAndValueField>> expectedValuesByTopicName = super.schemaAndValuesByTopicName();
         consumer.process(record -> assertReadRecord(record, expectedValuesByTopicName));
-
+        Testing.Print.enable();
         // check the offset information for each record
         while (!consumer.isEmpty()) {
             SourceRecord record = consumer.remove();
@@ -115,25 +87,14 @@ public void shouldGenerateSnapshotsForDefaultDatatypes() throws Exception {
 
     @Test
     public void shouldGenerateSnapshotsForCustomDatatypes() throws Exception {
-        final PostgresConnectorConfig config = new PostgresConnectorConfig(
-                TestHelper.defaultConfig()
-                    .with(PostgresConnectorConfig.SNAPSHOT_MODE, PostgresConnectorConfig.SnapshotMode.INITIAL)
-                    .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
-                    .build()
-        );
-        context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                PostgresTopicSelector.create(config)
-        );
-        snapshotProducer = buildNoStreamProducer(context, config);
-
-        final TestConsumer consumer = testConsumer(1, "public");
-
         TestHelper.execute(INSERT_CUSTOM_TYPES_STMT);
 
         //then start the producer and validate all records are there
-        snapshotProducer.start(consumer, e -> {});
+        buildNoStreamProducer(TestHelper.defaultConfig()
+                .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
+        );
+
+        final TestConsumer consumer = testConsumer(1, "public");
         consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         final Map<String, List<SchemaAndValueField>> expectedValuesByTopicName = Collect.hashMapOf("public.custom_table", schemasAndValuesForCustomTypes());
@@ -146,14 +107,6 @@ public void shouldGenerateSnapshotAndContinueStreaming() throws Exception {
         TestHelper.dropAllSchemas();
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig().build());
-        TopicSelector<TableId> selector = PostgresTopicSelector.create(config);
-        context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                selector
-        );
-
         String insertStmt = "INSERT INTO s1.a (aa) VALUES (1);" +
                             "INSERT INTO s2.a (aa) VALUES (1);";
 
@@ -164,42 +117,43 @@ public void shouldGenerateSnapshotAndContinueStreaming() throws Exception {
                             insertStmt;
         TestHelper.execute(statements);
 
-        snapshotProducer = buildWithStreamProducer(context, config);
+        buildWithStreamProducer(TestHelper.defaultConfig());
+
         TestConsumer consumer = testConsumer(2, "s1", "s2");
-        snapshotProducer.start(consumer, e -> {});
 
         // first make sure we get the initial records from both schemas...
         consumer.await(TestHelper.waitTimeForRecords(), TimeUnit.SECONDS);
         consumer.clear();
 
         // then insert some more data and check that we get it back
-        waitForStreamingToStart(snapshotProducer);
+        waitForStreamingToStart();
         TestHelper.execute(insertStmt);
         consumer.expects(2);
-        consumer.await(TestHelper.waitTimeForRecords(), TimeUnit.SECONDS);
+        consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         SourceRecord first = consumer.remove();
         VerifyRecord.isValidInsert(first, PK_FIELD, 2);
         assertEquals(topicName("s1.a"), first.topic());
         assertRecordOffsetAndSnapshotSource(first, false, false);
-        assertSourceInfo(first, "test_database", "s1", "a");
+        assertSourceInfo(first, TestHelper.TEST_DATABASE, "s1", "a");
 
         SourceRecord second = consumer.remove();
         VerifyRecord.isValidInsert(second, PK_FIELD, 2);
         assertEquals(topicName("s2.a"), second.topic());
         assertRecordOffsetAndSnapshotSource(second, false, false);
-        assertSourceInfo(second, "test_database", "s2", "a");
+        assertSourceInfo(second, TestHelper.TEST_DATABASE, "s2", "a");
 
         // now shut down the producers and insert some more records
-        snapshotProducer.stop();
+        stopConnector();
+        assertConnectorNotRunning();
         TestHelper.execute(insertStmt);
 
         // start a new producer back up, take a new snapshot (we expect all the records to be read back)
         int expectedRecordsCount = 6;
+        buildWithStreamProducer(TestHelper.defaultConfig());
+
         consumer = testConsumer(expectedRecordsCount, "s1", "s2");
-        snapshotProducer = buildWithStreamProducer(context, config);
-        snapshotProducer.start(consumer, e -> {});
-        consumer.await(TestHelper.waitTimeForRecords(), TimeUnit.SECONDS);
+        consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         AtomicInteger counter = new AtomicInteger(0);
         consumer.process(record -> {
@@ -212,7 +166,7 @@ public void shouldGenerateSnapshotAndContinueStreaming() throws Exception {
         consumer.clear();
 
         // now insert two more records and check that we only get those back from the stream
-        waitForStreamingToStart(snapshotProducer);
+        waitForStreamingToStart();
         TestHelper.execute(insertStmt);
         consumer.expects(2);
 
@@ -220,12 +174,12 @@ public void shouldGenerateSnapshotAndContinueStreaming() throws Exception {
         first = consumer.remove();
         VerifyRecord.isValidInsert(first, PK_FIELD, 4);
         assertRecordOffsetAndSnapshotSource(first, false, false);
-        assertSourceInfo(first, "test_database", "s1", "a");
+        assertSourceInfo(first, TestHelper.TEST_DATABASE, "s1", "a");
 
         second = consumer.remove();
         VerifyRecord.isValidInsert(second, PK_FIELD, 4);
         assertRecordOffsetAndSnapshotSource(second, false, false);
-        assertSourceInfo(second, "test_database", "s2", "a");
+        assertSourceInfo(second, TestHelper.TEST_DATABASE, "s2", "a");
     }
 
     @Test
@@ -235,23 +189,13 @@ public void shouldGenerateSnapshotAndSendHeartBeat() throws Exception {
         TestHelper.dropAllSchemas();
         TestHelper.execute("CREATE TABLE t1 (pk SERIAL, aa integer, PRIMARY KEY(pk)); INSERT INTO t1 VALUES (default, 11)");
 
-        PostgresConnectorConfig config = new PostgresConnectorConfig(
-                TestHelper.defaultConfig()
-                    .with(PostgresConnectorConfig.SNAPSHOT_MODE, PostgresConnectorConfig.SnapshotMode.INITIAL)
-                    .with(PostgresConnectorConfig.INCLUDE_SCHEMA_CHANGES, true)
-                    .with(Heartbeat.HEARTBEAT_INTERVAL, 300_000)
-                    .build()
-        );
-        TopicSelector<TableId> selector = PostgresTopicSelector.create(config);
-        context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                selector
+        buildWithStreamProducer(TestHelper.defaultConfig()
+            .with(PostgresConnectorConfig.SNAPSHOT_MODE, PostgresConnectorConfig.SnapshotMode.INITIAL)
+            .with(PostgresConnectorConfig.INCLUDE_SCHEMA_CHANGES, true)
+            .with(Heartbeat.HEARTBEAT_INTERVAL, 300_000)
         );
 
-        snapshotProducer = buildWithStreamProducer(context, config);
         TestConsumer consumer = testConsumer(2);
-        snapshotProducer.start(consumer, e -> {});
 
         // Make sure we get the table schema record and heartbeat record
         consumer.await(TestHelper.waitTimeForRecords(), TimeUnit.SECONDS);
@@ -262,21 +206,6 @@ public void shouldGenerateSnapshotAndSendHeartBeat() throws Exception {
         final SourceRecord second = consumer.remove();
         assertThat(second.topic()).startsWith("__debezium-heartbeat");
         assertRecordOffsetAndSnapshotSource(second, false, false);
-
-        // now shut down the producers and insert some more records
-        snapshotProducer.stop();
-    }
-
-    private RecordsSnapshotProducer buildNoStreamProducer(PostgresTaskContext ctx, PostgresConnectorConfig config) {
-        Snapshotter sn = new InitialOnlySnapshotter();
-        SnapshotterWrapper snw = new SnapshotterWrapper(sn, config, null, null);
-        return new RecordsSnapshotProducer(ctx, TestHelper.sourceInfo(), snw);
-    }
-
-    private RecordsSnapshotProducer buildWithStreamProducer(PostgresTaskContext ctx, PostgresConnectorConfig config) {
-        Snapshotter sn = new AlwaysSnapshotter();
-        SnapshotterWrapper snw = new SnapshotterWrapper(sn, config, null, null);
-        return new RecordsSnapshotProducer(ctx, TestHelper.sourceInfo(), snw);
     }
 
     private void assertReadRecord(SourceRecord record, Map<String, List<SchemaAndValueField>> expectedValuesByTopicName) {
@@ -290,28 +219,16 @@ private void assertReadRecord(SourceRecord record, Map<String, List<SchemaAndVal
     @Test
     @FixFor("DBZ-342")
     public void shouldGenerateSnapshotsForDefaultDatatypesAdaptiveMicroseconds() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(
-                TestHelper.defaultConfig()
-                        .with(PostgresConnectorConfig.TIME_PRECISION_MODE, TemporalPrecisionMode.ADAPTIVE_TIME_MICROSECONDS)
-                        .build());
-
-        TopicSelector<TableId> selector = PostgresTopicSelector.create(config);
-        context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                selector
-        );
-
-        snapshotProducer = buildNoStreamProducer(context, config);
-
-        TestConsumer consumer = testConsumer(ALL_STMTS.size(), "public", "Quoted__");
-
         //insert data for each of different supported types
         String statementsBuilder = ALL_STMTS.stream().collect(Collectors.joining(";" + System.lineSeparator())) + ";";
         TestHelper.execute(statementsBuilder);
 
         //then start the producer and validate all records are there
-        snapshotProducer.start(consumer, e -> {});
+        buildNoStreamProducer(TestHelper.defaultConfig()
+                .with(PostgresConnectorConfig.TIME_PRECISION_MODE, TemporalPrecisionMode.ADAPTIVE_TIME_MICROSECONDS)
+        );
+
+        TestConsumer consumer = testConsumer(ALL_STMTS.size(), "public", "Quoted__");
         consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         Map<String, List<SchemaAndValueField>> expectedValuesByTopicName = super.schemaAndValuesByTopicNameAdaptiveTimeMicroseconds();
@@ -332,27 +249,15 @@ public void shouldGenerateSnapshotsForDecimalDatatypesUsingStringEncoding() thro
         TestHelper.dropAllSchemas();
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        PostgresConnectorConfig config = new PostgresConnectorConfig(
-                TestHelper.defaultConfig()
-                        .with(PostgresConnectorConfig.DECIMAL_HANDLING_MODE, DecimalHandlingMode.STRING)
-                        .build());
-
-        TopicSelector<TableId> selector = PostgresTopicSelector.create(config);
-        context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                selector
-        );
-
-        snapshotProducer = buildNoStreamProducer(context, config);
-
-        TestConsumer consumer = testConsumer(1, "public", "Quoted_\"");
-
         //insert data for each of different supported types
         TestHelper.execute(INSERT_NUMERIC_DECIMAL_TYPES_STMT);
 
         //then start the producer and validate all records are there
-        snapshotProducer.start(consumer, e -> {});
+        buildNoStreamProducer(TestHelper.defaultConfig()
+                .with(PostgresConnectorConfig.DECIMAL_HANDLING_MODE, DecimalHandlingMode.STRING)
+        );
+
+        TestConsumer consumer = testConsumer(1, "public", "Quoted__");
         consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         Map<String, List<SchemaAndValueField>> expectedValuesByTopicName = super.schemaAndValuesByTopicNameStringEncodedDecimals();
@@ -386,21 +291,6 @@ public void shouldGenerateSnapshotsForPartitionedTables() throws Exception {
 
         TestHelper.execute(ddl);
 
-        PostgresConnectorConfig config = new PostgresConnectorConfig(
-                TestHelper.defaultConfig()
-                        .build());
-
-        TopicSelector<TableId> selector = PostgresTopicSelector.create(config);
-        context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                selector
-        );
-
-        snapshotProducer = buildNoStreamProducer(context, config);
-
-        TestConsumer consumer = testConsumer(1 + 2 * 30); // Every record comes once from partitioned table and from partition
-
         // add 1 record to `first_table`. To reproduce the bug we must process at
         // least one row before processing the partitioned table.
         TestHelper.execute("INSERT INTO first_table (pk, user_id) VALUES (1000, 1);");
@@ -414,7 +304,9 @@ public void shouldGenerateSnapshotsForPartitionedTables() throws Exception {
                 "FROM generate_series(1, 20);");
 
         // then start the producer and validate all records are there
-        snapshotProducer.start(consumer, e -> {});
+        buildNoStreamProducer(TestHelper.defaultConfig());
+
+        TestConsumer consumer = testConsumer(1 + 2 * 30); // Every record comes once from partitioned table and from partition
         consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         Set<Integer> ids = new HashSet<>();
@@ -459,27 +351,13 @@ public void shouldGenerateSnapshotsForHstores() throws Exception {
         TestHelper.dropAllSchemas();
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        PostgresConnectorConfig config = new PostgresConnectorConfig(
-                TestHelper.defaultConfig()
-                        .with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.JSON)
-                        .build());
-
-        TopicSelector<TableId> selector = PostgresTopicSelector.create(config);
-        context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                selector
-        );
-
-        snapshotProducer = buildNoStreamProducer(context, config);
-
-        TestConsumer consumer = testConsumer(1, "public", "Quoted_\"");
-
         //insert data for each of different supported types
         TestHelper.execute(INSERT_HSTORE_TYPE_STMT);
 
         //then start the producer and validate all records are there
-        snapshotProducer.start(consumer, e -> {});
+        buildNoStreamProducer(TestHelper.defaultConfig());
+
+        TestConsumer consumer = testConsumer(1, "public", "Quoted__");
         consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
 
@@ -491,24 +369,12 @@ public void shouldGenerateSnapshotsForHstores() throws Exception {
     @Test
     @FixFor("DBZ-1163")
     public void shouldGenerateSnapshotForATableWithoutPrimaryKey() throws Exception {
-        final PostgresConnectorConfig config = new PostgresConnectorConfig(
-                TestHelper.defaultConfig()
-                        .with(PostgresConnectorConfig.SNAPSHOT_MODE, PostgresConnectorConfig.SnapshotMode.INITIAL)
-                        .build()
-        );
-        context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                PostgresTopicSelector.create(config)
-        );
-        snapshotProducer = buildNoStreamProducer(context, config);
-
-        final TestConsumer consumer = testConsumer(1, "public");
-
         TestHelper.execute("insert into table_without_pk values(1, 1000)");
 
         //then start the producer and validate all records are there
-        snapshotProducer.start(consumer, e -> {});
+        buildNoStreamProducer(TestHelper.defaultConfig());
+
+        TestConsumer consumer = testConsumer(1, "public", "Quoted__");
         consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         List<SchemaAndValueField> schemaAndValueFields = Arrays.asList(
@@ -531,17 +397,13 @@ public void shouldGenerateSnapshotForMacaddr8Datatype() throws Exception {
         TestHelper.dropAllSchemas();
         TestHelper.execute("CREATE TABLE macaddr8_table(pk SERIAL, m MACADDR8, PRIMARY KEY(pk));");
 
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .build());
-        snapshotProducer = buildNoStreamProducer(context, config);
-
-        final TestConsumer consumer = testConsumer(1, "public");
-
         // insert macaddr8 data
         TestHelper.execute(INSERT_MACADDR8_TYPE_STMT);
 
         // then start the producer and validate the record are there
-        snapshotProducer.start(consumer, e -> {});
+        buildNoStreamProducer(TestHelper.defaultConfig());
+
+        TestConsumer consumer = testConsumer(1, "public");
         consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         final Map<String, List<SchemaAndValueField>> expectedValueByTopicName = Collect.hashMapOf("public.macaddr8_table", schemaAndValueForMacaddr8Type());
@@ -555,16 +417,12 @@ public void shouldGenerateSnapshotForTwentyFourHourTime() throws Exception {
         TestHelper.dropAllSchemas();
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .build());
-        snapshotProducer = buildNoStreamProducer(context, config);
-
-        final TestConsumer consumer = testConsumer(1, "public");
-
         // insert data and time data
         TestHelper.execute(INSERT_DATE_TIME_TYPES_STMT);
 
-        snapshotProducer.start(consumer, e -> {});
+        buildNoStreamProducer(TestHelper.defaultConfig());
+
+        TestConsumer consumer = testConsumer(1, "public");
         consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         final Map<String, List<SchemaAndValueField>> expectedValueByTopicName = Collect.hashMapOf("public.time_table", schemaAndValuesForDateTimeTypes());
@@ -579,19 +437,36 @@ public void shouldNotSnapshotMaterializedViews() throws Exception {
         TestHelper.execute("CREATE TABLE mv_real_table (pk SERIAL, i integer, s VARCHAR(50), PRIMARY KEY(pk));");
         TestHelper.execute("CREATE MATERIALIZED VIEW mv (pk, s) AS SELECT mrv.pk, mrv.s FROM mv_real_table mrv WITH DATA;");
 
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig().build());
-        snapshotProducer = buildNoStreamProducer(context, config);
-
-        final TestConsumer consumer = testConsumer(1, "public");
-
         // insert data
         TestHelper.execute("INSERT INTO mv_real_table (i,s) VALUES (1,'1');");
         TestHelper.execute("REFRESH MATERIALIZED VIEW mv WITH DATA;");
 
-        snapshotProducer.start(consumer, e -> {});
+        buildNoStreamProducer(TestHelper.defaultConfig());
+
+        TestConsumer consumer = testConsumer(1, "public");
         consumer.await(TestHelper.waitTimeForRecords() * 30, TimeUnit.SECONDS);
 
         final Map<String, List<SchemaAndValueField>> expectedValueByTopicName = Collect.hashMapOf("public.mv_real_table", schemaAndValueForMaterializedViewBaseType());
         consumer.process(record ->assertReadRecord(record, expectedValueByTopicName));
     }
+
+    private void buildNoStreamProducer(Configuration.Builder config) {
+        start(PostgresConnector.class, config
+                .with(PostgresConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL_ONLY)
+                .with(PostgresConnectorConfig.SNAPSHOT_MODE_CLASS, CustomTestSnapshot.class.getName())
+                .with(PostgresConnectorConfig.DROP_SLOT_ON_STOP, Boolean.FALSE)
+                .build()
+        );
+        assertConnectorIsRunning();
+    }
+
+    private void buildWithStreamProducer(Configuration.Builder config) {
+        start(PostgresConnector.class, config
+                .with(PostgresConnectorConfig.SNAPSHOT_MODE, SnapshotMode.ALWAYS)
+                .with(PostgresConnectorConfig.SNAPSHOT_MODE_CLASS, CustomTestSnapshot.class.getName())
+                .with(PostgresConnectorConfig.DROP_SLOT_ON_STOP, Boolean.FALSE)
+                .build()
+        );
+        assertConnectorIsRunning();
+    }
 }

debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/RecordsStreamProducerIT.java

@@ -8,7 +8,9 @@
 
 import static io.debezium.connector.postgresql.TestHelper.PK_FIELD;
 import static io.debezium.connector.postgresql.TestHelper.topicName;
+import static io.debezium.connector.postgresql.junit.SkipWhenDecoderPluginNameIs.DecoderPluginName.DECODERBUFS;
 import static io.debezium.connector.postgresql.junit.SkipWhenDecoderPluginNameIs.DecoderPluginName.PGOUTPUT;
+import static io.debezium.connector.postgresql.junit.SkipWhenDecoderPluginNameIsNot.DecoderPluginName.WAL2JSON;
 import static junit.framework.TestCase.assertEquals;
 import static org.fest.assertions.Assertions.assertThat;
 import static org.junit.Assert.assertFalse;
@@ -19,29 +21,26 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
+import java.util.function.Function;
 
-import static io.debezium.connector.postgresql.junit.SkipWhenDecoderPluginNameIsNot.DecoderPluginName.WAL2JSON;
-import static io.debezium.connector.postgresql.junit.SkipWhenDecoderPluginNameIs.DecoderPluginName.DECODERBUFS;
-
-import io.debezium.config.Configuration;
-import io.debezium.connector.postgresql.junit.SkipTestDependingOnDecoderPluginNameRule;
-import io.debezium.connector.postgresql.junit.SkipWhenDecoderPluginNameIs;
-import io.debezium.connector.postgresql.junit.SkipWhenDecoderPluginNameIsNot;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.kafka.connect.data.Decimal;
 import org.apache.kafka.connect.data.SchemaBuilder;
 import org.apache.kafka.connect.data.Struct;
 import org.apache.kafka.connect.source.SourceRecord;
 import org.fest.assertions.Assertions;
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestRule;
 
 import io.debezium.config.CommonConnectorConfig;
+import io.debezium.config.Configuration;
 import io.debezium.connector.postgresql.PostgresConnectorConfig.SchemaRefreshMode;
+import io.debezium.connector.postgresql.PostgresConnectorConfig.SnapshotMode;
+import io.debezium.connector.postgresql.junit.SkipTestDependingOnDecoderPluginNameRule;
+import io.debezium.connector.postgresql.junit.SkipWhenDecoderPluginNameIs;
+import io.debezium.connector.postgresql.junit.SkipWhenDecoderPluginNameIsNot;
 import io.debezium.data.Envelope;
 import io.debezium.data.VariableScaleDecimal;
 import io.debezium.data.VerifyRecord;
@@ -52,7 +51,6 @@
 import io.debezium.junit.ShouldFailWhen;
 import io.debezium.relational.Table;
 import io.debezium.relational.TableId;
-import io.debezium.schema.TopicSelector;
 import io.debezium.util.Testing;
 
 /**
@@ -63,9 +61,7 @@
  */
 public class RecordsStreamProducerIT extends AbstractRecordsProducerTest {
 
-    private RecordsStreamProducer recordsProducer;
     private TestConsumer consumer;
-    private final Consumer<Throwable> blackHole = t -> {};
 
     @Rule
     public final TestRule skip = new SkipTestDependingOnDecoderPluginNameRule();
@@ -97,25 +93,44 @@ public void before() throws Exception {
                     .with("assumeMinServerVersion.set", "9.4");
         }
 
-        PostgresConnectorConfig config = new PostgresConnectorConfig(configBuilder.build());
-        setupRecordsProducer(config);
+        Testing.Print.enable();
     }
 
-    @After
-    public void after() throws Exception {
-        if (recordsProducer != null) {
-            recordsProducer.stop();
+    private void startConnector(Function<Configuration.Builder, Configuration.Builder> customConfig, boolean waitForSnapshot) throws InterruptedException {
+        start(PostgresConnector.class, new PostgresConnectorConfig(customConfig.apply(TestHelper.defaultConfig()
+                .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, false)
+                .with(PostgresConnectorConfig.SCHEMA_BLACKLIST, "postgis")
+                .with(PostgresConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL))
+                .build()).getConfig()
+        );
+        assertConnectorIsRunning();
+        waitForStreamingToStart();
+
+        if (waitForSnapshot) {
+            // Wait for snapshot to be in progress
+            consumer = testConsumer(1);
+            consumer.await(TestHelper.waitTimeForRecords(), TimeUnit.SECONDS);
+            consumer.remove();
         }
     }
 
+    private void startConnector(Function<Configuration.Builder, Configuration.Builder> customConfig) throws InterruptedException {
+        startConnector(customConfig, true);
+    }
+
+    private void startConnector() throws InterruptedException {
+        startConnector(Function.identity(), true);
+    }
+
     @Test
     public void shouldReceiveChangesForInsertsWithDifferentDataTypes() throws Exception {
         TestHelper.executeDDL("postgres_create_tables.ddl");
+        startConnector();
 
         consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
 
         //numerical types
+        consumer.expects(1);
         assertInsert(INSERT_NUMERIC_TYPES_STMT, 1, schemasAndValuesForNumericType());
 
         //numerical decimal types
@@ -157,12 +172,10 @@ public void shouldReceiveChangesAfterConnectionRestart() throws Exception {
         TestHelper.dropDefaultReplicationSlot();
         TestHelper.dropPublication();
 
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                 .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
-                 .with(PostgresConnectorConfig.SCHEMA_BLACKLIST, "postgis")
-                 .build());
-
-        setupRecordsProducer(config);
+        startConnector(config -> config
+                .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
+                .with(PostgresConnectorConfig.SCHEMA_BLACKLIST, "postgis")
+        );
 
         TestHelper.execute("CREATE TABLE t0 (pk SERIAL, d INTEGER, PRIMARY KEY(pk));");
 
@@ -170,7 +183,6 @@ public void shouldReceiveChangesAfterConnectionRestart() throws Exception {
 
         // Start the producer and wait; the wait is to guarantee that the stream thread is polling
         // This appears to be a potential race condition problem
-        recordsProducer.start(consumer, blackHole);
         TimeUnit.SECONDS.sleep(TestHelper.waitTimeForRecords());
 
         // Insert new row and verify inserted
@@ -178,19 +190,21 @@ public void shouldReceiveChangesAfterConnectionRestart() throws Exception {
         assertRecordInserted("public.t0", PK_FIELD, 1);
 
         // simulate the connector is stopped
-        recordsProducer.stop();
-
-        setupRecordsProducer(config);
+        stopConnector();
 
         // Alter schema offline
         TestHelper.execute("ALTER TABLE t0 ADD COLUMN d2 INTEGER;");
         TestHelper.execute("ALTER TABLE t0 ALTER COLUMN d SET NOT NULL;");
 
-        consumer = testConsumer(1);
 
         // Start the producer and wait; the wait is to guarantee the stream thread is polling
         // This appears to be a potential race condition problem
-        recordsProducer.start(consumer, blackHole);
+        startConnector(config -> config
+                .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
+                .with(PostgresConnectorConfig.SCHEMA_BLACKLIST, "postgis"),
+                false
+        );
+        consumer = testConsumer(1);
         TimeUnit.SECONDS.sleep(TestHelper.waitTimeForRecords());
 
         // Insert new row and verify inserted
@@ -200,19 +214,11 @@ public void shouldReceiveChangesAfterConnectionRestart() throws Exception {
 
     @Test
     public void shouldReceiveChangesForInsertsCustomTypes() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
-                .with(PostgresConnectorConfig.SCHEMA_BLACKLIST, "postgis")
-                .build());
-        setupRecordsProducer(config);
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
-
+        startConnector(config -> config.with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true));
         // custom types + null value
         assertInsert(INSERT_CUSTOM_TYPES_STMT, 1, schemasAndValuesForCustomTypes());
-
     }
 
     @Test
@@ -261,17 +267,15 @@ public void shouldProcessNotNullColumnsAdaptiveMsDateTypes() throws Exception {
     }
 
     private Struct testProcessNotNullColumns(TemporalPrecisionMode temporalMode) throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
+        TestHelper.executeDDL("postgres_create_tables.ddl");
+
+        startConnector(config -> config
                 .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
                 .with(PostgresConnectorConfig.SCHEMA_BLACKLIST, "postgis")
                 .with(PostgresConnectorConfig.TIME_PRECISION_MODE, temporalMode)
-                .build());
-        setupRecordsProducer(config);
-        TestHelper.executeDDL("postgres_create_tables.ddl");
-
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        );
 
+        consumer.expects(1);
         executeAndWait("INSERT INTO not_null_table VALUES (default, 30, '2019-02-10 11:34:58', '2019-02-10 11:35:00', '10:20:11', '10:20:12', '2019-02-01', '$20', B'101')");
         consumer.remove();
 
@@ -286,9 +290,10 @@ private Struct testProcessNotNullColumns(TemporalPrecisionMode temporalMode) thr
     @Test(timeout = 30000)
     public void shouldReceiveChangesForInsertsWithPostgisTypes() throws Exception {
         TestHelper.executeDDL("postgis_create_tables.ddl");
+
+        startConnector();
         consumer = testConsumer(1, "public"); // spatial_ref_sys produces a ton of records in the postgis schema
         consumer.setIgnoreExtraRecords(true);
-        recordsProducer.start(consumer, blackHole);
 
         // need to wait for all the spatial_ref_sys to flow through and be ignored.
         // this exceeds the normal 2s timeout.
@@ -312,9 +317,10 @@ public void shouldReceiveChangesForInsertsWithPostgisTypes() throws Exception {
     @Test(timeout = 30000)
     public void shouldReceiveChangesForInsertsWithPostgisArrayTypes() throws Exception {
         TestHelper.executeDDL("postgis_create_tables.ddl");
+
+        startConnector();
         consumer = testConsumer(1, "public"); // spatial_ref_sys produces a ton of records in the postgis schema
         consumer.setIgnoreExtraRecords(true);
-        recordsProducer.start(consumer, blackHole);
 
         // need to wait for all the spatial_ref_sys to flow through and be ignored.
         // this exceeds the normal 2s timeout.
@@ -341,8 +347,7 @@ public void shouldReceiveChangesForInsertsWithPostgisArrayTypes() throws Excepti
     public void shouldReceiveChangesForInsertsWithQuotedNames() throws Exception {
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector();
 
         // Quoted column name
         assertInsert(INSERT_QUOTED_TYPES_STMT, 1, schemasAndValuesForQuotedTypes());
@@ -352,19 +357,17 @@ public void shouldReceiveChangesForInsertsWithQuotedNames() throws Exception {
     public void shouldReceiveChangesForInsertsWithArrayTypes() throws Exception {
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector();
 
         assertInsert(INSERT_ARRAY_TYPES_STMT, 1, schemasAndValuesForArrayTypes());
     }
 
     @Test
     @FixFor("DBZ-1029")
-    public void shouldReceiveChangesForInsertsIndependentOfReplicaIdentity() {
+    public void shouldReceiveChangesForInsertsIndependentOfReplicaIdentity() throws Exception {
         // insert statement should not be affected by replica identity settings in any way
 
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector();
 
         TestHelper.execute("ALTER TABLE test_table REPLICA IDENTITY DEFAULT;");
         String statement = "INSERT INTO test_table (text) VALUES ('pk_and_default');";
@@ -391,8 +394,7 @@ public void shouldReceiveChangesForInsertsIndependentOfReplicaIdentity() {
     public void shouldReceiveChangesForNullInsertsWithArrayTypes() throws Exception {
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector();
 
         assertInsert(INSERT_ARRAY_TYPES_WITH_NULL_VALUES_STMT, 1, schemasAndValuesForArrayTypesWithNullValues());
     }
@@ -402,8 +404,9 @@ public void shouldReceiveChangesForNewTable() throws Exception {
         String statement = "CREATE SCHEMA s1;" +
                            "CREATE TABLE s1.a (pk SERIAL, aa integer, PRIMARY KEY(pk));" +
                            "INSERT INTO s1.a (aa) VALUES (11);";
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+
+        startConnector();
+
         executeAndWait(statement);
         assertRecordInserted("s1.a", PK_FIELD, 1);
     }
@@ -413,8 +416,8 @@ public void shouldReceiveChangesForRenamedTable() throws Exception {
         String statement = "DROP TABLE IF EXISTS renamed_test_table;" +
                            "ALTER TABLE test_table RENAME TO renamed_test_table;" +
                            "INSERT INTO renamed_test_table (text) VALUES ('new');";
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector();
+
         executeAndWait(statement);
         assertRecordInserted("public.renamed_test_table", PK_FIELD, 2);
     }
@@ -422,8 +425,7 @@ public void shouldReceiveChangesForRenamedTable() throws Exception {
     @Test
     @SkipWhenDecoderPluginNameIs(value = PGOUTPUT, reason = "An update on a table with no primary key and default replica throws PSQLException as tables must have a PK")
     public void shouldReceiveChangesForUpdates() throws Exception {
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector();
         executeAndWait("UPDATE test_table set text='update' WHERE pk=1");
 
         // the update record should be the last record
@@ -480,8 +482,8 @@ public void shouldReceiveChangesForUpdatesWithColumnChanges() throws Exception {
                             "ALTER TABLE test_table REPLICA IDENTITY FULL;" +
                             "UPDATE test_table SET uvc ='aa' WHERE pk = 1;";
 
+        startConnector();
         consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statements);
 
         // the update should be the last record
@@ -551,8 +553,8 @@ public void shouldReceiveChangesForUpdatesWithColumnChanges() throws Exception {
 
     @Test
     public void shouldReceiveChangesForUpdatesWithPKChanges() throws Exception {
+        startConnector();
         consumer = testConsumer(3);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait("UPDATE test_table SET text = 'update', pk = 2");
 
         String topicName = topicName("public.test_table");
@@ -576,14 +578,11 @@ public void shouldReceiveChangesForUpdatesWithPKChanges() throws Exception {
     @Test
     @FixFor("DBZ-582")
     public void shouldReceiveChangesForUpdatesWithPKChangesWithoutTombstone() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
+        startConnector(config -> config
                 .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
                 .with(CommonConnectorConfig.TOMBSTONES_ON_DELETE, false)
-                .build()
         );
-        setupRecordsProducer(config);
         consumer = testConsumer(2);
-        recordsProducer.start(consumer, blackHole);
 
         executeAndWait("UPDATE test_table SET text = 'update', pk = 2");
 
@@ -605,8 +604,8 @@ public void shouldReceiveChangesForDefaultValues() throws Exception {
         String statements = "ALTER TABLE test_table REPLICA IDENTITY FULL;" +
                             "ALTER TABLE test_table ADD COLUMN default_column TEXT DEFAULT 'default';" +
                             "INSERT INTO test_table (text) VALUES ('update');";
+        startConnector();
         consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statements);
 
         SourceRecord insertRecord = consumer.remove();
@@ -625,8 +624,8 @@ public void shouldReceiveChangesForTypeConstraints() throws Exception {
                             "ALTER TABLE test_table REPLICA IDENTITY FULL;" +
                             "UPDATE test_table SET num_val = 123.45 WHERE pk = 1;";
 
+        startConnector();
         consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statements);
 
         // the update should be the last record
@@ -720,8 +719,9 @@ public void shouldReceiveChangesForDeletes() throws Exception {
         // add a new entry and remove both
         String statements = "INSERT INTO test_table (text) VALUES ('insert2');" +
                             "DELETE FROM test_table WHERE pk > 0;";
+
+        startConnector();
         consumer = testConsumer(5);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statements);
 
 
@@ -753,18 +753,14 @@ record = consumer.remove();
     @Test
     @FixFor("DBZ-582")
     public void shouldReceiveChangesForDeletesWithoutTombstone() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
-                .with(CommonConnectorConfig.TOMBSTONES_ON_DELETE, false)
-                .build()
-        );
-        setupRecordsProducer(config);
-
         // add a new entry and remove both
         String statements = "INSERT INTO test_table (text) VALUES ('insert2');" +
                             "DELETE FROM test_table WHERE pk > 0;";
+        startConnector(config -> config
+                .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
+                .with(CommonConnectorConfig.TOMBSTONES_ON_DELETE, false)
+        );
         consumer = testConsumer(3);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statements);
 
 
@@ -786,19 +782,16 @@ record = consumer.remove();
     @Test
     @SkipWhenDecoderPluginNameIs(value = PGOUTPUT, reason = "A delete on a table with no primary key and default replica throws PSQLException as tables must have a PK")
     public void shouldReceiveChangesForDeletesDependingOnReplicaIdentity() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                                                                               .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
-                                                                               .with(CommonConnectorConfig.TOMBSTONES_ON_DELETE, false)
-                                                                               .build()
-        );
-        setupRecordsProducer(config);
         String topicName = topicName("public.test_table");
 
         // With PK we should get delete event with default level of replica identity
         String statement = "ALTER TABLE test_table REPLICA IDENTITY DEFAULT;" +
                             "DELETE FROM test_table WHERE pk = 1;";
+        startConnector(config -> config
+                .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true)
+                .with(CommonConnectorConfig.TOMBSTONES_ON_DELETE, false)
+        );
         consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statement);
         SourceRecord record = consumer.remove();
         assertEquals(topicName, record.topic());
@@ -828,15 +821,9 @@ record = consumer.remove();
 
     @Test
     public void shouldReceiveNumericTypeAsDouble() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.DECIMAL_HANDLING_MODE, PostgresConnectorConfig.DecimalHandlingMode.DOUBLE)
-                .build());
-        setupRecordsProducer(config);
-
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector(config -> config.with(PostgresConnectorConfig.DECIMAL_HANDLING_MODE, PostgresConnectorConfig.DecimalHandlingMode.DOUBLE));
 
         assertInsert(INSERT_NUMERIC_DECIMAL_TYPES_STMT, 1, schemasAndValuesForDoubleEncodedNumericTypes());
     }
@@ -844,15 +831,9 @@ public void shouldReceiveNumericTypeAsDouble() throws Exception {
     @Test
     @FixFor("DBZ-611")
     public void shouldReceiveNumericTypeAsString() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.DECIMAL_HANDLING_MODE, PostgresConnectorConfig.DecimalHandlingMode.STRING)
-                .build());
-        setupRecordsProducer(config);
-
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector(config -> config.with(PostgresConnectorConfig.DECIMAL_HANDLING_MODE, PostgresConnectorConfig.DecimalHandlingMode.STRING));
 
         assertInsert(INSERT_NUMERIC_DECIMAL_TYPES_STMT, 1, schemasAndValuesForStringEncodedNumericTypes());
     }
@@ -860,13 +841,9 @@ public void shouldReceiveNumericTypeAsString() throws Exception {
     @Test
     @FixFor("DBZ-898")
     public void shouldReceiveHStoreTypeWithSingleValueAsMap() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.MAP)
-                .build());
-        setupRecordsProducer(config);
         TestHelper.executeDDL("postgres_create_tables.ddl");
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+
+        startConnector(config -> config.with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.MAP));
 
         assertInsert(INSERT_HSTORE_TYPE_STMT, 1, schemaAndValueFieldForMapEncodedHStoreType());
     }
@@ -874,13 +851,9 @@ public void shouldReceiveHStoreTypeWithSingleValueAsMap() throws Exception {
     @Test
     @FixFor("DBZ-898")
     public void shouldReceiveHStoreTypeWithMultipleValuesAsMap() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.MAP)
-                .build());
-        setupRecordsProducer(config);
         TestHelper.executeDDL("postgres_create_tables.ddl");
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+
+        startConnector(config -> config.with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.MAP));
 
         assertInsert(INSERT_HSTORE_TYPE_WITH_MULTIPLE_VALUES_STMT, 1, schemaAndValueFieldForMapEncodedHStoreTypeWithMultipleValues());
     }
@@ -888,13 +861,9 @@ public void shouldReceiveHStoreTypeWithMultipleValuesAsMap() throws Exception {
     @Test
     @FixFor("DBZ-898")
     public void shouldReceiveHStoreTypeWithNullValuesAsMap() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.MAP)
-                .build());
-        setupRecordsProducer(config);
         TestHelper.executeDDL("postgres_create_tables.ddl");
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+
+        startConnector(config -> config.with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.MAP));
 
         assertInsert(INSERT_HSTORE_TYPE_WITH_NULL_VALUES_STMT, 1, schemaAndValueFieldForMapEncodedHStoreTypeWithNullValues());
     }
@@ -902,13 +871,9 @@ public void shouldReceiveHStoreTypeWithNullValuesAsMap() throws Exception {
     @Test
     @FixFor("DBZ-898")
     public void shouldReceiveHStoreTypeWithSpecialCharactersInValuesAsMap() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.MAP)
-                .build());
-        setupRecordsProducer(config);
         TestHelper.executeDDL("postgres_create_tables.ddl");
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+
+        startConnector(config -> config.with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.MAP));
 
         assertInsert(INSERT_HSTORE_TYPE_WITH_SPECIAL_CHAR_STMT, 1, schemaAndValueFieldForMapEncodedHStoreTypeWithSpecialCharacters());
     }
@@ -916,13 +881,10 @@ public void shouldReceiveHStoreTypeWithSpecialCharactersInValuesAsMap() throws E
     @Test
     @FixFor("DBZ-898")
     public void shouldReceiveHStoreTypeAsJsonString() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.JSON)
-                .build());
-        setupRecordsProducer(config);
         TestHelper.executeDDL("postgres_create_tables.ddl");
         consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+
+        startConnector(config -> config.with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.JSON));
 
         assertInsert(INSERT_HSTORE_TYPE_STMT, 1, schemaAndValueFieldForJsonEncodedHStoreType());
     }
@@ -930,13 +892,9 @@ public void shouldReceiveHStoreTypeAsJsonString() throws Exception {
     @Test
     @FixFor("DBZ-898")
     public void shouldReceiveHStoreTypeWithMultipleValuesAsJsonString() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.JSON)
-                .build());
-        setupRecordsProducer(config);
         TestHelper.executeDDL("postgres_create_tables.ddl");
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+
+        startConnector(config -> config.with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.JSON));
 
         assertInsert(INSERT_HSTORE_TYPE_WITH_MULTIPLE_VALUES_STMT, 1, schemaAndValueFieldForJsonEncodedHStoreTypeWithMultipleValues());
     }
@@ -944,13 +902,9 @@ public void shouldReceiveHStoreTypeWithMultipleValuesAsJsonString() throws Excep
     @Test
     @FixFor("DBZ-898")
     public void shouldReceiveHStoreTypeWithSpecialValuesInJsonString() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.JSON)
-                .build());
-        setupRecordsProducer(config);
         TestHelper.executeDDL("postgres_create_tables.ddl");
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+
+        startConnector(config -> config.with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.JSON));
 
         assertInsert(INSERT_HSTORE_TYPE_WITH_SPECIAL_CHAR_STMT, 1, schemaAndValueFieldForJsonEncodedHStoreTypeWithSpcialCharacters());
     }
@@ -958,13 +912,9 @@ public void shouldReceiveHStoreTypeWithSpecialValuesInJsonString() throws Except
     @Test
     @FixFor("DBZ-898")
     public void shouldReceiveHStoreTypeWithNullValuesAsJsonString() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.JSON)
-                .build());
-        setupRecordsProducer(config);
         TestHelper.executeDDL("postgres_create_tables.ddl");
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+
+        startConnector(config -> config.with(PostgresConnectorConfig.HSTORE_HANDLING_MODE, PostgresConnectorConfig.HStoreHandlingMode.JSON));
 
         assertInsert(INSERT_HSTORE_TYPE_WITH_NULL_VALUES_STMT, 1, schemaAndValueFieldForJsonEncodedHStoreTypeWithNullValues());
     }
@@ -977,8 +927,8 @@ public void shouldProcessIntervalDelete() throws Exception {
                 "INSERT INTO table_with_interval VALUES (default, 'Bar', default);" +
                 "DELETE FROM table_with_interval WHERE id = 1;";
 
-        consumer = testConsumer(4);
-        recordsProducer.start(consumer, blackHole);
+        startConnector();
+        consumer.expects(4);
         executeAndWait(statements);
 
         final String topicPrefix = "public.table_with_interval";
@@ -998,32 +948,12 @@ record = consumer.remove();
         VerifyRecord.isValidTombstone(record, pk, 1);
     }
 
-    @Test
-    @FixFor("DBZ-501")
-    public void shouldNotStartAfterStop() throws Exception {
-        recordsProducer.stop();
-        recordsProducer.start(consumer, blackHole);
-
-        // Need to remove record created in @Before
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig().with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true).build());
-        setupRecordsProducer(config);
-
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
-    }
-
     @Test
     @FixFor("DBZ-644")
     public void shouldPropagateSourceColumnTypeToSchemaParameter() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with("column.propagate.source.type", ".*vc.*")
-                .build());
-        setupRecordsProducer(config);
-
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector(config -> config.with("column.propagate.source.type", ".*vc.*"));
 
         assertInsert(INSERT_STRING_TYPES_STMT, 1, schemasAndValuesForStringTypesWithSourceColumnTypeInfo());
     }
@@ -1031,16 +961,12 @@ public void shouldPropagateSourceColumnTypeToSchemaParameter() throws Exception
     @Test
     @FixFor("DBZ-1073")
     public void shouldPropagateSourceColumnTypeScaleToSchemaParameter() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-            .with("column.propagate.source.type", ".*(d|dzs)")
-            .with(PostgresConnectorConfig.DECIMAL_HANDLING_MODE, PostgresConnectorConfig.DecimalHandlingMode.DOUBLE)
-            .build());
-        setupRecordsProducer(config);
-
         TestHelper.executeDDL("postgres_create_tables.ddl");
 
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector(config -> config
+            .with("column.propagate.source.type", ".*(d|dzs)")
+            .with(PostgresConnectorConfig.DECIMAL_HANDLING_MODE, PostgresConnectorConfig.DecimalHandlingMode.DOUBLE)
+        );
 
         assertInsert(INSERT_NUMERIC_DECIMAL_TYPES_STMT, 1, schemasAndValuesForNumericTypesWithSourceColumnTypeInfo());
     }
@@ -1050,13 +976,11 @@ public void shouldPropagateSourceColumnTypeScaleToSchemaParameter() throws Excep
     public void shouldReceiveHeartbeatAlsoWhenChangingNonWhitelistedTable() throws Exception {
         // the low heartbeat interval should make sure that a heartbeat message is emitted after each change record
         // received from Postgres
-        Testing.Print.enable();
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
+        startConnector(config -> config
                 .with(Heartbeat.HEARTBEAT_INTERVAL, "1")
                 .with(PostgresConnectorConfig.POLL_INTERVAL_MS, "50")
                 .with(PostgresConnectorConfig.TABLE_WHITELIST, "s1\\.b")
-                .build());
-        setupRecordsProducer(config);
+        );
 
         String statement = "CREATE SCHEMA s1;" +
                            "CREATE TABLE s1.a (pk SERIAL, aa integer, PRIMARY KEY(pk));" +
@@ -1068,7 +992,6 @@ public void shouldReceiveHeartbeatAlsoWhenChangingNonWhitelistedTable() throws E
         final int expectedHeartbeats = 5;
         consumer = testConsumer(1 + expectedHeartbeats);
         consumer.setIgnoreExtraRecords(true);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statement);
 
         // change record for s1.b and heartbeats
@@ -1082,17 +1005,15 @@ public void shouldReceiveHeartbeatAlsoWhenChangingNonWhitelistedTable() throws E
     @FixFor("DBZ-911")
     @SkipWhenDecoderPluginNameIs(value = PGOUTPUT, reason = "Decoder synchronizes all schema columns when processing relation messages")
     public void shouldNotRefreshSchemaOnUnchangedToastedData() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
+        startConnector(config -> config
                 .with(PostgresConnectorConfig.SCHEMA_REFRESH_MODE, PostgresConnectorConfig.SchemaRefreshMode.COLUMNS_DIFF_EXCLUDE_UNCHANGED_TOAST)
-                .build());
-        setupRecordsProducer(config);
+        );
 
         String toastedValue = RandomStringUtils.randomAlphanumeric(10000);
 
         // inserting a toasted value should /always/ produce a correct record
         String statement = "ALTER TABLE test_table ADD COLUMN not_toast integer; INSERT INTO test_table (not_toast, text) values (10, '" + toastedValue + "')";
         consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statement);
 
         SourceRecord record = consumer.remove();
@@ -1110,7 +1031,7 @@ public void shouldNotRefreshSchemaOnUnchangedToastedData() throws Exception {
 
         consumer.expects(1);
         executeAndWait(statement);
-        Table tbl = recordsProducer.schema().tableFor(TableId.parse("public.test_table"));
+        Table tbl = getSchema().tableFor(TableId.parse("public.test_table"));
         assertEquals(Arrays.asList("pk", "text", "not_toast"), tbl.retrieveColumnNames());
     }
 
@@ -1118,17 +1039,15 @@ public void shouldNotRefreshSchemaOnUnchangedToastedData() throws Exception {
     @FixFor("DBZ-911")
     @SkipWhenDecoderPluginNameIsNot(value = SkipWhenDecoderPluginNameIsNot.DecoderPluginName.PGOUTPUT, reason = "Decoder synchronizes all schema columns when processing relation messages")
     public void shouldRefreshSchemaOnUnchangedToastedDataWhenSchemaChanged() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                 .with(PostgresConnectorConfig.SCHEMA_REFRESH_MODE, PostgresConnectorConfig.SchemaRefreshMode.COLUMNS_DIFF_EXCLUDE_UNCHANGED_TOAST)
-                 .build());
-        setupRecordsProducer(config);
+        startConnector(config -> config
+                .with(PostgresConnectorConfig.SCHEMA_REFRESH_MODE, PostgresConnectorConfig.SchemaRefreshMode.COLUMNS_DIFF_EXCLUDE_UNCHANGED_TOAST)
+        );
 
         String toastedValue = RandomStringUtils.randomAlphanumeric(10000);
 
         // inserting a toasted value should /always/ produce a correct record
         String statement = "ALTER TABLE test_table ADD COLUMN not_toast integer; INSERT INTO test_table (not_toast, text) values (10, '" + toastedValue + "')";
         consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statement);
 
         SourceRecord record = consumer.remove();
@@ -1146,7 +1065,7 @@ public void shouldRefreshSchemaOnUnchangedToastedDataWhenSchemaChanged() throws
 
         consumer.expects(1);
         executeAndWait(statement);
-        Table tbl = recordsProducer.schema().tableFor(TableId.parse("public.test_table"));
+        Table tbl = getSchema().tableFor(TableId.parse("public.test_table"));
 
         assertEquals(Arrays.asList("pk", "not_toast"), tbl.retrieveColumnNames());
     }
@@ -1154,10 +1073,9 @@ public void shouldRefreshSchemaOnUnchangedToastedDataWhenSchemaChanged() throws
     @Test
     @FixFor("DBZ-842")
     public void shouldNotPropagateUnchangedToastedData() throws Exception {
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
+        startConnector(config -> config
                 .with(PostgresConnectorConfig.SCHEMA_REFRESH_MODE, PostgresConnectorConfig.SchemaRefreshMode.COLUMNS_DIFF_EXCLUDE_UNCHANGED_TOAST)
-                .build());
-        setupRecordsProducer(config);
+        );
 
         final String toastedValue1 = RandomStringUtils.randomAlphanumeric(10000);
         final String toastedValue2 = RandomStringUtils.randomAlphanumeric(10000);
@@ -1172,7 +1090,6 @@ public void shouldNotPropagateUnchangedToastedData() throws Exception {
               + "INSERT INTO test_table (not_toast, text, mandatory_text) values (10, '" + toastedValue1 + "', '" + toastedValue1 + "');"
               + "INSERT INTO test_table (not_toast, text, mandatory_text) values (10, '" + toastedValue2 + "', '" + toastedValue2 + "');";
         consumer = testConsumer(2);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statement);
 
         // after record should contain the toasted value
@@ -1194,7 +1111,7 @@ public void shouldNotPropagateUnchangedToastedData() throws Exception {
         consumer.expects(6);
         executeAndWait(statement);
         consumer.process(record -> {
-            Table tbl = recordsProducer.schema().tableFor(TableId.parse("public.test_table"));
+            Table tbl = getSchema().tableFor(TableId.parse("public.test_table"));
             assertEquals(Arrays.asList("pk", "text", "not_toast", "mandatory_text"), tbl.retrieveColumnNames());
         });
         assertRecordSchemaAndValues(Arrays.asList(
@@ -1238,8 +1155,8 @@ public void shouldReceiveChangesForTableWithoutPrimaryKey() throws Exception{
                 "ALTER TABLE test_table REPLICA IDENTITY FULL"
         );
 
+        startConnector(Function.identity(), false);
         consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
 
         // INSERT
         String statement = "INSERT INTO test_table (text) VALUES ('a');";
@@ -1291,10 +1208,9 @@ public void shouldReceiveChangesForTableWithoutPrimaryKey() throws Exception{
     public void testPassingStreamParams() throws Exception {
         // Verify that passing stream parameters works by using the WAL2JSON add-tables parameter which acts as a
         // whitelist.
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
+        startConnector(config -> config
                 .with(PostgresConnectorConfig.STREAM_PARAMS, "add-tables=s1.should_stream")
-                .build());
-        setupRecordsProducer(config);
+        );
         String statement = "CREATE SCHEMA s1;" +
                 "CREATE TABLE s1.should_stream (pk SERIAL, aa integer, PRIMARY KEY(pk));" +
                 "CREATE TABLE s1.should_not_stream (pk SERIAL, aa integer, PRIMARY KEY(pk));" +
@@ -1304,7 +1220,6 @@ public void testPassingStreamParams() throws Exception {
 
         // Verify only one record made it
         consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statement);
 
         // Verify the record that made it was from the whitelisted table
@@ -1317,10 +1232,9 @@ public void testPassingStreamParams() throws Exception {
     @SkipWhenDecoderPluginNameIsNot(value = WAL2JSON, reason = "WAL2JSON specific: Pass multiple stream parameters and values verifying they work")
     public void testPassingStreamMultipleParams() throws Exception {
         // Verify that passing multiple stream parameters and multiple parameter values works.
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
+        startConnector(config -> config
                 .with(PostgresConnectorConfig.STREAM_PARAMS, "add-tables=s1.should_stream,s2.*;filter-tables=s2.should_not_stream")
-                .build());
-        setupRecordsProducer(config);
+        );
         String statement = "CREATE SCHEMA s1;" + "CREATE SCHEMA s2;" +
                 "CREATE TABLE s1.should_stream (pk SERIAL, aa integer, PRIMARY KEY(pk));" +
                 "CREATE TABLE s2.should_stream (pk SERIAL, aa integer, PRIMARY KEY(pk));" +
@@ -1334,7 +1248,6 @@ public void testPassingStreamMultipleParams() throws Exception {
 
         // Verify only the whitelisted record from s1 and s2 made it.
         consumer = testConsumer(2);
-        recordsProducer.start(consumer, blackHole);
         executeAndWait(statement);
 
         // Verify the record that made it was from the whitelisted table
@@ -1373,14 +1286,10 @@ public void shouldReceiveChangesForReplicaIdentityFullTableWithToastedValueTable
     public void testEmptyChangesProducesHeartbeat() throws Exception {
         // the low heartbeat interval should make sure that a heartbeat message is emitted after each change record
         // received from Postgres
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(Heartbeat.HEARTBEAT_INTERVAL, "1")
-                .build());
-        setupRecordsProducer(config);
+        startConnector(config -> config.with(Heartbeat.HEARTBEAT_INTERVAL, "1"));
 
         // Expecting 1 heartbeat + 1 data change
-        consumer = testConsumer(1 + 1);
-        recordsProducer.start(consumer, blackHole);
+        consumer.expects(1 + 1);
 
         executeAndWait(
                 "DROP TABLE IF EXISTS test_table;" +
@@ -1403,13 +1312,7 @@ public void testEmptyChangesProducesHeartbeat() throws Exception {
     @Test
     @FixFor("DBZ-1082")
     public void shouldHaveNoXminWhenNotEnabled() throws Exception {
-        // Verify that passing multiple stream parameters and multiple parameter values works.
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.XMIN_FETCH_INTERVAL, "0")
-                .build());
-        setupRecordsProducer(config);
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector(config -> config.with(PostgresConnectorConfig.XMIN_FETCH_INTERVAL, "0"));
 
         TestHelper.execute("ALTER TABLE test_table REPLICA IDENTITY DEFAULT;");
         String statement = "INSERT INTO test_table (text) VALUES ('no_xmin');";
@@ -1428,13 +1331,7 @@ public void shouldHaveNoXminWhenNotEnabled() throws Exception {
     @Test
     @FixFor("DBZ-1082")
     public void shouldHaveXminWhenEnabled() throws Exception {
-        // Verify that passing multiple stream parameters and multiple parameter values works.
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.XMIN_FETCH_INTERVAL, "10")
-                .build());
-        setupRecordsProducer(config);
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector(config -> config.with(PostgresConnectorConfig.XMIN_FETCH_INTERVAL, "10"));
 
         TestHelper.execute("ALTER TABLE test_table REPLICA IDENTITY DEFAULT;");
         String statement = "INSERT INTO test_table (text) VALUES ('with_xmin');";
@@ -1451,11 +1348,6 @@ public void shouldHaveXminWhenEnabled() throws Exception {
     }
 
     private void testReceiveChangesForReplicaIdentityFullTableWithToastedValue(PostgresConnectorConfig.SchemaRefreshMode mode, boolean tablesBeforeStart) throws Exception{
-        PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig()
-                .with(PostgresConnectorConfig.SCHEMA_REFRESH_MODE, mode)
-                .build());
-        setupRecordsProducer(config);
-
         if (tablesBeforeStart) {
             TestHelper.execute(
                     "DROP TABLE IF EXISTS test_table;",
@@ -1464,8 +1356,7 @@ private void testReceiveChangesForReplicaIdentityFullTableWithToastedValue(Postg
             );
         }
 
-        consumer = testConsumer(1);
-        recordsProducer.start(consumer, blackHole);
+        startConnector(config -> config.with(PostgresConnectorConfig.SCHEMA_REFRESH_MODE, mode));
 
         final String toastedValue = RandomStringUtils.randomAlphanumeric(10000);
 
@@ -1515,8 +1406,6 @@ private void testReceiveChangesForReplicaIdentityFullTableWithToastedValue(Postg
                     new SchemaAndValueField("not_toast", SchemaBuilder.OPTIONAL_INT32_SCHEMA, 20)
             ), updatedRecord, Envelope.FieldName.AFTER);
         }
-
-        recordsProducer.stop();
     }
 
     private void assertHeartBeatRecordInserted() {
@@ -1532,21 +1421,6 @@ private void assertHeartBeatRecordInserted() {
         assertThat(value.getInt64("ts_ms")).isLessThanOrEqualTo(Instant.now().toEpochMilli());
     }
 
-    private void setupRecordsProducer(PostgresConnectorConfig config) {
-        if (recordsProducer != null) {
-            recordsProducer.stop();
-        }
-
-        TopicSelector<TableId> selector = PostgresTopicSelector.create(config);
-
-        PostgresTaskContext context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                selector
-        );
-        recordsProducer = new RecordsStreamProducer(context, new SourceInfo(config));
-    }
-
     private void assertInsert(String statement, List<SchemaAndValueField> expectedSchemaAndValuesByColumn) {
         assertInsert(statement, null, expectedSchemaAndValuesByColumn);
     }

debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/SnapshotWithOverridesProducerIT.java

@@ -13,17 +13,13 @@
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
-import io.debezium.connector.postgresql.snapshot.InitialOnlySnapshotter;
-import io.debezium.connector.postgresql.snapshot.SnapshotterWrapper;
-import io.debezium.connector.postgresql.spi.Snapshotter;
 import org.apache.kafka.connect.source.SourceRecord;
 import org.fest.assertions.Assertions;
-import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import io.debezium.config.Configuration;
-import io.debezium.relational.TableId;
-import io.debezium.schema.TopicSelector;
+import io.debezium.connector.postgresql.PostgresConnectorConfig.SnapshotMode;
 
 /**
  * Integration test for {@link io.debezium.connector.postgresql.PostgresConnectorConfig.SNAPSHOT_SELECT_STATEMENT_OVERRIDES_BY_TABLE}
@@ -49,43 +45,23 @@ public class SnapshotWithOverridesProducerIT extends AbstractRecordsProducerTest
             "INSERT INTO over.t2 VALUES (102);" +
             "INSERT INTO over.t2 VALUES (103);";
 
-    private RecordsSnapshotProducer snapshotProducer;
-    private PostgresTaskContext context;
-    private PostgresConnectorConfig config;
-
-    public void before(Configuration overrides) throws SQLException {
+    @Before
+    public void before() throws SQLException {
         TestHelper.dropAllSchemas();
-
-        config = new PostgresConnectorConfig(TestHelper.defaultConfig().with(overrides).build());
-        TopicSelector<TableId> selector = PostgresTopicSelector.create(config);
-        context = new PostgresTaskContext(
-                config,
-                TestHelper.getSchema(config),
-                selector
-        );
-    }
-
-    @After
-    public void after() throws Exception {
-        if (snapshotProducer != null) {
-            snapshotProducer.stop();
-        }
     }
 
     @Test
     public void shouldUseOverriddenSelectStatementDuringSnapshotting() throws Exception {
-        before(Configuration.create()
+        TestHelper.execute(STATEMENTS);
+
+        buildProducer(TestHelper.defaultConfig()
                 .with(PostgresConnectorConfig.SNAPSHOT_SELECT_STATEMENT_OVERRIDES_BY_TABLE, "over.t1")
                 .with(PostgresConnectorConfig.SNAPSHOT_SELECT_STATEMENT_OVERRIDES_BY_TABLE.name() + ".over.t1", "SELECT * FROM over.t1 WHERE pk > 100")
-                .build());
-        snapshotProducer = buildStreamProducer(context, config);
+        );
 
         final int expectedRecordsCount = 3 + 6;
 
-        TestHelper.execute(STATEMENTS);
         TestConsumer consumer = testConsumer(expectedRecordsCount, "over");
-
-        snapshotProducer.start(consumer, e -> {});
         consumer.await(TestHelper.waitTimeForRecords(), TimeUnit.SECONDS);
 
         final Map<String, List<SourceRecord>> recordsByTopic = recordsByTopic(expectedRecordsCount, consumer);
@@ -95,19 +71,17 @@ public void shouldUseOverriddenSelectStatementDuringSnapshotting() throws Except
 
     @Test
     public void shouldUseMultipleOverriddenSelectStatementsDuringSnapshotting() throws Exception {
-        before(Configuration.create()
+        TestHelper.execute(STATEMENTS);
+
+        buildProducer(TestHelper.defaultConfig()
                 .with(PostgresConnectorConfig.SNAPSHOT_SELECT_STATEMENT_OVERRIDES_BY_TABLE, "over.t1,over.t2")
                 .with(PostgresConnectorConfig.SNAPSHOT_SELECT_STATEMENT_OVERRIDES_BY_TABLE.name() + ".over.t1", "SELECT * FROM over.t1 WHERE pk > 101")
                 .with(PostgresConnectorConfig.SNAPSHOT_SELECT_STATEMENT_OVERRIDES_BY_TABLE.name() + ".over.t2", "SELECT * FROM over.t2 WHERE pk > 100")
-                .build());
-        snapshotProducer = buildStreamProducer(context, config);
+        );
 
         final int expectedRecordsCount = 2 + 3;
 
-        TestHelper.execute(STATEMENTS);
         TestConsumer consumer = testConsumer(expectedRecordsCount, "over");
-
-        snapshotProducer.start(consumer, e -> {});
         consumer.await(TestHelper.waitTimeForRecords(), TimeUnit.SECONDS);
 
         final Map<String, List<SourceRecord>> recordsByTopic = recordsByTopic(expectedRecordsCount, consumer);
@@ -125,9 +99,11 @@ private Map<String, List<SourceRecord>> recordsByTopic(final int expectedRecords
         return recordsByTopic;
     }
 
-    private RecordsSnapshotProducer buildStreamProducer(PostgresTaskContext ctx, PostgresConnectorConfig config) {
-        Snapshotter sn = new InitialOnlySnapshotter();
-        SnapshotterWrapper snw = new SnapshotterWrapper(sn, config, null, null);
-        return new RecordsSnapshotProducer(ctx, TestHelper.sourceInfo(), snw);
+    private void buildProducer(Configuration.Builder config) {
+        start(PostgresConnector.class, config
+                .with(PostgresConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL_ONLY)
+                .build()
+        );
+        assertConnectorIsRunning();
     }
 }

debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/TestHelper.java

@@ -38,7 +38,7 @@
 public final class TestHelper {
 
     protected static final String TEST_SERVER = "test_server";
-    protected static final String TEST_DATABASE = "test_database";
+    protected static final String TEST_DATABASE = "postgres";
     protected static final String PK_FIELD = "pk";
     private static final String TEST_PROPERTY_PREFIX = "debezium.test.";
     private static final Logger LOGGER = LoggerFactory.getLogger(TestHelper.class);

debezium-connector-postgres/src/test/resources/log4j.properties

@@ -9,6 +9,7 @@ log4j.rootLogger=INFO, stdout
 
 # Set up the default logging to be INFO level, then override specific units
 log4j.logger.io.debezium=INFO
+log4j.logger.io.debezium.connector.postgresql=INFO
 log4j.logger.io.debezium.embedded.EmbeddedEngine$EmbeddedConfig=WARN
 #log4j.logger.io.debezium.embedded.EmbeddedEngine=DEBUG
 #log4j.logger.io.debezium.connector.postgresql.RecordsStreamProducer=DEBUG

debezium-connector-sqlserver/src/main/java/io/debezium/connector/sqlserver/SqlServerOffsetContext.java

@@ -108,14 +108,6 @@ public void setChangePosition(TxLogPosition position, int eventCount) {
         sourceInfo.setChangeLsn(position.getInTxLsn());
     }
 
-    public void setSourceTime(Instant instant) {
-        sourceInfo.setSourceTime(instant);
-    }
-
-    public void setTableId(TableId tableId) {
-        sourceInfo.setTableId(tableId);
-    }
-
     @Override
     public boolean isSnapshotRunning() {
         return sourceInfo.isSnapshot() && !snapshotCompleted;
@@ -186,4 +178,10 @@ public String toString() {
     public void markLastSnapshotRecord() {
         sourceInfo.setSnapshot(SnapshotRecord.LAST);
     }
+
+    @Override
+    public void event(TableId tableId, Instant timestamp) {
+        sourceInfo.setSourceTime(timestamp);
+        sourceInfo.setTableId(tableId);
+    }
 }

debezium-connector-sqlserver/src/main/java/io/debezium/connector/sqlserver/SqlServerSnapshotChangeEventSource.java

@@ -10,7 +10,7 @@
 import java.sql.SQLException;
 import java.sql.Savepoint;
 import java.sql.Statement;
-import java.time.Instant;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -20,16 +20,15 @@
 import io.debezium.connector.sqlserver.SqlServerConnectorConfig.SnapshotIsolationMode;
 import io.debezium.pipeline.EventDispatcher;
 import io.debezium.pipeline.source.spi.SnapshotProgressListener;
-import io.debezium.pipeline.spi.ChangeRecordEmitter;
 import io.debezium.pipeline.spi.OffsetContext;
-import io.debezium.relational.HistorizedRelationalSnapshotChangeEventSource;
+import io.debezium.relational.RelationalSnapshotChangeEventSource;
 import io.debezium.relational.Table;
 import io.debezium.relational.TableId;
 import io.debezium.schema.SchemaChangeEvent;
 import io.debezium.schema.SchemaChangeEvent.SchemaChangeEventType;
 import io.debezium.util.Clock;
 
-public class SqlServerSnapshotChangeEventSource extends HistorizedRelationalSnapshotChangeEventSource {
+public class SqlServerSnapshotChangeEventSource extends RelationalSnapshotChangeEventSource {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(SqlServerSnapshotChangeEventSource.class);
 
@@ -199,15 +198,8 @@ protected void complete(SnapshotContext snapshotContext) {
      * @return a valid query string
      */
     @Override
-    protected String getSnapshotSelect(SnapshotContext snapshotContext, TableId tableId) {
-        return String.format("SELECT * FROM [%s].[%s]", tableId.schema(), tableId.table());
-    }
-
-    @Override
-    protected ChangeRecordEmitter getChangeRecordEmitter(SnapshotContext snapshotContext, TableId tableId, Object[] row) {
-        ((SqlServerOffsetContext) snapshotContext.offset).setSourceTime(Instant.ofEpochMilli(getClock().currentTimeInMillis()));
-        ((SqlServerOffsetContext) snapshotContext.offset).setTableId(tableId);
-        return new SnapshotChangeRecordEmitter(snapshotContext.offset, row, getClock());
+    protected Optional<String> getSnapshotSelect(SnapshotContext snapshotContext, TableId tableId) {
+        return Optional.of(String.format("SELECT * FROM [%s].[%s]", tableId.schema(), tableId.table()));
     }
 
     /**

debezium-connector-sqlserver/src/main/java/io/debezium/connector/sqlserver/SqlServerStreamingChangeEventSource.java

@@ -222,8 +222,7 @@ public void execute(ChangeEventSourceContext context) throws InterruptedExceptio
                             final Object[] dataNext = (operation == SqlServerChangeRecordEmitter.OP_UPDATE_BEFORE) ? tableWithSmallestLsn.getData() : null;
 
                             offsetContext.setChangePosition(tableWithSmallestLsn.getChangePosition(), eventCount);
-                            offsetContext.setSourceTime(metadataConnection.timestampOfLsn(tableWithSmallestLsn.getChangePosition().getCommitLsn()));
-                            offsetContext.setTableId(tableWithSmallestLsn.getChangeTable().getSourceTableId());
+                            offsetContext.event(tableWithSmallestLsn.getChangeTable().getSourceTableId(), metadataConnection.timestampOfLsn(tableWithSmallestLsn.getChangePosition().getCommitLsn()));
 
                             dispatcher
                                     .dispatchDataChangeEvent(

debezium-connector-sqlserver/src/test/java/io/debezium/connector/sqlserver/SnapshotIT.java

@@ -87,6 +87,7 @@ public void takeSnapshotInExclusiveMode() throws Exception {
 
     @Test
     public void takeSnapshotInSnapshotMode() throws Exception {
+        Testing.Print.enable();
         takeSnapshot(SnapshotIsolationMode.SNAPSHOT);
     }
 

debezium-core/src/main/java/io/debezium/pipeline/EventDispatcher.java

@@ -6,6 +6,7 @@
 package io.debezium.pipeline;
 
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Supplier;
 
 import org.apache.kafka.connect.data.Schema;
@@ -15,7 +16,9 @@
 import org.slf4j.LoggerFactory;
 
 import io.debezium.config.CommonConnectorConfig;
+import io.debezium.connector.SnapshotRecord;
 import io.debezium.connector.base.ChangeEventQueue;
+import io.debezium.data.Envelope;
 import io.debezium.data.Envelope.Operation;
 import io.debezium.heartbeat.Heartbeat;
 import io.debezium.pipeline.source.spi.DataChangeEventListener;
@@ -54,6 +57,7 @@ public class EventDispatcher<T extends DataCollectionId> {
     private final Heartbeat heartbeat;
     private DataChangeEventListener eventListener = DataChangeEventListener.NO_OP;
     private final boolean emitTombstonesOnDelete;
+    private InconsistentSchemaHandler<T> inconsistentSchemaHandler = this::errorOnMissingSchema;
 
     /**
      * Change event receiver for events dispatched from a streaming change event source.
@@ -85,8 +89,7 @@ public void dispatchSnapshotEvent(T dataCollectionId, ChangeRecordEmitter change
 
         // TODO handle as per inconsistent schema info option
         if (dataCollectionSchema == null) {
-            eventListener.onErroneousEvent("source = " + dataCollectionId);
-            throw new IllegalArgumentException("No metadata registered for captured table " + dataCollectionId);
+            errorOnMissingSchema(dataCollectionId, changeRecordEmitter);
         }
 
         changeRecordEmitter.emitChangeRecords(dataCollectionSchema, new Receiver() {
@@ -122,9 +125,13 @@ public void dispatchDataChangeEvent(T dataCollectionId, ChangeRecordEmitter chan
 
             // TODO handle as per inconsistent schema info option
             if (dataCollectionSchema == null) {
-                eventListener.onErroneousEvent("source = " + dataCollectionId);
-                throw new IllegalArgumentException("No metadata registered for captured table " + dataCollectionId);
+                final Optional<DataCollectionSchema> replacementSchema = inconsistentSchemaHandler.handle(dataCollectionId, changeRecordEmitter); 
+                if (!replacementSchema.isPresent()) {
+                    return;
+                }
+                dataCollectionSchema = replacementSchema.get();
             }
+            dataCollectionSchema = changeRecordEmitter.synchronizeTableSchema(dataCollectionSchema);
 
             changeRecordEmitter.emitChangeRecords(dataCollectionSchema, new Receiver() {
 
@@ -144,6 +151,19 @@ public void changeRecord(DataCollectionSchema schema, Operation operation, Objec
         );
     }
 
+    public void setInconsistentSchemaHandler(InconsistentSchemaHandler<T> inconsistentSchemaHandler) {
+        this.inconsistentSchemaHandler = inconsistentSchemaHandler;
+    }
+
+    public Optional<DataCollectionSchema> errorOnMissingSchema(T dataCollectionId, ChangeRecordEmitter changeRecordEmitter) {
+        eventListener.onErroneousEvent("source = " + dataCollectionId);
+        throw new IllegalArgumentException("No metadata registered for captured table " + dataCollectionId);
+    }
+
+    public Optional<DataCollectionSchema> ignoreMissingSchema(T dataCollectionId, ChangeRecordEmitter changeRecordEmitter) {
+        return Optional.empty();
+    }
+
     public void dispatchSchemaChangeEvent(T dataCollectionId, SchemaChangeEventEmitter schemaChangeEventEmitter) throws InterruptedException {
         if(!filter.isIncluded(dataCollectionId)) {
             LOGGER.trace("Filtering schema change event for {}", dataCollectionId);
@@ -227,7 +247,6 @@ public void changeRecord(DataCollectionSchema dataCollectionSchema, Operation op
             bufferedEvent = () -> {
                 SourceRecord record = new SourceRecord(offsetContext.getPartition(), offsetContext.getOffset(),
                         topicName, null, keySchema, key, dataCollectionSchema.getEnvelopeSchema().schema(), value);
-
                 return changeEventCreator.createDataChangeEvent(record);
             };
         }
@@ -235,7 +254,19 @@ public void changeRecord(DataCollectionSchema dataCollectionSchema, Operation op
         @Override
         public void completeSnapshot() throws InterruptedException {
             if(bufferedEvent != null) {
-                queue.enqueue(bufferedEvent.get());
+                // It is possible that the last snapshotted table was empty
+                // this way we ensure that the last event is always marked as last
+                // even if it originates form non-last table
+                final DataChangeEvent event = bufferedEvent.get();
+                final Struct envelope = (Struct) event.getRecord().value();
+                if (envelope.schema().field(Envelope.FieldName.SOURCE) != null) {
+                    final Struct source = envelope.getStruct(Envelope.FieldName.SOURCE);
+                    final SnapshotRecord snapshot = SnapshotRecord.fromSource(source);
+                    if (snapshot == SnapshotRecord.TRUE) {
+                        SnapshotRecord.LAST.toSource(source);
+                    }
+                }
+                queue.enqueue(event);
                 bufferedEvent = null;
             }
         }
@@ -257,4 +288,17 @@ public void schemaChangeEvent(SchemaChangeEvent event) throws InterruptedExcepti
     public void setEventListener(DataChangeEventListener eventListener) {
         this.eventListener = eventListener;
     }
+
+    /**
+     * Reaction to an incoming change event for which schema is not found
+     */
+    @FunctionalInterface
+    public static interface InconsistentSchemaHandler<T extends DataCollectionId> {
+
+        /**
+         * @return collection schema if the schema was updated and event can be processed, {@code empty} to skip the processing
+         */
+        Optional<DataCollectionSchema> handle(T dataCollectionId, ChangeRecordEmitter changeRecordEmitter);
+    }
+
 }

debezium-core/src/main/java/io/debezium/pipeline/spi/ChangeRecordEmitter.java

@@ -29,6 +29,10 @@ public interface ChangeRecordEmitter {
      */
     OffsetContext getOffset();
 
+    default DataCollectionSchema synchronizeTableSchema(DataCollectionSchema dataCollectionSchema) {
+        return dataCollectionSchema;
+    }
+
     public interface Receiver {
         void changeRecord(DataCollectionSchema schema, Operation operation, Object key, Struct value, OffsetContext offset) throws InterruptedException;
     }

debezium-core/src/main/java/io/debezium/pipeline/spi/OffsetContext.java

@@ -5,11 +5,14 @@
  */
 package io.debezium.pipeline.spi;
 
+import java.time.Instant;
 import java.util.Map;
 
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.Struct;
 
+import io.debezium.relational.TableId;
+
 /**
  * Keeps track of the current offset within the source DB's change stream. This reflects in the offset as committed to
  * Kafka and in the source info block contained within CDC messages themselves.
@@ -57,4 +60,11 @@ interface Loader {
      * Signals that a snapshot has been completed, which should reflect in an updated offset state.
      */
     void postSnapshotCompletion();
+
+    /**
+     * Records the name of the table and the timestamp of the last event
+     * @param tableId
+     * @param timestamp
+     */
+    void event(TableId tableId, Instant timestamp);
 }

debezium-core/src/main/java/io/debezium/relational/RelationalChangeRecordEmitter.java

@@ -89,8 +89,9 @@ private void emitUpdateRecord(Receiver receiver, TableSchema tableSchema)
         Struct newValue = tableSchema.valueFromColumnData(newColumnValues);
         Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues);
 
-        // regular update
-        if (Objects.equals(oldKey, newKey)) {
+        // some configurations does not provide old values in case of updates
+        // in this case we handle all updates as regular ones
+        if (oldKey == null || Objects.equals(oldKey, newKey)) {
             Struct envelope = tableSchema.getEnvelopeSchema().update(oldValue, newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis());
             receiver.changeRecord(tableSchema, Operation.UPDATE, newKey, envelope, offsetContext);
         }

debezium-core/src/main/java/io/debezium/relational/RelationalDatabaseConnectorConfig.java

@@ -29,6 +29,8 @@
  * @author Gunnar Morling
  */
 public abstract class RelationalDatabaseConnectorConfig extends CommonConnectorConfig {
+    private static final String TABLE_BLACKLIST_NAME = "table.blacklist";
+    private static final String TABLE_WHITELIST_NAME = "table.whitelist";
 
     /**
      * The set of predefined DecimalHandlingMode options or aliases.
@@ -127,7 +129,7 @@ public static DecimalHandlingMode parse(String value, String defaultValue) {
      * {@code <databaseName>.<schemaName>.<tableName>}. May not be used with {@link #TABLE_BLACKLIST}, and superseded by database
      * inclusions/exclusions.
      */
-    public static final Field TABLE_WHITELIST = Field.create("table.whitelist")
+    public static final Field TABLE_WHITELIST = Field.create(TABLE_WHITELIST_NAME)
                                                      .withDisplayName("Included tables")
                                                      .withType(Type.LIST)
                                                      .withWidth(Width.LONG)
@@ -140,7 +142,7 @@ public static DecimalHandlingMode parse(String value, String defaultValue) {
      * monitoring. Fully-qualified names for tables are of the form {@code <databaseName>.<tableName>} or
      * {@code <databaseName>.<schemaName>.<tableName>}. May not be used with {@link #TABLE_WHITELIST}.
      */
-    public static final Field TABLE_BLACKLIST = Field.create("table.blacklist")
+    public static final Field TABLE_BLACKLIST = Field.create(TABLE_BLACKLIST_NAME)
                                                      .withDisplayName("Excluded tables")
                                                      .withType(Type.STRING)
                                                      .withWidth(Width.LONG)
@@ -192,6 +194,31 @@ public static DecimalHandlingMode parse(String value, String defaultValue) {
                     "The value of those properties is the select statement to use when retrieving data from the specific table during snapshotting. " +
                     "A possible use case for large append-only tables is setting a specific point where to start (resume) snapshotting, in case a previous snapshotting was interrupted.");
 
+    /**
+     * A comma-separated list of regular expressions that match schema names to be monitored.
+     * May not be used with {@link #SCHEMA_BLACKLIST}.
+     */
+    public static final Field SCHEMA_WHITELIST = Field.create("schema.whitelist")
+                                                      .withDisplayName("Schemas")
+                                                      .withType(Type.LIST)
+                                                      .withWidth(Width.LONG)
+                                                      .withImportance(Importance.HIGH)
+                                                      .withDependents(TABLE_WHITELIST_NAME)
+                                                      .withDescription("The schemas for which events should be captured");
+
+    /**
+     * A comma-separated list of regular expressions that match schema names to be excluded from monitoring.
+     * May not be used with {@link #SCHEMA_WHITELIST}.
+     */
+    public static final Field SCHEMA_BLACKLIST = Field.create("schema.blacklist")
+                                                      .withDisplayName("Exclude Schemas")
+                                                      .withType(Type.STRING)
+                                                      .withWidth(Width.LONG)
+                                                      .withImportance(Importance.MEDIUM)
+                                                      .withValidation(RelationalDatabaseConnectorConfig::validateSchemaBlacklist)
+                                                      .withInvisibleRecommender()
+                                                      .withDescription("The schemas for which events must not be captured");
+
     private final RelationalTableFilters tableFilters;
 
     protected RelationalDatabaseConnectorConfig(Configuration config, String logicalName, TableFilter systemTablesFilter,
@@ -254,4 +281,14 @@ public Map<TableId, String> getSnapshotSelectOverridesByTable() {
 
         return Collections.unmodifiableMap(snapshotSelectOverridesByTable);
     }
+
+    private static int validateSchemaBlacklist(Configuration config, Field field, Field.ValidationOutput problems) {
+        String whitelist = config.getString(SCHEMA_WHITELIST);
+        String blacklist = config.getString(SCHEMA_BLACKLIST);
+        if (whitelist != null && blacklist != null) {
+            problems.accept(SCHEMA_BLACKLIST, blacklist, "Schema whitelist is already specified");
+            return 1;
+        }
+        return 0;
+    }
 }

debezium-core/src/main/java/io/debezium/relational/RelationalSnapshotChangeEventSource.java

@@ -11,9 +11,11 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
+import java.util.Optional;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -42,7 +44,7 @@
 import io.debezium.util.Threads.Timer;
 
 /**
- * Base class for {@link SnapshotChangeEventSource} for relational databases with a schema history.
+ * Base class for {@link SnapshotChangeEventSource} for relational databases with or without a schema history.
  * <p>
  * A transaction is managed by this base class, sub-classes shouldn't rollback or commit this transaction. They are free
  * to use nested transactions or savepoints, though.
@@ -51,9 +53,9 @@
  */
 // TODO Mostly, this should be usable for Postgres as well; only the aspect of managing the schema history will have to
 // be made optional based on the connector
-public abstract class HistorizedRelationalSnapshotChangeEventSource implements SnapshotChangeEventSource {
+public abstract class RelationalSnapshotChangeEventSource implements SnapshotChangeEventSource {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(HistorizedRelationalSnapshotChangeEventSource.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(RelationalSnapshotChangeEventSource.class);
 
     /**
      * Interval for showing a log statement with the progress while scanning a single table.
@@ -65,10 +67,10 @@ public abstract class HistorizedRelationalSnapshotChangeEventSource implements S
     private final JdbcConnection jdbcConnection;
     private final HistorizedRelationalDatabaseSchema schema;
     private final EventDispatcher<TableId> dispatcher;
-    private final Clock clock;
+    protected final Clock clock;
     private final SnapshotProgressListener snapshotProgressListener;
 
-    public HistorizedRelationalSnapshotChangeEventSource(RelationalDatabaseConnectorConfig connectorConfig,
+    public RelationalSnapshotChangeEventSource(RelationalDatabaseConnectorConfig connectorConfig,
             OffsetContext previousOffset, JdbcConnection jdbcConnection, HistorizedRelationalDatabaseSchema schema,
             EventDispatcher<TableId> dispatcher, Clock clock, SnapshotProgressListener snapshotProgressListener) {
         this.connectorConfig = connectorConfig;
@@ -80,6 +82,12 @@ public HistorizedRelationalSnapshotChangeEventSource(RelationalDatabaseConnector
         this.snapshotProgressListener = snapshotProgressListener;
     }
 
+    public RelationalSnapshotChangeEventSource(RelationalDatabaseConnectorConfig connectorConfig,
+            OffsetContext previousOffset, JdbcConnection jdbcConnection,
+            EventDispatcher<TableId> dispatcher, Clock clock, SnapshotProgressListener snapshotProgressListener) {
+        this(connectorConfig, previousOffset, jdbcConnection, null, dispatcher, clock, snapshotProgressListener);
+    }
+
     @Override
     public SnapshotResult execute(ChangeEventSourceContext context) throws InterruptedException {
         SnapshottingTask snapshottingTask = getSnapshottingTask(previousOffset);
@@ -299,7 +307,9 @@ private void createSchemaChangeEventsForTables(ChangeEventSourceContext sourceCo
 
             Table table = snapshotContext.tables.forTable(tableId);
 
-            schema.applySchemaChange(getCreateTableEvent(snapshotContext, table));
+            if (schema != null) {
+                schema.applySchemaChange(getCreateTableEvent(snapshotContext, table));
+            }
         }
     }
 
@@ -339,11 +349,15 @@ private void createDataEventsForTable(ChangeEventSourceContext sourceContext, Sn
         long exportStart = clock.currentTimeInMillis();
         LOGGER.info("\t Exporting data from table '{}'", table.id());
 
-        final String selectStatement = determineSnapshotSelect(snapshotContext, table.id());
-        LOGGER.info("\t For table '{}' using select statement: '{}'", table.id(), selectStatement);
+        final Optional<String> selectStatement = determineSnapshotSelect(snapshotContext, table.id());
+        if (!selectStatement.isPresent()) {
+            LOGGER.warn("For table '{}' the select statement was not provided, skipping table", table.id());
+            return;
+        }
+        LOGGER.info("\t For table '{}' using select statement: '{}'", table.id(), selectStatement.get());
 
         try (Statement statement = readTableStatement();
-                ResultSet rs = statement.executeQuery(selectStatement)) {
+                ResultSet rs = statement.executeQuery(selectStatement.get())) {
 
             Column[] columns = getColumnsForResultSet(table, rs);
             final int numColumns = table.columns().size();
@@ -375,10 +389,13 @@ private void createDataEventsForTable(ChangeEventSourceContext sourceContext, Sn
                     if (snapshotContext.lastTable && snapshotContext.lastRecordInTable) {
                         snapshotContext.offset.markLastSnapshotRecord();
                     }
-                    dispatcher.dispatchSnapshotEvent(table.id(), getChangeRecordEmitter(snapshotContext, table.id(), row),
-                            snapshotReceiver);
+                    dispatcher.dispatchSnapshotEvent(table.id(), getChangeRecordEmitter(snapshotContext, table.id(), row), snapshotReceiver);
                 }
             }
+            else if (snapshotContext.lastTable) {
+                // if the last table does not contain any records we still need to mark the last processed event as the last one
+                snapshotContext.offset.markLastSnapshotRecord();
+            }
 
             LOGGER.info("\t Finished exporting {} records for table '{}'; total duration '{}'", rows,
                     table.id(), Strings.duration(clock.currentTimeInMillis() - exportStart));
@@ -396,16 +413,19 @@ private Timer getTableScanLogTimer() {
     /**
      * Returns a {@link ChangeRecordEmitter} producing the change records for the given table row.
      */
-    protected abstract ChangeRecordEmitter getChangeRecordEmitter(SnapshotContext snapshotContext, TableId tableId, Object[] row);
+    protected ChangeRecordEmitter getChangeRecordEmitter(SnapshotContext snapshotContext, TableId tableId, Object[] row) {
+        snapshotContext.offset.event(tableId, Instant.ofEpochMilli(getClock().currentTimeInMillis()));
+        return new SnapshotChangeRecordEmitter(snapshotContext.offset, row, getClock());
+    }
 
     /**
      * Returns a valid query string for the specified table, either given by the user via snapshot select overrides or
      * defaulting to a statement provided by the DB-specific change event source.
      *
      * @param tableId the table to generate a query for
-     * @return a valid query string
+     * @return a valid query string or empty if table will not be snapshotted
      */
-    private String determineSnapshotSelect(SnapshotContext snapshotContext, TableId tableId) {
+    private Optional<String> determineSnapshotSelect(SnapshotContext snapshotContext, TableId tableId) {
         String overriddenSelect = connectorConfig.getSnapshotSelectOverridesByTable().get(tableId);
 
         // try without catalog id, as this might or might not be populated based on the given connector
@@ -413,16 +433,17 @@ private String determineSnapshotSelect(SnapshotContext snapshotContext, TableId
             overriddenSelect = connectorConfig.getSnapshotSelectOverridesByTable().get(new TableId(null, tableId.schema(), tableId.table()));
         }
 
-        return overriddenSelect != null ? overriddenSelect : getSnapshotSelect(snapshotContext, tableId);
+        return overriddenSelect != null ? Optional.of(overriddenSelect) : getSnapshotSelect(snapshotContext, tableId);
     }
 
     /**
-     * Returns the SELECT statement to be used for scanning the given table
+     * Returns the SELECT statement to be used for scanning the given table or empty value if
+     * the table will be streamed from but not snapshotted
      */
     // TODO Should it be Statement or similar?
     // TODO Handle override option generically; a problem will be how to handle the dynamic part (Oracle's "... as of
     // scn xyz")
-    protected abstract String getSnapshotSelect(SnapshotContext snapshotContext, TableId tableId);
+    protected abstract Optional<String> getSnapshotSelect(SnapshotContext snapshotContext, TableId tableId);
 
     private Column[] getColumnsForResultSet(Table table, ResultSet rs) throws SQLException {
         ResultSetMetaData metaData = rs.getMetaData();
@@ -435,7 +456,7 @@ private Column[] getColumnsForResultSet(Table table, ResultSet rs) throws SQLExc
         return columns;
     }
 
-    private Object getColumnValue(ResultSet rs, int columnIndex, Column column) throws SQLException {
+    protected Object getColumnValue(ResultSet rs, int columnIndex, Column column) throws SQLException {
         return rs.getObject(columnIndex);
     }
 

debezium-core/src/main/java/io/debezium/relational/RelationalTableFilters.java

@@ -21,7 +21,8 @@ public RelationalTableFilters(Configuration config, TableFilter systemTablesFilt
         Predicate<TableId> predicate = Selectors.tableSelector()
 //                                                  .includeDatabases(config.getString(RelationalDatabaseConnectorConfig.DATABASE_WHITELIST))
 //                                                  .excludeDatabases(config.getString(RelationalDatabaseConnectorConfig.DATABASE_BLACKLIST))
-                                                  .includeTables(config.getString(RelationalDatabaseConnectorConfig.TABLE_WHITELIST), tableIdMapper)
+                                                  .includeSchemas(config.getString(RelationalDatabaseConnectorConfig.SCHEMA_WHITELIST))
+                                                  .excludeSchemas(config.getString(RelationalDatabaseConnectorConfig.SCHEMA_BLACKLIST))                                                  .includeTables(config.getString(RelationalDatabaseConnectorConfig.TABLE_WHITELIST), tableIdMapper)
                                                   .excludeTables(config.getString(RelationalDatabaseConnectorConfig.TABLE_BLACKLIST), tableIdMapper)
                                                   .build();
 

debezium-core/src/main/java/io/debezium/relational/SnapshotChangeRecordEmitter.java

@@ -3,11 +3,10 @@
  *
  * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
  */
-package io.debezium.connector.sqlserver;
+package io.debezium.relational;
 
 import io.debezium.data.Envelope.Operation;
 import io.debezium.pipeline.spi.OffsetContext;
-import io.debezium.relational.RelationalChangeRecordEmitter;
 import io.debezium.util.Clock;
 
 /**

debezium-core/src/main/java/io/debezium/relational/TableSchema.java

@@ -5,21 +5,20 @@
  */
 package io.debezium.relational;
 
+import java.util.Arrays;
 import java.util.Objects;
 import java.util.function.Function;
 
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.Struct;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import io.debezium.annotation.Immutable;
 import io.debezium.data.Envelope;
 import io.debezium.data.SchemaUtil;
 import io.debezium.schema.DataCollectionSchema;
 
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 /**
  * Defines the Kafka Connect {@link Schema} functionality associated with a given {@link Table table definition}, and which can
  * be used to send rows of data that match the table definition to Kafka Connect.
@@ -128,8 +127,7 @@ public Envelope getEnvelopeSchema() {
      */
     public Object keyFromColumnData(Object[] columnData) {
         if (logger.isTraceEnabled()) {
-            logger.trace("columnData from current stack: {}", columnData);
-            logger.trace("key from column data stack: ", new Throwable());
+            logger.trace("columnData from current stack: {}", Arrays.toString(columnData));
         }
         return columnData == null ? null : keyGenerator.apply(columnData);
     }

debezium-core/src/main/java/io/debezium/time/Conversions.java

@@ -246,4 +246,8 @@ public static Instant toInstant(long epochNanos) {
     public static Instant toInstantFromMicros(long epochMicros) {
         return toInstant(TimeUnit.MICROSECONDS.toNanos(epochMicros));
     }
+
+    public static Instant toInstantFromMillis(long epochMillis) {
+        return toInstant(TimeUnit.MILLISECONDS.toNanos(epochMillis));
+    }
 }

debezium-embedded/src/test/java/io/debezium/embedded/AbstractConnectorTest.java

@@ -8,7 +8,9 @@
 import static org.fest.assertions.Assertions.assertThat;
 import static org.junit.Assert.fail;
 
+import java.lang.management.ManagementFactory;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -27,6 +29,10 @@
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
+import javax.management.InstanceNotFoundException;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
 import org.apache.kafka.common.config.Config;
 import org.apache.kafka.common.config.ConfigValue;
 import org.apache.kafka.connect.data.Field;
@@ -44,6 +50,7 @@
 import org.apache.kafka.connect.storage.FileOffsetBackingStore;
 import org.apache.kafka.connect.storage.OffsetStorageReaderImpl;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.rules.TestRule;
@@ -59,7 +66,9 @@
 import io.debezium.junit.SkipTestRule;
 import io.debezium.junit.TestLogger;
 import io.debezium.relational.history.HistoryRecord;
+import io.debezium.util.Clock;
 import io.debezium.util.LoggingContext;
+import io.debezium.util.Metronome;
 import io.debezium.util.Testing;
 
 /**
@@ -290,10 +299,14 @@ public void taskStarted() {
                                        logger.error("Stopping connector after record as requested");
                                        throw new ConnectException("Stopping connector after record as requested");
                                    }
-                                   try {
-                                       consumedLines.put(record);
-                                   } catch (InterruptedException e) {
-                                       Thread.interrupted();
+                                   // Test stopped the connector, remaining records are ignored
+                                   if (!engine.isRunning() || Thread.currentThread().isInterrupted()) {
+                                       return;
+                                   }
+                                   while (!consumedLines.offer(record)) {
+                                       if (!engine.isRunning() || Thread.currentThread().isInterrupted()) {
+                                           return;
+                                       }
                                    }
                                })
                                .using(this.getClass().getClassLoader())
@@ -540,7 +553,7 @@ protected void assertConnectorIsRunning() {
      * Assert that the connector is NOT currently running.
      */
     protected void assertConnectorNotRunning() {
-        assertThat(engine.isRunning()).isFalse();
+        assertThat(engine != null && engine.isRunning()).isFalse();
     }
 
     /**
@@ -733,4 +746,54 @@ protected <T> Map<Map<String, T>, Map<String, Object>> readLastCommittedOffsets(
             offsetStore.stop();
         }
     }
+
+    public static void waitForSnapshotToBeCompleted(String connector, String server) throws InterruptedException {
+        int waitForSeconds = 60;
+        final MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
+        final Metronome metronome = Metronome.sleeper(Duration.ofSeconds(1), Clock.system());
+
+        while (true) {
+            if (waitForSeconds-- <= 0) {
+                Assert.fail("Snapshot was not completed on time");
+            }
+            try {
+                final boolean completed = (boolean) mbeanServer.getAttribute(new ObjectName("debezium." + connector + ":type=connector-metrics,context=snapshot,server=" + server), "SnapshotCompleted");
+                if (completed) {
+                    break;
+                }
+            }
+            catch (InstanceNotFoundException e) {
+                Testing.print("Metrics has not started yet");
+            }
+            catch (Exception e) {
+                throw new IllegalStateException(e);
+            }
+            metronome.pause();
+        }
+    }
+
+    public static void waitForStreamingRunning(String connector, String server) throws InterruptedException {
+        int waitForSeconds = 60;
+        final MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
+        final Metronome metronome = Metronome.sleeper(Duration.ofSeconds(1), Clock.system());
+
+        while (true) {
+            if (waitForSeconds-- <= 0) {
+                Assert.fail("Streaming was not started on time");
+            }
+            try {
+                final boolean completed = (boolean) mbeanServer.getAttribute(new ObjectName("debezium." + connector + ":type=connector-metrics,context=streaming,server=" + server), "Connected");
+                if (completed) {
+                    break;
+                }
+            }
+            catch (InstanceNotFoundException e) {
+                Testing.print("Metrics has not started yet");
+            }
+            catch (Exception e) {
+                throw new IllegalStateException(e);
+            }
+            metronome.pause();
+        }
+    }
 }