DBZ-777 Fix pgoutput decoder

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresChangeRecordEmitter.java

@@ -18,6 +18,7 @@
 import io.debezium.connector.postgresql.connection.PostgresConnection;
 import io.debezium.connector.postgresql.connection.ReplicationMessage;
 import io.debezium.data.Envelope.Operation;
+import io.debezium.function.Predicates;
 import io.debezium.pipeline.spi.ChangeRecordEmitter;
 import io.debezium.pipeline.spi.OffsetContext;
 import io.debezium.relational.Column;
@@ -111,7 +112,7 @@ public DataCollectionSchema synchronizeTableSchema(DataCollectionSchema tableSch
         final boolean metadataInMessage = message.hasTypeMetadata();
         final TableId tableId = (TableId) tableSchema.id();
         final Table table = schema.tableFor(tableId);
-        if (getOperation() == Operation.DELETE || message.shouldSchemaBeSynchronized()) {
+        if (getOperation() == Operation.DELETE || !message.shouldSchemaBeSynchronized()) {
             return tableSchema;
         }
         final List<ReplicationMessage.Column> columns = message.getNewTupleList();
@@ -137,10 +138,12 @@ private Object[] columnValues(List<ReplicationMessage.Column> columns, TableId t
 
         // based on the schema columns, create the values on the same position as the columns
         List<Column> schemaColumns = table.columns();
+        // based on the replication message without toasted columns for now
+        List<ReplicationMessage.Column> columnsWithoutToasted = columns.stream().filter(Predicates.not(ReplicationMessage.Column::isToastedColumn)).collect(Collectors.toList());
         // JSON does not deliver a list of all columns for REPLICA IDENTITY DEFAULT
-        Object[] values = new Object[columns.size() < schemaColumns.size() ? schemaColumns.size() : columns.size()];
+        Object[] values = new Object[columnsWithoutToasted.size() < schemaColumns.size() ? schemaColumns.size() : columnsWithoutToasted.size()];
 
-        for (ReplicationMessage.Column column: columns) {
+        for (ReplicationMessage.Column column: columnsWithoutToasted) {
             //DBZ-298 Quoted column names will be sent like that in messages, but stored unquoted in the column names
             final String columnName = Strings.unquoteIdentifierPart(column.getName());
             final Column tableColumn = table.columnWithName(columnName);

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresStreamingChangeEventSource.java

@@ -96,6 +96,7 @@ public void execute(ChangeEventSourceContext context) throws InterruptedExceptio
                     if (message == null) {
                         LOGGER.trace("Received empty message");
                         lastCompletelyProcessedLsn = lsn;
+                        dispatcher.dispatchHeartbeatEvent(offsetContext);
                         return;
                     }
                     if (message.isLastEventForLsn()) {

debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputMessageDecoder.java

@@ -336,12 +336,11 @@ private void decodeInsert(ByteBuffer buffer, TypeRegistry typeRegistry, Replicat
 
         Optional<Table> resolvedTable = resolveRelation(relationId);
         if (!resolvedTable.isPresent()) {
-            return;
+            processor.process(null);
         }
-
+        else {
             Table table = resolvedTable.get();
             List<Column> columns = resolveColumnsFromStreamTupleData(buffer, typeRegistry, table);
-
             processor.process(new PgOutputReplicationMessage(
                     Operation.INSERT,
                     table.id().toDoubleQuotedString(),
@@ -350,6 +349,7 @@ private void decodeInsert(ByteBuffer buffer, TypeRegistry typeRegistry, Replicat
                     null,
                     columns));
         }
+    }
 
     /**
      * Callback handler for the 'U' update replication stream message.
@@ -365,9 +365,9 @@ private void decodeUpdate(ByteBuffer buffer, TypeRegistry typeRegistry, Replicat
 
         Optional<Table> resolvedTable = resolveRelation(relationId);
         if (!resolvedTable.isPresent()) {
-            return;
+            processor.process(null);
         }
-
+        else {
             Table table = resolvedTable.get();
 
             // When reading the tuple-type, we could get 3 different values, 'O', 'K', or 'N'.
@@ -386,7 +386,6 @@ private void decodeUpdate(ByteBuffer buffer, TypeRegistry typeRegistry, Replicat
             }
 
             List<Column> columns = resolveColumnsFromStreamTupleData(buffer, typeRegistry, table);
-
             processor.process(new PgOutputReplicationMessage(
                     Operation.UPDATE,
                     table.id().toDoubleQuotedString(),
@@ -395,6 +394,7 @@ private void decodeUpdate(ByteBuffer buffer, TypeRegistry typeRegistry, Replicat
                     oldColumns,
                     columns));
         }
+    }
 
     /**
      * Callback handler for the 'D' delete replication stream message.
@@ -412,9 +412,9 @@ private void decodeDelete(ByteBuffer buffer, TypeRegistry typeRegistry, Replicat
 
         Optional<Table> resolvedTable = resolveRelation(relationId);
         if (!resolvedTable.isPresent()) {
-            return;
+            processor.process(null);
         }
-
+        else {
             Table table = resolvedTable.get();
             List<Column> columns = resolveColumnsFromStreamTupleData(buffer, typeRegistry, table);
             processor.process(new PgOutputReplicationMessage(
@@ -425,6 +425,7 @@ private void decodeDelete(ByteBuffer buffer, TypeRegistry typeRegistry, Replicat
                     columns,
                     null));
         }
+    }
 
     /**
      * Resolves a given replication message relation identifier to a {@link Table}.