From 21e3c144912fb60041a2ee4b1e63d64d6169b2f6 Mon Sep 17 00:00:00 2001 From: jcechace Date: Fri, 14 Apr 2023 01:25:41 +0200 Subject: [PATCH] DBZ-6363 Support for MongoDB tests in UI build with Debezium 2.2 --- .../ConnectorConfiguration.java | 8 +++ .../testcontainers/MongoDbContainer.java | 56 +++++++++++++++---- .../testcontainers/MongoDbReplicaSet.java | 23 ++++++++ .../testcontainers/MongoDbShardedCluster.java | 12 ++++ 4 files changed, 87 insertions(+), 12 deletions(-) diff --git a/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/ConnectorConfiguration.java b/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/ConnectorConfiguration.java index 5c3fed686..c8152a201 100644 --- a/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/ConnectorConfiguration.java +++ b/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/ConnectorConfiguration.java @@ -74,6 +74,14 @@ public static ConnectorConfiguration forMongoDbContainer(MongoDbContainer mongoD return configuration; } + public static ConnectorConfiguration forMongoDbReplicaSet(MongoDbReplicaSet rs) { + ConnectorConfiguration configuration = new ConnectorConfiguration(); + configuration.with(CONNECTOR, "io.debezium.connector.mongodb.MongoDbConnector") + .with(CONNECTION_STRING, rs.getConnectionString()); + + return configuration; + } + private static boolean isMySQL(String driverClassName) { return "com.mysql.cj.jdbc.Driver".equals(driverClassName) || "com.mysql.jdbc.Driver".equals(driverClassName); } diff --git a/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbContainer.java b/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbContainer.java index 1a4eeca6c..1b5b439c9 100644 --- a/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbContainer.java +++ b/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbContainer.java @@ -12,18 +12,22 @@ import static java.util.stream.IntStream.range; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.function.Function; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.shaded.com.fasterxml.jackson.databind.JsonNode; import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.InspectContainerResponse; @@ -74,7 +78,9 @@ public static final class Builder { private boolean skipDockerDesktopLogWarning = false; public Builder imageName(DockerImageName imageName) { - this.imageName = imageName; + if (imageName != null) { + this.imageName = imageName; + } return this; } @@ -188,6 +194,23 @@ public void initReplicaSet(boolean configServer, Address... Addresses) { "]})"); } + /** + * Uploads given file to container and executes is as mongodb javascript + * + * @param file file to be uploaded + * @param containerPath path in the container + * @return execution result + */ + public Container.ExecResult execMongoScriptInContainer(MountableFile file, String containerPath) { + try { + copyFileToContainer(file, containerPath); + return execMongoInContainer(containerPath); + } + catch (IOException | InterruptedException e) { + throw new IllegalStateException(e); + } + } + /** * Invokes rs.stepDown on the * container to instruct the primary of the replica set to become the primary. @@ -228,24 +251,33 @@ private void dockerCommand(Function> action) { action.apply(DockerClientFactory.instance().client()).exec(); } - public JsonNode eval(String command) { + public Container.ExecResult execMongoInContainer(String... command) throws IOException, InterruptedException { checkStarted(); + // Support newer and older MongoDB versions respectively + var mongoCommand = Stream.concat( + Stream.of( + isLegacy() ? "" : "mongosh", + "mongo", + "--quiet", + "--host " + name, + "--port " + port), + Arrays.stream(command)).collect(joining(" ")); + LOGGER.debug("Running command inside container: {}", mongoCommand); + var result = execInContainer("sh", "-c", mongoCommand); + LOGGER.debug(result.getStdout()); + + checkExitCode(result); + return result; + } + + public JsonNode eval(String command) { try { - var mongoCommand = "mongo " + - "--quiet " + - "--host " + name + " " + - "--port " + port + " " + - "--eval \"JSON.stringify(" + command + ")\""; - LOGGER.debug("Running command inside container: {}", mongoCommand); - // Support newer and older MongoDB versions respectively - var result = execInContainer("sh", "-c", isLegacy() ? mongoCommand : "mongosh " + mongoCommand); - checkExitCode(result); + var result = execMongoInContainer("--eval", "\"JSON.stringify(" + command + ")\""); String stdout = result.getStdout(); var response = parseResponse(stdout); LOGGER.info("{}:", response); - return response; } catch (IOException | InterruptedException e) { diff --git a/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbReplicaSet.java b/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbReplicaSet.java index 454a726c6..a9ba86abb 100644 --- a/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbReplicaSet.java +++ b/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbReplicaSet.java @@ -24,9 +24,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; import org.testcontainers.containers.Network; import org.testcontainers.lifecycle.Startable; import org.testcontainers.shaded.com.fasterxml.jackson.databind.JsonNode; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; import io.debezium.testing.testcontainers.MongoDbContainer.Address; import io.debezium.testing.testcontainers.util.MoreStartables; @@ -47,6 +50,7 @@ public class MongoDbReplicaSet implements MongoDbDeployment { private final PortResolver portResolver; private final List members = new ArrayList<>(); + private final DockerImageName imageName; private boolean started; @@ -64,6 +68,12 @@ public static class Builder { private Network network = Network.newNetwork(); private PortResolver portResolver = new RandomPortResolver(); private boolean skipDockerDesktopLogWarning = false; + private DockerImageName imageName; + + public Builder imageName(DockerImageName imageName) { + this.imageName = imageName; + return this; + } public Builder name(String name) { this.name = name; @@ -111,6 +121,7 @@ private MongoDbReplicaSet(Builder builder) { this.configServer = builder.configServer; this.network = builder.network; this.portResolver = builder.portResolver; + this.imageName = builder.imageName; for (int i = 1; i <= memberCount; i++) { members.add(node() @@ -119,6 +130,7 @@ private MongoDbReplicaSet(Builder builder) { .replicaSet(name) .portResolver(portResolver) .skipDockerDesktopLogWarning(true) + .imageName(imageName) .build()); } @@ -201,6 +213,17 @@ private void initializeReplicaSet() { arbitraryNode.initReplicaSet(configServer, serverAddresses); } + /** + * Upload and executes mongodb javascript file against current primary + * + * See {@link MongoDbContainer#execMongoScriptInContainer(MountableFile, String)} + */ + public Container.ExecResult execMongoScript(MountableFile file, String containerPath) { + return tryPrimary() + .map(primary -> primary.execMongoScriptInContainer(file, containerPath)) + .orElseThrow(); + } + public void awaitReplicaPrimary() { await() .atMost(1, MINUTES) diff --git a/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbShardedCluster.java b/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbShardedCluster.java index 2f9dc8308..c90866671 100644 --- a/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbShardedCluster.java +++ b/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbShardedCluster.java @@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory; import org.testcontainers.containers.Network; import org.testcontainers.lifecycle.Startable; +import org.testcontainers.utility.DockerImageName; import io.debezium.testing.testcontainers.MongoDbContainer.Address; import io.debezium.testing.testcontainers.util.MoreStartables; @@ -46,6 +47,7 @@ public class MongoDbShardedCluster implements MongoDbDeployment { private final MongoDbReplicaSet configServers; private final List shards; private final List routers; + private final DockerImageName imageName; private volatile boolean started; @@ -62,6 +64,12 @@ public static class Builder { private Network network = Network.newNetwork(); private PortResolver portResolver = new RandomPortResolver(); private boolean skipDockerDesktopLogWarning = false; + private DockerImageName imageName; + + public Builder imageName(DockerImageName imageName) { + this.imageName = imageName; + return this; + } public Builder shardCount(int shardCount) { this.shardCount = shardCount; @@ -104,6 +112,7 @@ private MongoDbShardedCluster(Builder builder) { this.routerCount = builder.routerCount; this.network = builder.network; this.portResolver = builder.portResolver; + this.imageName = builder.imageName; this.shards = createShards(); this.configServers = createConfigServers(); @@ -189,6 +198,7 @@ private MongoDbReplicaSet createShard(int i) { .memberCount(replicaCount) .portResolver(portResolver) .skipDockerDesktopLogWarning(true) + .imageName(imageName) .build(); shard.getMembers().forEach(node -> node.setCommand( @@ -210,6 +220,7 @@ private MongoDbReplicaSet createConfigServers() { .portResolver(portResolver) .configServer(true) .skipDockerDesktopLogWarning(true) + .imageName(imageName) .build(); configServers.getMembers().forEach(node -> node.setCommand( @@ -234,6 +245,7 @@ private MongoDbContainer createRouter(Network network, int i) { .name("test-mongos" + i) .portResolver(portResolver) .skipDockerDesktopLogWarning(true) + .imageName(imageName) .build(); router.setCommand(