diff --git a/.gitignore b/.gitignore index e3f91faac..51923d0d9 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,3 @@ gen/ jenkins-jobs/docker/rhel_kafka/plugins jenkins-jobs/docker/artifact-server/plugins -/jenkins-jobs/docker/debezium-testing-system/secret.yml diff --git a/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/OpenShiftUtils.java b/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/OpenShiftUtils.java index c9db003c5..7ec343dad 100644 --- a/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/OpenShiftUtils.java +++ b/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/OpenShiftUtils.java @@ -253,9 +253,7 @@ public void waitForPods(String project, Map labels) { } public static boolean isRunningFromOcp() { - return ConfigProperties.OCP_URL.isEmpty() || - ConfigProperties.OCP_USERNAME.isEmpty() || - ConfigProperties.OCP_PASSWORD.isEmpty(); + return ConfigProperties.OCP_URL.isEmpty(); } /** diff --git a/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/databases/AbstractOcpDatabaseController.java b/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/databases/AbstractOcpDatabaseController.java index 424e7e1ac..ab9e1331d 100644 --- a/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/databases/AbstractOcpDatabaseController.java +++ b/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/databases/AbstractOcpDatabaseController.java @@ -99,7 +99,7 @@ public int getDatabasePort() { @Override public String getPublicDatabaseHostname() { if (isRunningFromOcp()) { - LOGGER.info("Running from OCP, using local database hostname"); + LOGGER.info("Running from OCP, using internal database hostname"); return getDatabaseHostname(); } awaitIngress(); @@ -110,7 +110,7 @@ public String getPublicDatabaseHostname() { @Override public int getPublicDatabasePort() { if (isRunningFromOcp()) { - LOGGER.info("Running from OCP, using local database port"); + LOGGER.info("Running from OCP, using internal database port"); return getDatabasePort(); } awaitIngress(); diff --git a/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/databases/AbstractOcpDatabaseDeployer.java b/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/databases/AbstractOcpDatabaseDeployer.java index d9355bf91..53d0125ff 100644 --- a/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/databases/AbstractOcpDatabaseDeployer.java +++ b/debezium-testing/debezium-testing-system/src/main/java/io/debezium/testing/system/tools/databases/AbstractOcpDatabaseDeployer.java @@ -139,7 +139,8 @@ public B withPullSecrets(String yamlPath) { } private boolean isLbService(String yamlPath) { - return yamlPath.contains("-lb."); + Service service = YAML.fromResource(yamlPath, Service.class); + return "LoadBalancer".equals(service.getSpec().getType()); } } } diff --git a/debezium-testing/debezium-testing-system/src/test/java/io/debezium/testing/system/fixtures/OcpClient.java b/debezium-testing/debezium-testing-system/src/test/java/io/debezium/testing/system/fixtures/OcpClient.java index df0aa0456..6f6ad1524 100644 --- a/debezium-testing/debezium-testing-system/src/test/java/io/debezium/testing/system/fixtures/OcpClient.java +++ b/debezium-testing/debezium-testing-system/src/test/java/io/debezium/testing/system/fixtures/OcpClient.java @@ -34,15 +34,14 @@ public OcpClient(@NotNull ExtensionContext.Store store) { public void setup() { ConfigBuilder configBuilder = new ConfigBuilder(); if (!isRunningFromOcp()) { + LOGGER.info("Running outside OCP, using OCP credentials passed from parameters"); configBuilder.withMasterUrl(ConfigProperties.OCP_URL.get()) .withUsername(ConfigProperties.OCP_USERNAME.get()) - .withPassword(ConfigProperties.OCP_PASSWORD.get()) - .withRequestRetryBackoffLimit(ConfigProperties.OCP_REQUEST_RETRY_BACKOFF_LIMIT) - .withTrustCerts(true); - } - else { - LOGGER.info("OCP credentials not provided, using default config."); + .withPassword(ConfigProperties.OCP_PASSWORD.get()); } + configBuilder.withRequestRetryBackoffLimit(ConfigProperties.OCP_REQUEST_RETRY_BACKOFF_LIMIT) + .withTrustCerts(true); + client = new DefaultOpenShiftClient(configBuilder.build()); store(OpenShiftClient.class, client); } diff --git a/jenkins-jobs/docker/debezium-testing-system/Dockerfile b/jenkins-jobs/docker/debezium-testing-system/Dockerfile index 2a2a9fb17..86a8969ee 100644 --- a/jenkins-jobs/docker/debezium-testing-system/Dockerfile +++ b/jenkins-jobs/docker/debezium-testing-system/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:35 +FROM fedora:35 AS base RUN dnf -y install git unzip zip findutils @@ -11,15 +11,14 @@ ARG sdkman_init="/root/.sdkman/bin/sdkman-init.sh" RUN curl -s "https://get.sdkman.io" | bash RUN source $sdkman_init &&\ - sdk install java 17.0.2-open&&\ - sdk install maven 3.8.4 + sdk install java 17.0.2-open # this runs when only when dbz repo/branch is changed ARG repository ARG branch RUN git clone --branch $branch $repository /testsuite/debezium -RUN source $sdkman_init && mvn clean install -DskipTests -DskipITs -f /testsuite/debezium/pom.xml +RUN source $sdkman_init && pushd /testsuite/debezium && ./mvnw clean install -DskipTests -DskipITs COPY testsuite-deployment.sh /testsuite/testsuite-deployment.sh RUN chmod a+x /testsuite/testsuite-deployment.sh @@ -27,3 +26,8 @@ RUN chmod a+x /testsuite/testsuite-deployment.sh COPY library.sh /testsuite/library.sh ENTRYPOINT /testsuite/testsuite-deployment.sh + +FROM base AS downstream + +COPY apicurio-registry-install-examples.zip /testsuite/apicurio.zip +COPY amq-streams-install-examples.zip /testsuite/strimzi.zip diff --git a/jenkins-jobs/docker/debezium-testing-system/downstream/Dockerfile b/jenkins-jobs/docker/debezium-testing-system/downstream/Dockerfile deleted file mode 100644 index 9515b8052..000000000 --- a/jenkins-jobs/docker/debezium-testing-system/downstream/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM testsuite-base:latest - -COPY apicurio-registry-install-examples.zip /testsuite/apicurio.zip -COPY amq-streams-install-examples.zip /testsuite/strimzi.zip diff --git a/jenkins-jobs/docker/debezium-testing-system/library.sh b/jenkins-jobs/docker/debezium-testing-system/library.sh index 7eb0bd38e..35e1ea6a5 100644 --- a/jenkins-jobs/docker/debezium-testing-system/library.sh +++ b/jenkins-jobs/docker/debezium-testing-system/library.sh @@ -19,6 +19,7 @@ clone_component() if [ "${COMPONENT}" != "apicurio" ] && [ "${COMPONENT}" != "strimzi" ] ; then + echo "unknown component: ${COMPONENT}" exit 2 ; fi ; diff --git a/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh b/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh index 40e545b67..3826dcc71 100644 --- a/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh +++ b/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh @@ -59,7 +59,7 @@ if [ -n "${DBZ_KAFKA_VERSION}" ] ; then OPTIONAL_ARGS+=("-Dversion.kafka=${DBZ_KAFKA_VERSION}") fi -mvn install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \ +./mvnw install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \ -Docp.project.debezium="${DBZ_OCP_PROJECT_DEBEZIUM}" \ -Docp.project.db2="${DBZ_OCP_PROJECT_DB2}" \ -Docp.project.mongo="${DBZ_OCP_PROJECT_MONGO}" \ diff --git a/jenkins-jobs/job-dsl/ocp_downstream_system_test_image_prepare.groovy b/jenkins-jobs/job-dsl/ocp_downstream_system_test_image_prepare.groovy index e72e602f7..ef6f5e609 100644 --- a/jenkins-jobs/job-dsl/ocp_downstream_system_test_image_prepare.groovy +++ b/jenkins-jobs/job-dsl/ocp_downstream_system_test_image_prepare.groovy @@ -11,8 +11,6 @@ pipelineJob('ocp-debezium-testing-downstream-system-image-prepare') { } parameters { - stringParam('PARENT_DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository used to build the docker image') - stringParam('PARENT_DBZ_GIT_BRANCH', 'main') stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned into docker image') stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources') stringParam('TAG', 'latest', 'Docker image tag') diff --git a/jenkins-jobs/job-dsl/ocp_system_test.groovy b/jenkins-jobs/job-dsl/ocp_system_test.groovy index 021893cde..27a6ea213 100644 --- a/jenkins-jobs/job-dsl/ocp_system_test.groovy +++ b/jenkins-jobs/job-dsl/ocp_system_test.groovy @@ -19,8 +19,8 @@ pipelineJob('ocp-system-test') { booleanParam('TEST_APICURIO_REGISTRY', false, 'Run tests with Apicurio Registry and Avro serialization') booleanParam('STRIMZI_KC_BUILD', false, 'True -> use artifact server, false -> dbz connect image') - stringParam('DBZ_CONNECT_IMAGE', 'quay.io/rh_integration/test-strimzi-kafka:strz-latest-kafka-3.1.0-apc-2.2.3.Final-dbz-2.0.0-SNAPSHOT', 'Debezium connect image') - stringParam('ARTIFACT_SERVER_IMAGE', 'quay.io/rh_integration/test-artifact-server:2.0.0-SNAPSHOT', 'Artifact server image') + stringParam('DBZ_CONNECT_IMAGE', '', 'Debezium connect image') + stringParam('ARTIFACT_SERVER_IMAGE', '', 'Artifact server image') stringParam('APICURIO_VERSION', '2.2.3.Final', 'Apicurio version') stringParam('KAFKA_VERSION', '', 'Kafka version') diff --git a/jenkins-jobs/job-dsl/ocp_upstream_system_test_image_prepare.groovy b/jenkins-jobs/job-dsl/ocp_upstream_system_test_image_prepare.groovy index 283562bda..ff416919f 100644 --- a/jenkins-jobs/job-dsl/ocp_upstream_system_test_image_prepare.groovy +++ b/jenkins-jobs/job-dsl/ocp_upstream_system_test_image_prepare.groovy @@ -11,8 +11,6 @@ pipelineJob('ocp-debezium-testing-upstream-system-image-prepare') { } parameters { - stringParam('PARENT_DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository used to build the docker image') - stringParam('PARENT_DBZ_GIT_BRANCH', 'main') stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned into docker image') stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources') stringParam('TAG', 'latest', 'Docker image tag') diff --git a/jenkins-jobs/pipelines/downstream_system_test_image_pipeline.groovy b/jenkins-jobs/pipelines/downstream_system_test_image_pipeline.groovy index 62762068c..536944e97 100644 --- a/jenkins-jobs/pipelines/downstream_system_test_image_pipeline.groovy +++ b/jenkins-jobs/pipelines/downstream_system_test_image_pipeline.groovy @@ -14,8 +14,8 @@ pipeline { steps { checkout([ $class : 'GitSCM', - branches : [[name: "${PARENT_DBZ_GIT_BRANCH}"]], - userRemoteConfigs: [[url: "${PARENT_DBZ_GIT_REPOSITORY}"]], + branches : [[name: "${DBZ_GIT_BRANCH}"]], + userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]], extensions : [[$class : 'RelativeTargetDirectory', relativeTargetDir: 'debezium']], ]) @@ -30,7 +30,7 @@ pipeline { expression { !params.APICURIO_PREPARE_BUILD_NUMBER } } steps { - copyArtifacts projectName: 'ocp-downstream-apicurio-prepare-job', target: 'debezium/jenkins-jobs/docker/debezium-testing-system/downstream' ,filter: 'apicurio-registry-install-examples.zip', selector: lastSuccessful() + copyArtifacts projectName: 'ocp-downstream-apicurio-prepare-job', target: 'debezium/jenkins-jobs/docker/debezium-testing-system' ,filter: 'apicurio-registry-install-examples.zip', selector: lastSuccessful() } } stage('Copy apicurio artifacts') { @@ -38,7 +38,7 @@ pipeline { expression { params.APICURIO_PREPARE_BUILD_NUMBER } } steps { - copyArtifacts projectName: 'ocp-downstream-apicurio-prepare-job', target: 'debezium/jenkins-jobs/docker/debezium-testing-system/downstream' , filter: 'apicurio-registry-install-examples.zip', selector: specific(params.APICURIO_PREPARE_BUILD_NUMBER) + copyArtifacts projectName: 'ocp-downstream-apicurio-prepare-job', target: 'debezium/jenkins-jobs/docker/debezium-testing-system' , filter: 'apicurio-registry-install-examples.zip', selector: specific(params.APICURIO_PREPARE_BUILD_NUMBER) } } @@ -47,7 +47,7 @@ pipeline { expression { !params.STRIMZI_PREPARE_BUILD_NUMBER } } steps { - copyArtifacts projectName: 'ocp-downstream-strimzi-prepare-job', target: 'debezium/jenkins-jobs/docker/debezium-testing-system/downstream' , filter: 'amq-streams-install-examples.zip', selector: lastSuccessful() + copyArtifacts projectName: 'ocp-downstream-strimzi-prepare-job', target: 'debezium/jenkins-jobs/docker/debezium-testing-system' , filter: 'amq-streams-install-examples.zip', selector: lastSuccessful() } } stage('Copy strimzi artifacts') { @@ -55,7 +55,7 @@ pipeline { expression { params.STRIMZI_PREPARE_BUILD_NUMBER } } steps { - copyArtifacts projectName: 'ocp-downstream-strimzi-prepare-job', target: 'debezium/jenkins-jobs/docker/debezium-testing-system/downstream' , filter: 'amq-streams-install-examples.zip', selector: specific(params.STRIMZI_PREPARE_BUILD_NUMBER) + copyArtifacts projectName: 'ocp-downstream-strimzi-prepare-job', target: 'debezium/jenkins-jobs/docker/debezium-testing-system' , filter: 'amq-streams-install-examples.zip', selector: specific(params.STRIMZI_PREPARE_BUILD_NUMBER) } } @@ -67,13 +67,10 @@ pipeline { sh ''' cd debezium/jenkins-jobs/docker/debezium-testing-system - docker build --build-arg branch=${DBZ_GIT_BRANCH} --build-arg repository=${DBZ_GIT_REPOSITORY} -t testsuite-base:latest . - - cd downstream - docker build -t testsuite:latest . - docker tag testsuite:latest quay.io/rh_integration/dbz-testing-system:${TAG} + DOCKER_IMAGE=quay.io/rh_integration/dbz-testing-system:${TAG} + docker build --build-arg branch=${DBZ_GIT_BRANCH} --build-arg repository=${DBZ_GIT_REPOSITORY} --target downstream -t ${DOCKER_IMAGE} . docker login -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} quay.io - docker push quay.io/rh_integration/dbz-testing-system:${TAG} + docker push ${DOCKER_IMAGE} ''' } } diff --git a/jenkins-jobs/pipelines/ocp_system_pipeline.groovy b/jenkins-jobs/pipelines/ocp_system_pipeline.groovy index f5c3e7a5a..aafd1383d 100644 --- a/jenkins-jobs/pipelines/ocp_system_pipeline.groovy +++ b/jenkins-jobs/pipelines/ocp_system_pipeline.groovy @@ -44,7 +44,7 @@ pipeline { fi FILENAME="testsuite-job" - SECRET_NAME=$(cat ${SECRET_PATH} | grep name | awk '{print $2;}') + PULL_SECRET_NAME=$(cat ${SECRET_PATH} | grep name | awk '{print $2;}') cd ${WORKSPACE}/debezium jenkins-jobs/docker/debezium-testing-system/deployment-template.sh --filename "${FILENAME}" \ diff --git a/jenkins-jobs/pipelines/upstream_system_test_image_pipeline.groovy b/jenkins-jobs/pipelines/upstream_system_test_image_pipeline.groovy index a1f3ad2de..709485a39 100644 --- a/jenkins-jobs/pipelines/upstream_system_test_image_pipeline.groovy +++ b/jenkins-jobs/pipelines/upstream_system_test_image_pipeline.groovy @@ -14,8 +14,8 @@ pipeline { steps { checkout([ $class : 'GitSCM', - branches : [[name: "${PARENT_DBZ_GIT_BRANCH}"]], - userRemoteConfigs: [[url: "${PARENT_DBZ_GIT_REPOSITORY}"]], + branches : [[name: "${DBZ_GIT_BRANCH}"]], + userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]], extensions : [[$class : 'RelativeTargetDirectory', relativeTargetDir: 'debezium']], ]) @@ -32,10 +32,10 @@ pipeline { ]) { sh ''' pushd debezium/jenkins-jobs/docker/debezium-testing-system - docker build --build-arg branch=${DBZ_GIT_BRANCH} --build-arg repository=${DBZ_GIT_REPOSITORY} -t testsuite:upstream . - docker tag testsuite:upstream quay.io/rh_integration/dbz-testing-system:${TAG} + DOCKER_IMAGE=quay.io/rh_integration/dbz-testing-system:${TAG} + docker build --build-arg branch=${DBZ_GIT_BRANCH} --build-arg repository=${DBZ_GIT_REPOSITORY} --target base -t ${DOCKER_IMAGE} . docker login -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} quay.io - docker push quay.io/rh_integration/dbz-testing-system:${TAG} + docker push ${DOCKER_IMAGE} ''' } }