diff --git a/jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh b/jenkins-jobs/docker/debezium-testing-system/deployment-template.sh old mode 100644 new mode 100755 similarity index 53% rename from jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh rename to jenkins-jobs/docker/debezium-testing-system/deployment-template.sh index e920e1b62..433aa7624 --- a/jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh +++ b/jenkins-jobs/docker/debezium-testing-system/deployment-template.sh @@ -1,8 +1,6 @@ #!/bin/bash -PULL_SECRET_NAME="aswerf" - -OPTS=$(getopt -o h: --long pull-secret-name:,docker-tag:,project-name:,product-build:,strimzi-kc-build:,dbz-connect-image:,artifact-server-image:,apicurio-version:,groups-arg:,help -n 'parse-options' -- "$@") +OPTS=$(getopt -o h: --long filename:,pull-secret-name:,docker-tag:,project-name:,product-build:,strimzi-kc-build:,dbz-connect-image:,artifact-server-image:,apicurio-version:,groups-arg:,strz-git-repository:,strz-git-branch:,strz-downstream-url:,apic-git-repository:,apic-git-branch:,apic-downstream-url:,help -n 'parse-options' -- "$@") if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi eval set -- "$OPTS" @@ -10,6 +8,7 @@ echo "${OPTS[@]}" while true; do case "$1" in + --filename ) FILENAME=$2; shift 2;; --pull-secret-name ) PULL_SECRET_NAME=$2; shift 2;; --docker-tag ) DOCKER_TAG=$2; shift 2;; --project-name ) PROJECT_NAME=$2; shift 2;; @@ -19,6 +18,12 @@ while true; do --artifact-server-image ) ARTIFACT_SERVER_IMAGE=$2; shift 2;; --apicurio-version ) APICURIO_VERSION=$2; shift 2;; --groups-arg ) GROUPS_ARG=$2; shift 2;; + --strz-git-repository ) STRZ_GIT_REPOSITORY shift 2;; + --strz-git-branch ) STRZ_GIT_BRANCH shift 2;; + --strz-downstream-url ) STRZ_DOWNSTREAM_URL shift 2;; + --apic-git-repository ) APIC_GIT_REPOSITORY shift 2;; + --apic-git-branch ) APIC_GIT_BRANCH shift 2;; + --apic-downstream-url ) APIC_DOWNSTREAM_URL shift 2;; -h | --help ) PRINT_HELP=true shift ;; -- ) shift; break ;; * ) break ;; @@ -46,40 +51,52 @@ spec: - containerPort: 8080 protocol: \"TCP\" env: - - name: OCP_PROJECT_DEBEZIUM + - name: DBZ_OCP_PROJECT_DEBEZIUM value: \"${PROJECT_NAME}\" - - name: OCP_PROJECT_DB2 + - name: DBZ_OCP_PROJECT_DB2 value: \"${PROJECT_NAME}-db2\" - - name: OCP_PROJECT_MONGO + - name: DBZ_OCP_PROJECT_MONGO value: \"${PROJECT_NAME}-mongo\" - - name: OCP_PROJECT_MYSQL + - name: DBZ_OCP_PROJECT_MYSQL value: \"${PROJECT_NAME}-mysql\" - - name: OCP_PROJECT_ORACLE + - name: DBZ_OCP_PROJECT_ORACLE value: \"${PROJECT_NAME}-oracle\" - - name: OCP_PROJECT_POSTGRESQL + - name: DBZ_OCP_PROJECT_POSTGRESQL value: \"${PROJECT_NAME}-postgresql\" - - name: OCP_PROJECT_SQLSERVER + - name: DBZ_OCP_PROJECT_SQLSERVER value: \"${PROJECT_NAME}-sqlserver\" - - name: OCP_PROJECT_REGISTRY + - name: DBZ_OCP_PROJECT_REGISTRY value: \"${PROJECT_NAME}-registry\" - - name: SECRET_PATH + - name: DBZ_SECRET_PATH value: \"/testsuite/secret.yml\" - - name: TEST_WAIT_SCALE + - name: DBZ_TEST_WAIT_SCALE value: \"10\" - - name: PRODUCT_BUILD + - name: DBZ_PRODUCT_BUILD value: \"${PRODUCT_BUILD}\" - - name: STRIMZI_KC_BUILD + - name: DBZ_STRIMZI_KC_BUILD value: \"${STRIMZI_KC_BUILD}\" - name: DBZ_CONNECT_IMAGE value: \"${DBZ_CONNECT_IMAGE}\" - - name: ARTIFACT_SERVER_IMAGE + - name: DBZ_ARTIFACT_SERVER_IMAGE value: \"${ARTIFACT_SERVER_IMAGE}\" - - name: APICURIO_VERSION + - name: DBZ_APICURIO_VERSION value: \"${APICURIO_VERSION}\" - - name: GROUPS_ARG + - name: DBZ_GROUPS_ARG value: \"${GROUPS_ARG}\" - - name: DELETE_PROJECTS + - name: DBZ_OCP_DELETE_PROJECTS value: \"true\" + - name: STRZ_GIT_REPOSITORY + value: \"${STRZ_GIT_REPOSITORY}\" + - name: STRZ_GIT_BRANCH + value: \"${STRZ_GIT_BRANCH}\" + - name: STRZ_DOWNSTREAM_URL + value: \"${STRZ_DOWNSTREAM_URL}\" + - name: APIC_GIT_REPOSITORY + value: \"${APIC_GIT_REPOSITORY}}\" + - name: APIC_GIT_BRANCH + value: \"${APIC_GIT_BRANCH}}\" + - name: APIC_DOWNSTREAM_URL + value: \"${APIC_DOWNSTREAM_URL}}\" triggers: - type: \"ConfigChange\" paused: false @@ -88,4 +105,4 @@ spec: " -printf "%s" "$output" >> test-job-deployment.yml \ No newline at end of file +printf "%s" "$output" >> "${FILENAME}.yml" \ No newline at end of file diff --git a/jenkins-jobs/docker/debezium-testing-system/deployment.yml b/jenkins-jobs/docker/debezium-testing-system/deployment.yml index 8b45b4cc0..a29246b63 100644 --- a/jenkins-jobs/docker/debezium-testing-system/deployment.yml +++ b/jenkins-jobs/docker/debezium-testing-system/deployment.yml @@ -19,39 +19,39 @@ spec: - containerPort: 8080 protocol: "TCP" env: - - name: OCP_PROJECT_DEBEZIUM + - name: DBZ_OCP_PROJECT_DEBEZIUM value: "debezium-test" - - name: OCP_PROJECT_DB2 + - name: DBZ_OCP_PROJECT_DB2 value: "debezium-test-db2" - - name: OCP_PROJECT_MONGO + - name: DBZ_OCP_PROJECT_MONGO value: "debezium-test-mongo" - - name: OCP_PROJECT_MYSQL + - name: DBZ_OCP_PROJECT_MYSQL value: "debezium-test-mysql" - - name: OCP_PROJECT_ORACLE + - name: DBZ_OCP_PROJECT_ORACLE value: "debezium-test-oracle" - - name: OCP_PROJECT_POSTGRESQL + - name: DBZ_OCP_PROJECT_POSTGRESQL value: "debezium-test-postgresql" - - name: OCP_PROJECT_SQLSERVER + - name: DBZ_OCP_PROJECT_SQLSERVER value: "debezium-test-sqlserver" - - name: OCP_PROJECT_REGISTRY + - name: DBZ_OCP_PROJECT_REGISTRY value: "debezium-test-registry" - - name: SECRET_PATH + - name: DBZ_SECRET_PATH value: "/testsuite/secret.yml" - - name: TEST_WAIT_SCALE + - name: DBZ_TEST_WAIT_SCALE value: "10" - - name: PRODUCT_BUILD + - name: DBZ_PRODUCT_BUILD value: "false" - - name: STRIMZI_KC_BUILD + - name: DBZ_STRIMZI_KC_BUILD value: "true" - name: DBZ_CONNECT_IMAGE value: "quay.io/rh_integration/test-strimzi-kafka:strz-latest-kafka-3.1.0-apc-2.2.3.Final-dbz-2.0.0-SNAPSHOT" - - name: ARTIFACT_SERVER_IMAGE + - name: DBZ_ARTIFACT_SERVER_IMAGE value: "quay.io/rh_integration/test-artifact-server:2.0.0-SNAPSHOT" - - name: APICURIO_VERSION + - name: DBZ_APICURIO_VERSION value: "2.2.3.Final" - - name: GROUPS_ARG + - name: DBZ_GROUPS_ARG value: "postgresql & !docker" - - name: DELETE_PROJECTS + - name: DBZ_OCP_DELETE_PROJECTS value: "true" - name: STRZ_GIT_REPOSITORY value: "" diff --git a/jenkins-jobs/docker/debezium-testing-system/secret.yml b/jenkins-jobs/docker/debezium-testing-system/secret.yml new file mode 100644 index 000000000..45bfaa8e0 --- /dev/null +++ b/jenkins-jobs/docker/debezium-testing-system/secret.yml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: mmedek-pull-secret +data: + .dockerconfigjson: ewogICJhdXRocyI6IHsKICAgICJxdWF5LmlvIjogewogICAgICAiYXV0aCI6ICJiVzFsWkdWck9rSlpkRFV4TkRsNU1FSnlXWE5IVjIwNFZHcGtUMHAxUW1aSlUzcGpOMGRQU0ZONWVIZGFiRWxqYjFacGJVTkliSGhOV1N0NmJtMXhWRkJwTVdkblZVdz0iLAogICAgICAiZW1haWwiOiAiIgogICAgfQogIH0KfQ== +type: kubernetes.io/dockerconfigjson \ No newline at end of file diff --git a/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh b/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh index cfae17cd2..861aa3270 100644 --- a/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh +++ b/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh @@ -5,10 +5,9 @@ source /testsuite/library.sh DEBEZIUM_LOCATION="/testsuite/debezium" OCP_PROJECTS="${DEBEZIUM_LOCATION}/jenkins-jobs/scripts/ocp-projects.sh" -SECRET_PATH=/testsuite/secret.yml -if [ ! -f "${SECRET_PATH}" ]; then - echo "secret ${SECRET_PATH} does not exist!" +if [ ! -f "${DBZ_SECRET_PATH}" ]; then + echo "secret ${DBZ_SECRET_PATH} does not exist!" exit 1 fi @@ -20,7 +19,7 @@ git -C /testsuite/debezium log -1 mvn clean install -DskipTests -DskipITs -f /testsuite/debezium/pom.xml # create projects -${OCP_PROJECTS} --project "${OCP_PROJECT_DEBEZIUM}" --create +${OCP_PROJECTS} --project "${DBZ_OCP_PROJECT_DEBEZIUM}" --create # prepare strimzi # TODO remove the defaults once it's being ran from jenkins? @@ -33,8 +32,8 @@ if [ -z "${STRZ_GIT_BRANCH}" ]; then fi clone_component --component strimzi --git-repository "${STRZ_GIT_REPOSITORY}" --git-branch "${STRZ_GIT_BRANCH}" --product-build "${PRODUCT_BUILD}" --downstream-url "${STRZ_DOWNSTREAM_URL}" ; -sed -i 's/namespace: .*/namespace: '"${OCP_PROJECT_DEBEZIUM}"'/' strimzi/install/cluster-operator/*RoleBinding*.yaml ; -oc create -f strimzi/install/cluster-operator/ -n "${OCP_PROJECT_DEBEZIUM}" ; +sed -i 's/namespace: .*/namespace: '"${DBZ_OCP_PROJECT_DEBEZIUM}"'/' strimzi/install/cluster-operator/*RoleBinding*.yaml ; +oc create -f strimzi/install/cluster-operator/ -n "${DBZ_OCP_PROJECT_DEBEZIUM}" ; # prepare apicurio if not disabled AVRO_PATTERN='.*!avro.*' @@ -60,29 +59,28 @@ fi pushd ${DEBEZIUM_LOCATION} || exit 1; -oc project "${OCP_PROJECT_DEBEZIUM}" mvn install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \ - -Docp.project.debezium="${OCP_PROJECT_DEBEZIUM}" \ - -Docp.project.db2="${OCP_PROJECT_DB2}" \ - -Docp.project.mongo="${OCP_PROJECT_MONGO}" \ - -Docp.project.mysql="${OCP_PROJECT_MYSQL}" \ - -Docp.project.oracle="${OCP_PROJECT_ORACLE}" \ - -Docp.project.postgresql="${OCP_PROJECT_POSTGRESQL}" \ - -Docp.project.sqlserver="${OCP_PROJECT_SQLSERVER}" \ - -Docp.project.registry="${OCP_PROJECT_REGISTRY}" \ - -Docp.pull.secret.paths="${SECRET_PATH}" \ - -Dtest.wait.scale="${TEST_WAIT_SCALE}" \ - -Dtest.strimzi.kc.build="${STRIMZI_KC_BUILD}" \ + -Docp.project.debezium="${DBZ_OCP_PROJECT_DEBEZIUM}" \ + -Docp.project.db2="${DBZ_OCP_PROJECT_DB2}" \ + -Docp.project.mongo="${DBZ_OCP_PROJECT_MONGO}" \ + -Docp.project.mysql="${DBZ_OCP_PROJECT_MYSQL}" \ + -Docp.project.oracle="${DBZ_OCP_PROJECT_ORACLE}" \ + -Docp.project.postgresql="${DBZ_OCP_PROJECT_POSTGRESQL}" \ + -Docp.project.sqlserver="${DBZ_OCP_PROJECT_SQLSERVER}" \ + -Docp.project.registry="${DBZ_OCP_PROJECT_REGISTRY}" \ + -Docp.pull.secret.paths="${DBZ_SECRET_PATH}" \ + -Dtest.wait.scale="${DBZ_TEST_WAIT_SCALE}" \ + -Dtest.strimzi.kc.build="${DBZ_STRIMZI_KC_BUILD}" \ -Dimage.kc="${DBZ_CONNECT_IMAGE}" \ - -Dimage.as="${ARTIFACT_SERVER_IMAGE}" \ - -Das.apicurio.version="${APICURIO_VERSION}" \ - -Dgroups="${GROUPS_ARG}" + -Dimage.as="${DBZ_ARTIFACT_SERVER_IMAGE}" \ + -Das.apicurio.version="${DBZ_APICURIO_VERSION}" \ + -Dgroups="${DBZ_GROUPS_ARG}" popd || exit 1; cp debezium/debezium-testing/debezium-testing-system/target/failsafe-reports/*.xml /testsuite/logs -if [ "${DELETE_PROJECTS}" = true ] ; +if [ "${DBZ_OCP_DELETE_PROJECTS}" = true ] ; then - ${OCP_PROJECTS} --project "${OCP_PROJECT_DEBEZIUM}" --delete + ${OCP_PROJECTS} --project "${DBZ_OCP_PROJECT_DEBEZIUM}" --delete fi ; diff --git a/jenkins-jobs/job-dsl/ocp_system_test.groovy b/jenkins-jobs/job-dsl/ocp_system_test.groovy new file mode 100644 index 000000000..15c41c008 --- /dev/null +++ b/jenkins-jobs/job-dsl/ocp_system_test.groovy @@ -0,0 +1,40 @@ +pipelineJob('ocp-system-test') { + displayName('System Tests inside OCP') + description('Runs System Tests inside OCP') + + parameters { + stringParam('MAIL_TO', 'jcechace@redhat.com') + booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?') + + stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned') + stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources') + + stringParam('PULL_SECRET', 'rh-integration-quay-secret', 'Quay.io OpenShift secret') + stringParam('DOCKER_TAG', 'latest', 'Docker image tag') + stringParam('OCP_PROJECT_NAME', 'debezium-test', 'OCP projects name') + stringParam('OCP_CREDENTIALS', 'openshift-dbz-creds', 'Jenkins credentials id') + stringParam('OCP_URL', '', 'Ocp url') + + booleanParam('TEST_APICURIO_REGISTRY', false, 'Run tests with Apicurio Registry and Avro serialization') + + booleanParam('STRIMZI_KC_BUILD', false, 'Use connect image instead of artifact server') + stringParam('DBZ_CONNECT_IMAGE', 'quay.io/rh_integration/test-strimzi-kafka:strz-latest-kafka-3.1.0-apc-2.2.3.Final-dbz-2.0.0-SNAPSHOT', 'Debezium connect image') + stringParam('ARTIFACT_SERVER_IMAGE', 'quay.io/rh_integration/test-artifact-server:2.0.0-SNAPSHOT', 'Artifact server image') + stringParam('APICURIO_VERSION', '2.2.3.Final', 'Apicurio version') + + stringParam('STRZ_GIT_REPOSITORY', 'https://github.com/strimzi/strimzi-kafka-operator.git', 'Repository from which Strimzi is cloned') + stringParam('STRZ_GIT_BRANCH', 'main', 'A branch/tag from which Strimzi is built') + stringParam('STRZ_DOWNSTREAM_URL', '', 'URL to productised strimzi sources') + + stringParam('APIC_GIT_REPOSITORY', 'https://github.com/Apicurio/apicurio-registry-operator.git', 'Repository from which Apicurio is cloned') + stringParam('APIC_GIT_BRANCH', 'main', 'A branch/tag from which Apicurio is built') + stringParam('APIC_DOWNSTREAM_URL', '', 'URL to productised apicurio sources') + } + + definition { + cps { + script(readFileFromWorkspace('jenkins-jobs/pipelines/ocp_system_pipeline.groovy')) + sandbox() + } + } +} \ No newline at end of file diff --git a/jenkins-jobs/pipelines/ocp_system_pipeline.groovy b/jenkins-jobs/pipelines/ocp_system_pipeline.groovy index 1e4b7759e..c5f4d7879 100644 --- a/jenkins-jobs/pipelines/ocp_system_pipeline.groovy +++ b/jenkins-jobs/pipelines/ocp_system_pipeline.groovy @@ -4,16 +4,30 @@ pipeline { } stages { + stage('Checkout - Debezium') { + steps { + checkout([ + $class : 'GitSCM', + branches : [[name: "${DBZ_GIT_BRANCH}"]], + userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]], + extensions : [[$class : 'RelativeTargetDirectory', + relativeTargetDir: 'debezium']], + ]) + } + } + stage('Prepare project') { steps { withCredentials([ usernamePassword(credentialsId: "${OCP_CREDENTIALS}", usernameVariable: 'OCP_USERNAME', passwordVariable: 'OCP_PASSWORD'), file(credentialsId: "${PULL_SECRET}", variable: 'SECRET_PATH'), ]) { - oc login -u ${OCP_USERNAME} -p ${OCP_PASSWORD} - oc new-project debezium-test-parent - oc adm policy add-cluster-role-to-user cluster-admin system:serviceaccount:debezium-test-parent:default - oc create -f ${SECRET_PATH} + sh ''' + oc login -u "${OCP_USERNAME}" -p "${OCP_PASSWORD}" --insecure-skip-tls-verify=true "${OCP_URL}" + oc new-project "${OCP_PROJECT_NAME}-parent" || oc project "${OCP_PROJECT_NAME}-parent" + oc adm policy add-cluster-role-to-user cluster-admin "system:serviceaccount:${OCP_PROJECT_NAME}-parent:default" + oc apply -f "${SECRET_PATH}" + ''' } } } @@ -24,17 +38,63 @@ pipeline { file(credentialsId: "${PULL_SECRET}", variable: 'SECRET_PATH'), ]) { sh ''' - debezium/jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh --pull-secret-name '${PULL_SECRET}' \ - --docker-tag "docker-test" \ - --project-name debezium-test \ + GROUPS_ARG="!docker" + if [ ${TEST_APICURIO_REGISTRY} == false ]; then + GROUPS_ARG="${GROUPS_ARG} & !avro" + fi + + FILENAME="testsuite-job" + + OPTIONAL_PARAMS="" + if [ ! -z "${DBZ_CONNECT_IMAGE}"]; then + OPTIONAL_PARAMS="$OPTIONAL_PARAMS --dbz-connect-image ${DBZ_CONNECT_IMAGE}" + fi + + if [ ! -z "${ARTIFACT_SERVER_IMAGE}"]; then + OPTIONAL_PARAMS="$OPTIONAL_PARAMS --artifact-server-image ${ARTIFACT_SERVER_IMAGE}" + fi + + if [ ! -z "${APICURIO_VERSION}"]; then + OPTIONAL_PARAMS="$OPTIONAL_PARAMS --apicurio-version ${APICURIO_VERSION}" + fi + + if [ ! -z "${STRZ_GIT_REPOSITORY}"]; then + OPTIONAL_PARAMS="$OPTIONAL_PARAMS --strz-git-repository ${STRZ_GIT_REPOSITORY}" + fi + + if [ ! -z "${STRZ_GIT_BRANCH}"]; then + OPTIONAL_PARAMS="$OPTIONAL_PARAMS --strz-git-branch ${STRZ_GIT_BRANCH}" + fi + + if [ ! -z "${STRZ_DOWNSTREAM_URL}"]; then + OPTIONAL_PARAMS="$OPTIONAL_PARAMS --strz-downstream-url ${STRZ_DOWNSTREAM_URL}" + fi + + if [ ! -z "${APIC_GIT_REPOSITORY}"]; then + OPTIONAL_PARAMS="$OPTIONAL_PARAMS --apic-git-repository ${APIC_GIT_REPOSITORY}" + fi + + if [ ! -z "${APIC_GIT_BRANCH}"]; then + OPTIONAL_PARAMS="$OPTIONAL_PARAMS --apic-git-branch ${APIC_GIT_BRANCH}" + fi + + if [ ! -z "${APIC_DOWNSTREAM_URL}"]; then + OPTIONAL_PARAMS="$OPTIONAL_PARAMS --apic-downstream-url ${APIC_DOWNSTREAM_URL}" + fi + + cd ${WORKSPACE}/debezium + jenkins-jobs/docker/debezium-testing-system/deployment-template.sh --filename "${FILENAME}" \ + --pull-secret-name '${PULL_SECRET}' \ + --docker-tag "${DOCKER_TAG}" \ + --project-name "${OCP_PROJECT_NAME}" \ --product-build false \ - --strimzi-kc-build true \ - --dbz-connect-image "quay.io/rh_integration/test-strimzi-kafka:strz-latest-kafka-3.1.0-apc-2.2.3.Final-dbz-2.0.0-SNAPSHOT" \ - --artifact-server-image "quay.io/rh_integration/test-artifact-server:2.0.0-SNAPSHOT" \ - --apicurio-version "2.2.3.Final" \ - --groups-arg 'postgresql & !docker' + --strimzi-kc-build ${STRIMZI_KC_BUILD} \ + --apicurio-version "${APICURIO_VERSION}" \ + --groups-arg "${GROUPS_ARG}" \ + ${OPTIONAL_PARAMS} + oc delete -f "${FILENAME}.yml" --ignore-not-found + oc create -f "${FILENAME}.yml" ''' - oc create -f test-job-deployment.yml } } }