diff --git a/jenkins-jobs/docker/debezium-testing-system/deployment.yml b/jenkins-jobs/docker/debezium-testing-system/deployment.yml index 28e7377c5..8b45b4cc0 100644 --- a/jenkins-jobs/docker/debezium-testing-system/deployment.yml +++ b/jenkins-jobs/docker/debezium-testing-system/deployment.yml @@ -53,6 +53,18 @@ spec: value: "postgresql & !docker" - name: DELETE_PROJECTS value: "true" + - name: STRZ_GIT_REPOSITORY + value: "" + - name: STRZ_GIT_BRANCH + value: "" + - name: STRZ_DOWNSTREAM_URL + value: "" + - name: APIC_GIT_REPOSITORY + value: "" + - name: APIC_GIT_BRANCH + value: "" + - name: APIC_DOWNSTREAM_URL + value: "" triggers: - type: "ConfigChange" paused: false diff --git a/jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh b/jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh new file mode 100644 index 000000000..e920e1b62 --- /dev/null +++ b/jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +PULL_SECRET_NAME="aswerf" + +OPTS=$(getopt -o h: --long pull-secret-name:,docker-tag:,project-name:,product-build:,strimzi-kc-build:,dbz-connect-image:,artifact-server-image:,apicurio-version:,groups-arg:,help -n 'parse-options' -- "$@") +if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi +eval set -- "$OPTS" + +echo "${OPTS[@]}" + +while true; do + case "$1" in + --pull-secret-name ) PULL_SECRET_NAME=$2; shift 2;; + --docker-tag ) DOCKER_TAG=$2; shift 2;; + --project-name ) PROJECT_NAME=$2; shift 2;; + --product-build ) PRODUCT_BUILD=$2; shift 2;; + --strimzi-kc-build ) STRIMZI_KC_BUILD=$2; shift 2;; + --dbz-connect-image ) DBZ_CONNECT_IMAGE=$2; shift 2;; + --artifact-server-image ) ARTIFACT_SERVER_IMAGE=$2; shift 2;; + --apicurio-version ) APICURIO_VERSION=$2; shift 2;; + --groups-arg ) GROUPS_ARG=$2; shift 2;; + -h | --help ) PRINT_HELP=true shift ;; + -- ) shift; break ;; + * ) break ;; + esac +done + +output="apiVersion: batch/v1 +kind: Job +metadata: + name: \"testsuite\" +spec: + template: + metadata: + labels: + name: \"testsuite\" + spec: + restartPolicy: Never + imagePullSecrets: + - name: ${PULL_SECRET_NAME} + containers: + - name: \"dbz-testing-system\" + image: \"quay.io/rh_integration/dbz-testing-system:${DOCKER_TAG}\" + imagePullPolicy: Always + ports: + - containerPort: 8080 + protocol: \"TCP\" + env: + - name: OCP_PROJECT_DEBEZIUM + value: \"${PROJECT_NAME}\" + - name: OCP_PROJECT_DB2 + value: \"${PROJECT_NAME}-db2\" + - name: OCP_PROJECT_MONGO + value: \"${PROJECT_NAME}-mongo\" + - name: OCP_PROJECT_MYSQL + value: \"${PROJECT_NAME}-mysql\" + - name: OCP_PROJECT_ORACLE + value: \"${PROJECT_NAME}-oracle\" + - name: OCP_PROJECT_POSTGRESQL + value: \"${PROJECT_NAME}-postgresql\" + - name: OCP_PROJECT_SQLSERVER + value: \"${PROJECT_NAME}-sqlserver\" + - name: OCP_PROJECT_REGISTRY + value: \"${PROJECT_NAME}-registry\" + - name: SECRET_PATH + value: \"/testsuite/secret.yml\" + - name: TEST_WAIT_SCALE + value: \"10\" + - name: PRODUCT_BUILD + value: \"${PRODUCT_BUILD}\" + - name: STRIMZI_KC_BUILD + value: \"${STRIMZI_KC_BUILD}\" + - name: DBZ_CONNECT_IMAGE + value: \"${DBZ_CONNECT_IMAGE}\" + - name: ARTIFACT_SERVER_IMAGE + value: \"${ARTIFACT_SERVER_IMAGE}\" + - name: APICURIO_VERSION + value: \"${APICURIO_VERSION}\" + - name: GROUPS_ARG + value: \"${GROUPS_ARG}\" + - name: DELETE_PROJECTS + value: \"true\" + triggers: + - type: \"ConfigChange\" + paused: false + revisionHistoryLimit: 2 + minReadySeconds: 0 +" + + +printf "%s" "$output" >> test-job-deployment.yml \ No newline at end of file diff --git a/jenkins-jobs/docker/debezium-testing-system/library.sh b/jenkins-jobs/docker/debezium-testing-system/library.sh index c5b5bab6b..eed2e49d8 100644 --- a/jenkins-jobs/docker/debezium-testing-system/library.sh +++ b/jenkins-jobs/docker/debezium-testing-system/library.sh @@ -1,47 +1,37 @@ #!/bin/bash -clone_strimzi() +clone_component() { - local STRIMZI_REPO="https://github.com/strimzi/strimzi-kafka-operator.git" ; - local PRODUCT_BUILD=false ; - local STRIMZI_DOWNSTREAM_URL="" ; - local STRIMZI_BRANCH="main" ; - - OPTS=$(getopt -o --long strimzi-repository:,strimzi-branch:,product-build,strimzi-downstream:,product-build: -n 'clone-repositories' -- "$@") + OPTS=$(getopt -o c: --long component:,git-repository:,git-branch:,downstream-url:,product-build: -n 'clone-repositories' -- "$@") if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi eval set -- "$OPTS" while true; do case "$1" in - --strimzi-repository ) STRIMZI_REPO=$2; shift 2;; - --product-build ) PRODUCT_BUILD=$2; shift 2;; - --strimzi-downstream ) STRIMZI_DOWNSTREAM_URL=$2; shift 2;; - --strimzi-branch ) STRIMZI_BRANCH=$2; shift 2;; + -c | --component ) COMPONENT=$2; shift 2;; + --git-repository ) GIT_REPO=$2; shift 2;; + --git-branch ) GIT_BRANCH=$2; shift 2;; + --product-build ) PRODUCT_BUILD=$2; shift 2;; + --downstream-url ) DOWNSTREAM_URL=$2; shift 2;; -- ) shift; break ;; * ) break ;; esac done + if [ "${COMPONENT}" != "apicurio" ] && [ "${COMPONENT}" != "strimzi" ] ; + then + exit 2 ; + fi ; + if [ "${PRODUCT_BUILD}" = false ] ; then - # TODO why clone AND checkout? - git clone --branch "${STRIMZI_BRANCH}" "${STRIMZI_REPO}" strimzi && git -C strimzi checkout "${STRIMZI_BRANCH}" || exit 2 ; - elif [ -z "${STRIMZI_DOWNSTREAM_URL}" ] ; - then + git clone --branch "${GIT_BRANCH}" "${GIT_REPO}" "${COMPONENT}" || exit 2 ; + elif [ -z "${DOWNSTREAM_URL}" ] ; + then exit 2 ; - else - curl --retry 7 -Lo /tmp/strimzi_downstream.zip "${STRIMZI_DOWNSTREAM_URL}" ; - unzip /tmp/strimzi_downstream.zip -d strimzi ; - rm /tmp/strimzi_downstream.zip ; + else + curl --retry 7 -Lo /tmp/"${COMPONENT}".zip "${DOWNSTREAM_URL}" ; + unzip /tmp/"${COMPONENT}".zip -d "${COMPONENT}" ; + rm /tmp/"${COMPONENT}".zip ; fi ; } - -# TODO downstream apicurio? -clone_apicurio() -{ - local APICURIO_REPO="https://github.com/Apicurio/apicurio-registry-operator.git" ; - local APICURIO_BRANCH="master" ; - - # TODO why clone AND checkout? - git clone --branch "${APICURIO_BRANCH}" "${APICURIO_REPO}" apicurio && git -C apicurio checkout "${APICURIO_BRANCH}" || exit 2 ; -} diff --git a/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh b/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh index f4a1342db..cfae17cd2 100644 --- a/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh +++ b/jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh @@ -12,6 +12,7 @@ if [ ! -f "${SECRET_PATH}" ]; then exit 1 fi +# TODO remove git pull and rebuild once the development is done git -C /testsuite/debezium stash git -C /testsuite/debezium pull --rebase origin DBZ-5165 git -C /testsuite/debezium log -1 @@ -22,32 +23,44 @@ mvn clean install -DskipTests -DskipITs -f /testsuite/debezium/pom.xml ${OCP_PROJECTS} --project "${OCP_PROJECT_DEBEZIUM}" --create # prepare strimzi -clone_strimzi --strimzi-repository "${STRZ_GIT_REPOSITORY}" --strimzi-branch "${STRZ_GIT_BRANCH}" --product-build "${PRODUCT_BUILD}" --strimzi-downstream "${OCP_STRIMZI_DOWNSTREAM_URL}"; +# TODO remove the defaults once it's being ran from jenkins? +if [ -z "${STRZ_GIT_REPOSITORY}" ]; then + STRZ_GIT_REPOSITORY="https://github.com/strimzi/strimzi-kafka-operator.git" ; +fi + +if [ -z "${STRZ_GIT_BRANCH}" ]; then + STRZ_GIT_BRANCH="main" ; +fi + +clone_component --component strimzi --git-repository "${STRZ_GIT_REPOSITORY}" --git-branch "${STRZ_GIT_BRANCH}" --product-build "${PRODUCT_BUILD}" --downstream-url "${STRZ_DOWNSTREAM_URL}" ; sed -i 's/namespace: .*/namespace: '"${OCP_PROJECT_DEBEZIUM}"'/' strimzi/install/cluster-operator/*RoleBinding*.yaml ; oc create -f strimzi/install/cluster-operator/ -n "${OCP_PROJECT_DEBEZIUM}" ; -#prepare apicurio +# prepare apicurio if not disabled AVRO_PATTERN='.*!avro.*' if [[ ! ${GROUPS_ARG} =~ ${AVRO_PATTERN} ]]; then - APICURIO_RESOURCE="install/apicurio-registry-operator-1.1.0-dev.yaml" - clone_apicurio - sed -i "s/namespace: apicurio-registry-operator-namespace/namespace: ${OCP_PROJECT_REGISTRY}/" apicurio/install/*.yaml + # TODO remove the defaults once it's being ran from jenkins? + if [ -z "${APIC_GIT_REPOSITORY}" ]; then + APIC_GIT_REPOSITORY="https://github.com/Apicurio/apicurio-registry-operator.git" ; + fi + + if [ -z "${APIC_GIT_BRANCH}" ]; then + APIC_GIT_BRANCH="master" ; + fi + + if [ -z "${APICURIO_RESOURCE}" ]; then + APICURIO_RESOURCE="install/apicurio-registry-operator-1.1.0-dev.yaml" + fi + + clone_component --component apicurio --git-repository "${APIC_GIT_REPOSITORY}" --git-branch "${APIC_GIT_BRANCH}" --product-build "${PRODUCT_BUILD}" --downstream-url "${APIC_DOWNSTREAM_URL}" ; + sed -i "s/namespace: apicurio-registry-operator-namespace/namespace: ${OCP_PROJECT_REGISTRY}/" apicurio/install/*.yaml ; oc create -f apicurio/${APICURIO_RESOURCE} -n "${OCP_PROJECT_REGISTRY}" ; fi pushd ${DEBEZIUM_LOCATION} || exit 1; -if [ -z "${TEST_VERSION_KAFKA}" ]; then - TEST_PROPERTIES=""; -else - TEST_PROPERTIES="-Dversion.kafka=${TEST_VERSION_KAFKA}" ; -fi - -if [ -n "${DBZ_CONNECT_IMAGE}" ]; then - TEST_PROPERTIES="$TEST_PROPERTIES -Dimage.kc=${DBZ_CONNECT_IMAGE}" ; -fi - +oc project "${OCP_PROJECT_DEBEZIUM}" mvn install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \ -Docp.project.debezium="${OCP_PROJECT_DEBEZIUM}" \ -Docp.project.db2="${OCP_PROJECT_DB2}" \ diff --git a/jenkins-jobs/pipelines/ocp_system_pipeline.groovy b/jenkins-jobs/pipelines/ocp_system_pipeline.groovy new file mode 100644 index 000000000..1e4b7759e --- /dev/null +++ b/jenkins-jobs/pipelines/ocp_system_pipeline.groovy @@ -0,0 +1,44 @@ +pipeline { + agent { + label 'Slave' + } + + stages { + stage('Prepare project') { + steps { + withCredentials([ + usernamePassword(credentialsId: "${OCP_CREDENTIALS}", usernameVariable: 'OCP_USERNAME', passwordVariable: 'OCP_PASSWORD'), + file(credentialsId: "${PULL_SECRET}", variable: 'SECRET_PATH'), + ]) { + oc login -u ${OCP_USERNAME} -p ${OCP_PASSWORD} + oc new-project debezium-test-parent + oc adm policy add-cluster-role-to-user cluster-admin system:serviceaccount:debezium-test-parent:default + oc create -f ${SECRET_PATH} + } + } + } + + stage('Run tests') { + steps { + withCredentials([ + file(credentialsId: "${PULL_SECRET}", variable: 'SECRET_PATH'), + ]) { + sh ''' + debezium/jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh --pull-secret-name '${PULL_SECRET}' \ + --docker-tag "docker-test" \ + --project-name debezium-test \ + --product-build false \ + --strimzi-kc-build true \ + --dbz-connect-image "quay.io/rh_integration/test-strimzi-kafka:strz-latest-kafka-3.1.0-apc-2.2.3.Final-dbz-2.0.0-SNAPSHOT" \ + --artifact-server-image "quay.io/rh_integration/test-artifact-server:2.0.0-SNAPSHOT" \ + --apicurio-version "2.2.3.Final" \ + --groups-arg 'postgresql & !docker' + ''' + oc create -f test-job-deployment.yml + } + } + } + + } + +} \ No newline at end of file diff --git a/jenkins-jobs/pipelines/system_test_image_prepare_pipeline.groovy b/jenkins-jobs/pipelines/system_test_image_prepare_pipeline.groovy index 268a9227f..1864f7f04 100644 --- a/jenkins-jobs/pipelines/system_test_image_prepare_pipeline.groovy +++ b/jenkins-jobs/pipelines/system_test_image_prepare_pipeline.groovy @@ -39,6 +39,7 @@ pipeline { docker tag testsuite:docker-test quay.io/rh_integration/dbz-testing-system:${TAG} docker login -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} quay.io docker push quay.io/rh_integration/dbz-testing-system:${TAG} + rm debezium/jenkins-jobs/docker/debezium-testing-system/secret.yml ''' } }