From e705ba3a3dce54d2fbbf880a5270e62abbb4b61d Mon Sep 17 00:00:00 2001 From: Martin Medek Date: Tue, 15 Nov 2022 20:46:11 +0100 Subject: [PATCH] DBZ-5785 Modify ocp system tests to archive test results and logs (#4039) DBZ-5785 Changed OCP testsuite job to pod. Added readiness probe to the pod description to indicate testsuite is finished. Modified the testsuite script to save logs and trigger the readiness probe once finished. Modified the OCP testsuite jenkins job to wait for the readiness probe and copy the testsuite logs. Added report portal post step to ocp system pipeline Moved script location in container from /testsuite to /root/testsuite --- .../docker/debezium-testing-system/Dockerfile | 9 +- .../deployment-template.sh | 93 ++++++++++--------- .../debezium-testing-system/testsuite.sh | 74 +++++++++------ jenkins-jobs/job-dsl/ocp_system_test.groovy | 2 +- .../pipelines/ocp_system_pipeline.groovy | 52 +++++++++-- 5 files changed, 143 insertions(+), 87 deletions(-) diff --git a/jenkins-jobs/docker/debezium-testing-system/Dockerfile b/jenkins-jobs/docker/debezium-testing-system/Dockerfile index 8a3dc18ef..6776c4ee6 100644 --- a/jenkins-jobs/docker/debezium-testing-system/Dockerfile +++ b/jenkins-jobs/docker/debezium-testing-system/Dockerfile @@ -1,6 +1,6 @@ FROM fedora:35 AS base -RUN dnf -y install git unzip zip findutils +RUN dnf -y install git unzip zip findutils rsync RUN curl --retry 7 -Lo /tmp/client-tools.tar.gz "https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/latest/openshift-client-linux.tar.gz" RUN tar zxf /tmp/client-tools.tar.gz -C /usr/local/bin oc \ @@ -12,7 +12,8 @@ RUN curl -s "https://get.sdkman.io" | bash RUN source $sdkman_init &&\ sdk install java 17.0.2-open -COPY testsuite.sh /testsuite/testsuite.sh -RUN chmod a+x /testsuite/testsuite.sh +RUN mkdir /root/testsuite +COPY testsuite.sh /root/testsuite/testsuite.sh +RUN chmod a+x /root/testsuite/testsuite.sh -ENTRYPOINT /testsuite/testsuite.sh +ENTRYPOINT /root/testsuite/testsuite.sh diff --git a/jenkins-jobs/docker/debezium-testing-system/deployment-template.sh b/jenkins-jobs/docker/debezium-testing-system/deployment-template.sh index 71fbb5307..a52453d0c 100755 --- a/jenkins-jobs/docker/debezium-testing-system/deployment-template.sh +++ b/jenkins-jobs/docker/debezium-testing-system/deployment-template.sh @@ -1,6 +1,6 @@ #!/bin/bash -OPTS=$(getopt -o h: --long dbz-git-repository:,dbz-git-branch:,filename:,pull-secret-name:,docker-tag:,project-name:,product-build:,strimzi-kc-build:,dbz-connect-image:,artifact-server-image:,apicurio-version:,kafka-version:,groups-arg:,help -n 'parse-options' -- "$@") +OPTS=$(getopt -o h: --long dbz-git-repository:,dbz-git-branch:,filename:,pull-secret-name:,docker-tag:,project-name:,product-build:,strimzi-kc-build:,dbz-connect-image:,artifact-server-image:,apicurio-version:,kafka-version:,groups-arg:,testsuite-log:,help -n 'parse-options' -- "$@") if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi eval set -- "$OPTS" @@ -19,61 +19,62 @@ while true; do --dbz-connect-image ) DBZ_CONNECT_IMAGE=$2; shift 2;; --artifact-server-image ) ARTIFACT_SERVER_IMAGE=$2; shift 2;; --apicurio-version ) APICURIO_VERSION=$2; shift 2;; - --kafka-version ) KAFKA_VERSION=$2; shift 2;; + --kafka-version ) KAFKA_VERSION=$2; shift 2;; --groups-arg ) GROUPS_ARG=$2; shift 2;; + --testsuite-log ) TESTSUITE_LOG=$2; shift 2;; -h | --help ) PRINT_HELP=true=$2; shift ;; -- ) shift; break ;; * ) break ;; esac done -output="apiVersion: batch/v1 -kind: Job +output="apiVersion: v1 +kind: Pod metadata: name: \"testsuite\" spec: - template: - metadata: - labels: - name: \"testsuite\" - spec: - restartPolicy: Never - imagePullSecrets: - - name: ${PULL_SECRET_NAME} - containers: - - name: \"dbz-testing-system\" - image: \"quay.io/rh_integration/dbz-testing-system:${DOCKER_TAG}\" - imagePullPolicy: Always - ports: - - containerPort: 8080 - protocol: \"TCP\" - env: - - name: DBZ_GIT_REPOSITORY - value: \"${DBZ_GIT_REPOSITORY}\" - - name: DBZ_GIT_BRANCH - value: \"${DBZ_GIT_BRANCH}\" - - name: DBZ_OCP_PROJECT_DEBEZIUM - value: \"${PROJECT_NAME}\" - - name: DBZ_SECRET_NAME - value: \"${PULL_SECRET_NAME}\" - - name: DBZ_TEST_WAIT_SCALE - value: \"10\" - - name: DBZ_PRODUCT_BUILD - value: \"${PRODUCT_BUILD}\" - - name: DBZ_STRIMZI_KC_BUILD - value: \"${STRIMZI_KC_BUILD}\" - - name: DBZ_CONNECT_IMAGE - value: \"${DBZ_CONNECT_IMAGE}\" - - name: DBZ_ARTIFACT_SERVER_IMAGE - value: \"${ARTIFACT_SERVER_IMAGE}\" - - name: DBZ_APICURIO_VERSION - value: \"${APICURIO_VERSION}\" - - name: DBZ_KAFKA_VERSION - value: \"${KAFKA_VERSION}\" - - name: DBZ_GROUPS_ARG - value: \"${GROUPS_ARG}\" - - name: DBZ_OCP_DELETE_PROJECTS - value: \"true\" + restartPolicy: Never + imagePullSecrets: + - name: ${PULL_SECRET_NAME} + containers: + - name: \"dbz-testing-system\" + image: \"quay.io/rh_integration/dbz-testing-system:${DOCKER_TAG}\" + imagePullPolicy: Always + ports: + - containerPort: 8080 + protocol: \"TCP\" + env: + - name: DBZ_GIT_REPOSITORY + value: \"${DBZ_GIT_REPOSITORY}\" + - name: DBZ_GIT_BRANCH + value: \"${DBZ_GIT_BRANCH}\" + - name: DBZ_OCP_PROJECT_DEBEZIUM + value: \"${PROJECT_NAME}\" + - name: DBZ_SECRET_NAME + value: \"${PULL_SECRET_NAME}\" + - name: DBZ_TEST_WAIT_SCALE + value: \"10\" + - name: DBZ_PRODUCT_BUILD + value: \"${PRODUCT_BUILD}\" + - name: DBZ_STRIMZI_KC_BUILD + value: \"${STRIMZI_KC_BUILD}\" + - name: DBZ_CONNECT_IMAGE + value: \"${DBZ_CONNECT_IMAGE}\" + - name: DBZ_ARTIFACT_SERVER_IMAGE + value: \"${ARTIFACT_SERVER_IMAGE}\" + - name: DBZ_APICURIO_VERSION + value: \"${APICURIO_VERSION}\" + - name: DBZ_KAFKA_VERSION + value: \"${KAFKA_VERSION}\" + - name: DBZ_GROUPS_ARG + value: \"${GROUPS_ARG}\" + - name: DBZ_OCP_DELETE_PROJECTS + value: \"true\" + readinessProbe: + exec: + command: + - cat + - ${TESTSUITE_LOG} triggers: - type: \"ConfigChange\" paused: false diff --git a/jenkins-jobs/docker/debezium-testing-system/testsuite.sh b/jenkins-jobs/docker/debezium-testing-system/testsuite.sh index d4a6ab83e..8db94d402 100644 --- a/jenkins-jobs/docker/debezium-testing-system/testsuite.sh +++ b/jenkins-jobs/docker/debezium-testing-system/testsuite.sh @@ -1,40 +1,56 @@ #!/bin/bash +SCRIPT_LOCATION=${HOME}/testsuite +cd "${SCRIPT_LOCATION}" || exit 1 +{ + source "${HOME}/.sdkman/bin/sdkman-init.sh" -source /root/.sdkman/bin/sdkman-init.sh + set -x -set -x + DEBEZIUM_LOCATION=${SCRIPT_LOCATION}/debezium -DEBEZIUM_LOCATION="/testsuite/debezium" + # copy secret to debezium project + TESTSUITE_SECRET=${PWD}/testsuite_secret.yml + oc get secret -n "${DBZ_OCP_PROJECT_DEBEZIUM}-testsuite" "${DBZ_SECRET_NAME}" -o yaml | sed "s/namespace: .*//" | sed "s/uid: .*//" > "${TESTSUITE_SECRET}" -# copy secret to debezium project -TESTSUITE_SECRET=/testsuite/testsuite_secret.yml -oc get secret -n "${DBZ_OCP_PROJECT_DEBEZIUM}-testsuite" "${DBZ_SECRET_NAME}" -o yaml | sed "s/namespace: .*//" | sed "s/uid: .*//" > ${TESTSUITE_SECRET} + mkdir "${DEBEZIUM_LOCATION}" -mkdir ${DEBEZIUM_LOCATION} -pushd /testsuite || exit 1; + OPTIONAL_ARGS=() + if [ "${DBZ_PRODUCT_BUILD}" == true ] ; then + OPTIONAL_ARGS+=("-Pproduct") + fi -OPTIONAL_ARGS=() -if [ "${DBZ_PRODUCT_BUILD}" == true ] ; then - OPTIONAL_ARGS+=("-Pproduct") -fi + if [ -n "${DBZ_KAFKA_VERSION}" ] ; then + OPTIONAL_ARGS+=("-Dversion.kafka=${DBZ_KAFKA_VERSION}") + fi -if [ -n "${DBZ_KAFKA_VERSION}" ] ; then - OPTIONAL_ARGS+=("-Dversion.kafka=${DBZ_KAFKA_VERSION}") -fi + # clone, compile debezium and run tests + git clone --branch "${DBZ_GIT_BRANCH}" "${DBZ_GIT_REPOSITORY}" + pushd "${DEBEZIUM_LOCATION}" || exit 1 -# clone, compile debezium and run tests -git clone --branch "${DBZ_GIT_BRANCH}" "${DBZ_GIT_REPOSITORY}" -pushd debezium || exit 1 + ./mvnw install -DskipTests -DskipITs -./mvnw install -DskipTests -DskipITs + ./mvnw install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \ + -Docp.project.debezium="${DBZ_OCP_PROJECT_DEBEZIUM}" \ + -Docp.pull.secret.paths="${TESTSUITE_SECRET}" \ + -Dtest.wait.scale="${DBZ_TEST_WAIT_SCALE}" \ + -Dtest.strimzi.kc.build="${DBZ_STRIMZI_KC_BUILD}" \ + -Dimage.kc="${DBZ_CONNECT_IMAGE}" \ + -Dimage.as="${DBZ_ARTIFACT_SERVER_IMAGE}" \ + -Das.apicurio.version="${DBZ_APICURIO_VERSION}" \ + -Dgroups="${DBZ_GROUPS_ARG}" \ + "${OPTIONAL_ARGS[@]}" -./mvnw install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \ - -Docp.project.debezium="${DBZ_OCP_PROJECT_DEBEZIUM}" \ - -Docp.pull.secret.paths="${TESTSUITE_SECRET}" \ - -Dtest.wait.scale="${DBZ_TEST_WAIT_SCALE}" \ - -Dtest.strimzi.kc.build="${DBZ_STRIMZI_KC_BUILD}" \ - -Dimage.kc="${DBZ_CONNECT_IMAGE}" \ - -Dimage.as="${DBZ_ARTIFACT_SERVER_IMAGE}" \ - -Das.apicurio.version="${DBZ_APICURIO_VERSION}" \ - -Dgroups="${DBZ_GROUPS_ARG}" \ - "${OPTIONAL_ARGS[@]}" + pushd debezium-testing/debezium-testing-system/target/failsafe-reports || exit 1 + zip artifacts ./*.xml + mv artifacts.zip "${SCRIPT_LOCATION}" +} 2>&1 | tee "${SCRIPT_LOCATION}/tmp_log" + +# move to a location checked by a readiness probe to signal jenkins artefacts are ready +cd "${SCRIPT_LOCATION}" || exit 1 +mv tmp_log testsuite_log + +# keep alive until test results and logs are copied and pod is killed by jenkins job. +while true; do + echo "Waiting to die." + sleep 60 +done diff --git a/jenkins-jobs/job-dsl/ocp_system_test.groovy b/jenkins-jobs/job-dsl/ocp_system_test.groovy index a49dfd150..1bf09e4ad 100644 --- a/jenkins-jobs/job-dsl/ocp_system_test.groovy +++ b/jenkins-jobs/job-dsl/ocp_system_test.groovy @@ -3,7 +3,7 @@ pipelineJob('ocp-system-test') { description('Runs System Tests inside OCP') parameters { - stringParam('MAIL_TO', 'jcechace@redhat.com') + stringParam('MAIL_TO', 'debezium-qe@redhat.com') booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?') stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned') diff --git a/jenkins-jobs/pipelines/ocp_system_pipeline.groovy b/jenkins-jobs/pipelines/ocp_system_pipeline.groovy index 57f22bfed..6d6dee4d9 100644 --- a/jenkins-jobs/pipelines/ocp_system_pipeline.groovy +++ b/jenkins-jobs/pipelines/ocp_system_pipeline.groovy @@ -193,10 +193,12 @@ pipeline { DBZ_GROUPS_ARG="${DBZ_GROUPS_ARG} & !avro" fi - JOB_DESC_FILE="testsuite-job.yml" + POD_DESCRIPTION="testsuite.yml" PULL_SECRET_NAME=$(cat ${SECRET_PATH} | grep name | awk '{print $2;}') - jenkins-jobs/docker/debezium-testing-system/deployment-template.sh --filename "${JOB_DESC_FILE}" \ + LOG_LOCATION_IN_POD=/root/testsuite/testsuite_log + + jenkins-jobs/docker/debezium-testing-system/deployment-template.sh --filename "${POD_DESCRIPTION}" \ --pull-secret-name "${PULL_SECRET_NAME}" \ --docker-tag "${DOCKER_TAG}" \ --project-name "${OCP_PROJECT_NAME}" \ @@ -208,14 +210,26 @@ pipeline { --dbz-connect-image "${DBZ_CONNECT_IMAGE}" \ --artifact-server-image "${ARTIFACT_SERVER_IMAGE}" \ --dbz-git-repository "${DBZ_GIT_REPOSITORY}" \ - --dbz-git-branch "${DBZ_GIT_BRANCH}" - - oc create -f "${JOB_DESC_FILE}" + --dbz-git-branch "${DBZ_GIT_BRANCH}" \ + --testsuite-log "${LOG_LOCATION_IN_POD}" + oc create -f "${POD_DESCRIPTION}" + pod_name=$(oc get pods | tail -1 | awk '{print $1;}') + + { + oc wait --timeout=10h --for=condition=Ready pod/${pod_name} + + # copy log and test results + mkdir ${WORKSPACE}/testsuite_artifacts + oc rsync ${pod_name}:${LOG_LOCATION_IN_POD} ${WORKSPACE}/testsuite_artifacts + oc rsync ${pod_name}:/root/testsuite/artifacts.zip ${WORKSPACE}/testsuite_artifacts || oc delete pod ${pod_name} + oc delete pod ${pod_name} + } & + + # wait for container to start and print logs for i in {1..100}; do sleep 2 - pod_name=$(oc get pods | tail -1 | awk '{print $1;}') - oc logs -f ${pod_name} && break + oc logs -f ${pod_name} && break done ''' } @@ -228,6 +242,30 @@ pipeline { cd ${DEBEZIUM_LOCATION} ./jenkins-jobs/scripts/ocp-projects.sh --delete --testsuite --project ${OCP_PROJECT_NAME} ''' + archiveArtifacts "**/testsuite_artifacts/*" + withCredentials([ + usernamePassword(credentialsId: "rh-integration-quay-creds", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'), + string(credentialsId: "report-portal-token", variable: 'RP_TOKEN'), + ]) { + sh ''' + if [ "${PRODUCT_BUILD}" == true ] ; then + export ATTRIBUTES="downstream ocp" + else + export ATTRIBUTES="upstream ocp" + fi + + cd ${WORKSPACE}/testsuite_artifacts + mkdir results + unzip artifacts.zip -d results + + RESULTS_FOLDER="." + rm -rf ${RESULTS_FOLDER}/failsafe-summary.xml + + docker login quay.io -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD" + + ${DEBEZIUM_LOCATION}/jenkins-jobs/scripts/report.sh --connector false --env-file env-file.env --results-folder ${RESULTS_FOLDER} --attributes "${ATTRIBUTES}" + ''' + } } } }