DBZ-5785 Modify ocp system tests to archive test results and logs (#4039)

DBZ-5785 Changed OCP testsuite job to pod. Added readiness probe to the pod description to indicate testsuite is finished. Modified the testsuite script to save logs and trigger the readiness probe once finished. Modified the OCP testsuite jenkins job to wait for the readiness probe and copy the testsuite logs. Added report portal post step to ocp system pipeline Moved script location in container from /testsuite to /root/testsuite
2022-11-15 20:46:11 +01:00 · 2022-11-15 20:46:11 +01:00 · e705ba3a3d
commit e705ba3a3d
parent 8a10d7307c
5 changed files with 143 additions and 87 deletions
--- a/jenkins-jobs/docker/debezium-testing-system/Dockerfile
+++ b/jenkins-jobs/docker/debezium-testing-system/Dockerfile
@ -1,6 +1,6 @@
 FROM fedora:35 AS base

-RUN dnf -y install git unzip zip findutils
+RUN dnf -y install git unzip zip findutils rsync

 RUN curl --retry 7 -Lo /tmp/client-tools.tar.gz "https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/latest/openshift-client-linux.tar.gz"
 RUN tar zxf /tmp/client-tools.tar.gz -C /usr/local/bin oc \
@ -12,7 +12,8 @@ RUN curl -s "https://get.sdkman.io" | bash
 RUN source $sdkman_init &&\
    sdk install java 17.0.2-open

-COPY testsuite.sh /testsuite/testsuite.sh
-RUN chmod a+x /testsuite/testsuite.sh
+RUN mkdir /root/testsuite
+COPY testsuite.sh /root/testsuite/testsuite.sh
+RUN chmod a+x /root/testsuite/testsuite.sh

-ENTRYPOINT /testsuite/testsuite.sh
+ENTRYPOINT /root/testsuite/testsuite.sh
--- a/jenkins-jobs/docker/debezium-testing-system/deployment-template.sh
+++ b/jenkins-jobs/docker/debezium-testing-system/deployment-template.sh
@ -1,6 +1,6 @@
 #!/bin/bash

-OPTS=$(getopt -o h: --long dbz-git-repository:,dbz-git-branch:,filename:,pull-secret-name:,docker-tag:,project-name:,product-build:,strimzi-kc-build:,dbz-connect-image:,artifact-server-image:,apicurio-version:,kafka-version:,groups-arg:,help  -n 'parse-options' -- "$@")
+OPTS=$(getopt -o h: --long dbz-git-repository:,dbz-git-branch:,filename:,pull-secret-name:,docker-tag:,project-name:,product-build:,strimzi-kc-build:,dbz-connect-image:,artifact-server-image:,apicurio-version:,kafka-version:,groups-arg:,testsuite-log:,help  -n 'parse-options' -- "$@")
 if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi
 eval set -- "$OPTS"

@ -19,61 +19,62 @@ while true; do
        --dbz-connect-image )     DBZ_CONNECT_IMAGE=$2;       shift 2;;
        --artifact-server-image ) ARTIFACT_SERVER_IMAGE=$2;   shift 2;;
        --apicurio-version )      APICURIO_VERSION=$2;        shift 2;;
-        --kafka-version )         KAFKA_VERSION=$2;        shift 2;;
+        --kafka-version )         KAFKA_VERSION=$2;           shift 2;;
        --groups-arg )            GROUPS_ARG=$2;              shift 2;;
+        --testsuite-log )         TESTSUITE_LOG=$2;           shift 2;;
        -h | --help )             PRINT_HELP=true=$2;         shift ;;
        -- ) shift; break ;;
        * ) break ;;
    esac
 done

-output="apiVersion: batch/v1
-kind: Job
+output="apiVersion: v1
+kind: Pod
 metadata:
  name: \"testsuite\"
 spec:
-  template:
-    metadata:
-      labels:
-        name: \"testsuite\"
-    spec:
-      restartPolicy: Never
-      imagePullSecrets:
-        - name: ${PULL_SECRET_NAME}
-      containers:
-        - name: \"dbz-testing-system\"
-          image: \"quay.io/rh_integration/dbz-testing-system:${DOCKER_TAG}\"
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 8080
-              protocol: \"TCP\"
-          env:
-            - name: DBZ_GIT_REPOSITORY
-              value: \"${DBZ_GIT_REPOSITORY}\"
-            - name: DBZ_GIT_BRANCH
-              value: \"${DBZ_GIT_BRANCH}\"
-            - name: DBZ_OCP_PROJECT_DEBEZIUM
-              value: \"${PROJECT_NAME}\"
-            - name: DBZ_SECRET_NAME
-              value: \"${PULL_SECRET_NAME}\"
-            - name: DBZ_TEST_WAIT_SCALE
-              value: \"10\"
-            - name: DBZ_PRODUCT_BUILD
-              value: \"${PRODUCT_BUILD}\"
-            - name: DBZ_STRIMZI_KC_BUILD
-              value: \"${STRIMZI_KC_BUILD}\"
-            - name: DBZ_CONNECT_IMAGE
-              value: \"${DBZ_CONNECT_IMAGE}\"
-            - name: DBZ_ARTIFACT_SERVER_IMAGE
-              value: \"${ARTIFACT_SERVER_IMAGE}\"
-            - name: DBZ_APICURIO_VERSION
-              value: \"${APICURIO_VERSION}\"
-            - name: DBZ_KAFKA_VERSION
-              value: \"${KAFKA_VERSION}\"
-            - name: DBZ_GROUPS_ARG
-              value: \"${GROUPS_ARG}\"
-            - name: DBZ_OCP_DELETE_PROJECTS
-              value: \"true\"
+  restartPolicy: Never
+  imagePullSecrets:
+    - name: ${PULL_SECRET_NAME}
+  containers:
+    - name: \"dbz-testing-system\"
+      image: \"quay.io/rh_integration/dbz-testing-system:${DOCKER_TAG}\"
+      imagePullPolicy: Always
+      ports:
+        - containerPort: 8080
+          protocol: \"TCP\"
+      env:
+        - name: DBZ_GIT_REPOSITORY
+          value: \"${DBZ_GIT_REPOSITORY}\"
+        - name: DBZ_GIT_BRANCH
+          value: \"${DBZ_GIT_BRANCH}\"
+        - name: DBZ_OCP_PROJECT_DEBEZIUM
+          value: \"${PROJECT_NAME}\"
+        - name: DBZ_SECRET_NAME
+          value: \"${PULL_SECRET_NAME}\"
+        - name: DBZ_TEST_WAIT_SCALE
+          value: \"10\"
+        - name: DBZ_PRODUCT_BUILD
+          value: \"${PRODUCT_BUILD}\"
+        - name: DBZ_STRIMZI_KC_BUILD
+          value: \"${STRIMZI_KC_BUILD}\"
+        - name: DBZ_CONNECT_IMAGE
+          value: \"${DBZ_CONNECT_IMAGE}\"
+        - name: DBZ_ARTIFACT_SERVER_IMAGE
+          value: \"${ARTIFACT_SERVER_IMAGE}\"
+        - name: DBZ_APICURIO_VERSION
+          value: \"${APICURIO_VERSION}\"
+        - name: DBZ_KAFKA_VERSION
+          value: \"${KAFKA_VERSION}\"
+        - name: DBZ_GROUPS_ARG
+          value: \"${GROUPS_ARG}\"
+        - name: DBZ_OCP_DELETE_PROJECTS
+          value: \"true\"
+      readinessProbe:
+        exec:
+          command:
+          - cat
+          - ${TESTSUITE_LOG}
  triggers:
    - type: \"ConfigChange\"
  paused: false
--- a/jenkins-jobs/docker/debezium-testing-system/testsuite.sh
+++ b/jenkins-jobs/docker/debezium-testing-system/testsuite.sh
@ -1,40 +1,56 @@
 #!/bin/bash
+SCRIPT_LOCATION=${HOME}/testsuite
+cd "${SCRIPT_LOCATION}" || exit 1
+{
+  source "${HOME}/.sdkman/bin/sdkman-init.sh"

-source /root/.sdkman/bin/sdkman-init.sh
+  set -x

-set -x
+  DEBEZIUM_LOCATION=${SCRIPT_LOCATION}/debezium

-DEBEZIUM_LOCATION="/testsuite/debezium"
+  # copy secret to debezium project
+  TESTSUITE_SECRET=${PWD}/testsuite_secret.yml
+  oc get secret -n "${DBZ_OCP_PROJECT_DEBEZIUM}-testsuite" "${DBZ_SECRET_NAME}" -o yaml | sed "s/namespace: .*//" | sed "s/uid: .*//" > "${TESTSUITE_SECRET}"

-# copy secret to debezium project
-TESTSUITE_SECRET=/testsuite/testsuite_secret.yml
-oc get secret -n "${DBZ_OCP_PROJECT_DEBEZIUM}-testsuite" "${DBZ_SECRET_NAME}" -o yaml | sed "s/namespace: .*//" | sed "s/uid: .*//" > ${TESTSUITE_SECRET}
+  mkdir "${DEBEZIUM_LOCATION}"

-mkdir ${DEBEZIUM_LOCATION}
-pushd /testsuite || exit 1;
+  OPTIONAL_ARGS=()
+  if [ "${DBZ_PRODUCT_BUILD}" == true ] ; then
+    OPTIONAL_ARGS+=("-Pproduct")
+  fi

-OPTIONAL_ARGS=()
-if [ "${DBZ_PRODUCT_BUILD}" == true ] ; then
-  OPTIONAL_ARGS+=("-Pproduct")
-fi
+  if [ -n "${DBZ_KAFKA_VERSION}" ] ; then
+    OPTIONAL_ARGS+=("-Dversion.kafka=${DBZ_KAFKA_VERSION}")
+  fi

-if [ -n "${DBZ_KAFKA_VERSION}" ] ; then
-  OPTIONAL_ARGS+=("-Dversion.kafka=${DBZ_KAFKA_VERSION}")
-fi
+  # clone, compile debezium and run tests
+  git clone --branch "${DBZ_GIT_BRANCH}" "${DBZ_GIT_REPOSITORY}"
+  pushd "${DEBEZIUM_LOCATION}" || exit 1

-# clone, compile debezium and run tests
-git clone --branch "${DBZ_GIT_BRANCH}" "${DBZ_GIT_REPOSITORY}"
-pushd debezium || exit 1
+  ./mvnw install -DskipTests -DskipITs

-./mvnw install -DskipTests -DskipITs
+  ./mvnw install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \
+                      -Docp.project.debezium="${DBZ_OCP_PROJECT_DEBEZIUM}" \
+                      -Docp.pull.secret.paths="${TESTSUITE_SECRET}" \
+                      -Dtest.wait.scale="${DBZ_TEST_WAIT_SCALE}" \
+                      -Dtest.strimzi.kc.build="${DBZ_STRIMZI_KC_BUILD}" \
+                      -Dimage.kc="${DBZ_CONNECT_IMAGE}" \
+                      -Dimage.as="${DBZ_ARTIFACT_SERVER_IMAGE}" \
+                      -Das.apicurio.version="${DBZ_APICURIO_VERSION}" \
+                      -Dgroups="${DBZ_GROUPS_ARG}" \
+                      "${OPTIONAL_ARGS[@]}"

-./mvnw install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \
-                    -Docp.project.debezium="${DBZ_OCP_PROJECT_DEBEZIUM}" \
-                    -Docp.pull.secret.paths="${TESTSUITE_SECRET}" \
-                    -Dtest.wait.scale="${DBZ_TEST_WAIT_SCALE}" \
-                    -Dtest.strimzi.kc.build="${DBZ_STRIMZI_KC_BUILD}" \
-                    -Dimage.kc="${DBZ_CONNECT_IMAGE}" \
-                    -Dimage.as="${DBZ_ARTIFACT_SERVER_IMAGE}" \
-                    -Das.apicurio.version="${DBZ_APICURIO_VERSION}" \
-                    -Dgroups="${DBZ_GROUPS_ARG}" \
-                    "${OPTIONAL_ARGS[@]}"
+  pushd debezium-testing/debezium-testing-system/target/failsafe-reports || exit 1
+  zip artifacts ./*.xml
+  mv artifacts.zip "${SCRIPT_LOCATION}"
+} 2>&1 | tee "${SCRIPT_LOCATION}/tmp_log"
+
+# move to a location checked by a readiness probe to signal jenkins artefacts are ready
+cd "${SCRIPT_LOCATION}" || exit 1
+mv tmp_log testsuite_log
+
+# keep alive until test results and logs are copied and pod is killed by jenkins job.
+while true; do
+  echo "Waiting to die."
+  sleep 60
+done
--- a/jenkins-jobs/job-dsl/ocp_system_test.groovy
+++ b/jenkins-jobs/job-dsl/ocp_system_test.groovy
@ -3,7 +3,7 @@ pipelineJob('ocp-system-test') {
    description('Runs System Tests inside OCP')

    parameters {
-        stringParam('MAIL_TO', 'jcechace@redhat.com')
+        stringParam('MAIL_TO', 'debezium-qe@redhat.com')
        booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')

        stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned')
--- a/jenkins-jobs/pipelines/ocp_system_pipeline.groovy
+++ b/jenkins-jobs/pipelines/ocp_system_pipeline.groovy
@ -193,10 +193,12 @@ pipeline {
                        DBZ_GROUPS_ARG="${DBZ_GROUPS_ARG} & !avro"
                    fi

-                    JOB_DESC_FILE="testsuite-job.yml"
+                    POD_DESCRIPTION="testsuite.yml"
                    PULL_SECRET_NAME=$(cat ${SECRET_PATH} | grep name | awk '{print $2;}')

-                    jenkins-jobs/docker/debezium-testing-system/deployment-template.sh --filename "${JOB_DESC_FILE}" \
+                    LOG_LOCATION_IN_POD=/root/testsuite/testsuite_log
+
+                    jenkins-jobs/docker/debezium-testing-system/deployment-template.sh --filename "${POD_DESCRIPTION}" \
                        --pull-secret-name "${PULL_SECRET_NAME}" \
                        --docker-tag "${DOCKER_TAG}" \
                        --project-name "${OCP_PROJECT_NAME}" \
@ -208,14 +210,26 @@ pipeline {
                        --dbz-connect-image "${DBZ_CONNECT_IMAGE}" \
                        --artifact-server-image "${ARTIFACT_SERVER_IMAGE}" \
                        --dbz-git-repository "${DBZ_GIT_REPOSITORY}" \
-                        --dbz-git-branch "${DBZ_GIT_BRANCH}"
-
-                    oc create -f "${JOB_DESC_FILE}"
+                        --dbz-git-branch "${DBZ_GIT_BRANCH}" \
+                        --testsuite-log "${LOG_LOCATION_IN_POD}"

+                    oc create -f "${POD_DESCRIPTION}"
+                    pod_name=$(oc get pods | tail -1 | awk '{print $1;}')
+                    
+                    {
+                        oc wait --timeout=10h --for=condition=Ready pod/${pod_name}
+                        
+                        # copy log and test results                 
+                        mkdir ${WORKSPACE}/testsuite_artifacts
+                        oc rsync ${pod_name}:${LOG_LOCATION_IN_POD} ${WORKSPACE}/testsuite_artifacts
+                        oc rsync ${pod_name}:/root/testsuite/artifacts.zip ${WORKSPACE}/testsuite_artifacts || oc delete pod ${pod_name}
+                        oc delete pod ${pod_name}
+                    } & 
+                    
+                    # wait for container to start and print logs
                    for i in {1..100}; do
                        sleep 2
-                        pod_name=$(oc get pods | tail -1 | awk '{print $1;}')
-                        oc logs -f ${pod_name} && break
+                        oc logs -f ${pod_name}  && break
                    done
                    '''
                }
@ -228,6 +242,30 @@ pipeline {
            cd ${DEBEZIUM_LOCATION}
            ./jenkins-jobs/scripts/ocp-projects.sh --delete --testsuite --project ${OCP_PROJECT_NAME}
            '''
+            archiveArtifacts "**/testsuite_artifacts/*"
+            withCredentials([
+                    usernamePassword(credentialsId: "rh-integration-quay-creds", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'),
+                    string(credentialsId: "report-portal-token", variable: 'RP_TOKEN'),
+            ]) {
+                sh '''
+                if [ "${PRODUCT_BUILD}" == true ] ; then
+                    export ATTRIBUTES="downstream ocp"
+                else
+                    export ATTRIBUTES="upstream ocp"
+                fi
+
+                cd ${WORKSPACE}/testsuite_artifacts
+                mkdir results    
+                unzip artifacts.zip -d results
+                
+                RESULTS_FOLDER="."
+                rm -rf ${RESULTS_FOLDER}/failsafe-summary.xml            
+                
+                docker login quay.io -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD"
+
+                ${DEBEZIUM_LOCATION}/jenkins-jobs/scripts/report.sh --connector false --env-file env-file.env --results-folder ${RESULTS_FOLDER} --attributes "${ATTRIBUTES}"
+                '''
+            }
        }
    }
 }