DBZ-5165 refactoring the testsuite scripts

jenkins-jobs/docker/debezium-testing-system/deployment.yml

@@ -53,6 +53,18 @@ spec:
               value: "postgresql & !docker"
             - name: DELETE_PROJECTS
               value: "true"
+            - name: STRZ_GIT_REPOSITORY
+              value: ""
+            - name: STRZ_GIT_BRANCH
+              value: ""
+            - name: STRZ_DOWNSTREAM_URL
+              value: ""
+            - name: APIC_GIT_REPOSITORY
+              value: ""
+            - name: APIC_GIT_BRANCH
+              value: ""
+            - name: APIC_DOWNSTREAM_URL
+              value: ""
   triggers:
     - type: "ConfigChange" 
   paused: false 

jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+PULL_SECRET_NAME="aswerf"
+
+OPTS=$(getopt -o h: --long pull-secret-name:,docker-tag:,project-name:,product-build:,strimzi-kc-build:,dbz-connect-image:,artifact-server-image:,apicurio-version:,groups-arg:,help  -n 'parse-options' -- "$@")
+if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi
+eval set -- "$OPTS"
+
+echo "${OPTS[@]}"
+
+while true; do
+    case "$1" in
+        --pull-secret-name )      PULL_SECRET_NAME=$2;        shift 2;;
+        --docker-tag )            DOCKER_TAG=$2;              shift 2;;
+        --project-name )          PROJECT_NAME=$2;            shift 2;;
+        --product-build )         PRODUCT_BUILD=$2;           shift 2;;
+        --strimzi-kc-build )      STRIMZI_KC_BUILD=$2;        shift 2;;
+        --dbz-connect-image )     DBZ_CONNECT_IMAGE=$2;       shift 2;;
+        --artifact-server-image ) ARTIFACT_SERVER_IMAGE=$2;   shift 2;;
+        --apicurio-version )      APICURIO_VERSION=$2;        shift 2;;
+        --groups-arg )            GROUPS_ARG=$2;              shift 2;;
+        -h | --help )             PRINT_HELP=true             shift ;;
+        -- ) shift; break ;;
+        * ) break ;;
+    esac
+done
+
+output="apiVersion: batch/v1
+kind: Job
+metadata:
+  name: \"testsuite\"
+spec:
+  template:
+    metadata:
+      labels:
+        name: \"testsuite\"
+    spec:
+      restartPolicy: Never
+      imagePullSecrets:
+        - name: ${PULL_SECRET_NAME}
+      containers:
+        - name: \"dbz-testing-system\"
+          image: \"quay.io/rh_integration/dbz-testing-system:${DOCKER_TAG}\"
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 8080
+              protocol: \"TCP\"
+          env:
+            - name: OCP_PROJECT_DEBEZIUM
+              value: \"${PROJECT_NAME}\"
+            - name: OCP_PROJECT_DB2
+              value: \"${PROJECT_NAME}-db2\"
+            - name: OCP_PROJECT_MONGO
+              value: \"${PROJECT_NAME}-mongo\"
+            - name: OCP_PROJECT_MYSQL
+              value: \"${PROJECT_NAME}-mysql\"
+            - name: OCP_PROJECT_ORACLE
+              value: \"${PROJECT_NAME}-oracle\"
+            - name: OCP_PROJECT_POSTGRESQL
+              value: \"${PROJECT_NAME}-postgresql\"
+            - name: OCP_PROJECT_SQLSERVER
+              value: \"${PROJECT_NAME}-sqlserver\"
+            - name: OCP_PROJECT_REGISTRY
+              value: \"${PROJECT_NAME}-registry\"
+            - name: SECRET_PATH
+              value: \"/testsuite/secret.yml\"
+            - name: TEST_WAIT_SCALE
+              value: \"10\"
+            - name: PRODUCT_BUILD
+              value: \"${PRODUCT_BUILD}\"
+            - name: STRIMZI_KC_BUILD
+              value: \"${STRIMZI_KC_BUILD}\"
+            - name: DBZ_CONNECT_IMAGE
+              value: \"${DBZ_CONNECT_IMAGE}\"
+            - name: ARTIFACT_SERVER_IMAGE
+              value: \"${ARTIFACT_SERVER_IMAGE}\"
+            - name: APICURIO_VERSION
+              value: \"${APICURIO_VERSION}\"
+            - name: GROUPS_ARG
+              value: \"${GROUPS_ARG}\"
+            - name: DELETE_PROJECTS
+              value: \"true\"
+  triggers:
+    - type: \"ConfigChange\"
+  paused: false
+  revisionHistoryLimit: 2
+  minReadySeconds: 0
+"
+
+
+printf "%s" "$output" >> test-job-deployment.yml

jenkins-jobs/docker/debezium-testing-system/library.sh

@@ -1,47 +1,37 @@
 #!/bin/bash
 
-clone_strimzi()
+clone_component()
 {
-    local STRIMZI_REPO="https://github.com/strimzi/strimzi-kafka-operator.git" ;
+    OPTS=$(getopt -o c: --long component:,git-repository:,git-branch:,downstream-url:,product-build:  -n 'clone-repositories' -- "$@")
-    local PRODUCT_BUILD=false ;
-    local STRIMZI_DOWNSTREAM_URL="" ;
-    local STRIMZI_BRANCH="main" ;
-
-    OPTS=$(getopt -o --long strimzi-repository:,strimzi-branch:,product-build,strimzi-downstream:,product-build:  -n 'clone-repositories' -- "$@")
     if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi
     eval set -- "$OPTS"
 
     while true; do
         case "$1" in
-            --strimzi-repository ) STRIMZI_REPO=$2;                shift 2;;
+            -c | --component )  COMPONENT=$2;          shift 2;;
+            --git-repository )  GIT_REPO=$2;           shift 2;;
+            --git-branch )      GIT_BRANCH=$2;         shift 2;;
             --product-build )   PRODUCT_BUILD=$2;      shift 2;;
-            --strimzi-downstream ) STRIMZI_DOWNSTREAM_URL=$2;      shift 2;;
+            --downstream-url )  DOWNSTREAM_URL=$2;     shift 2;;
-            --strimzi-branch )     STRIMZI_BRANCH=$2;              shift 2;;
             -- ) shift; break ;;
             * ) break ;;
         esac
     done
 
+    if [ "${COMPONENT}" != "apicurio" ] && [ "${COMPONENT}" != "strimzi" ] ;
+    then
+      exit 2 ;
+    fi ;
+
     if [ "${PRODUCT_BUILD}" = false ] ;
     then
-      # TODO why clone AND checkout?
+        git clone --branch "${GIT_BRANCH}" "${GIT_REPO}" "${COMPONENT}" || exit 2 ;
-        git clone --branch "${STRIMZI_BRANCH}" "${STRIMZI_REPO}" strimzi && git -C strimzi checkout "${STRIMZI_BRANCH}" || exit 2 ;
+    elif [ -z "${DOWNSTREAM_URL}" ] ;
-    elif [ -z "${STRIMZI_DOWNSTREAM_URL}" ] ;
     then
         exit 2 ;
     else
-        curl --retry 7 -Lo /tmp/strimzi_downstream.zip "${STRIMZI_DOWNSTREAM_URL}" ; 
+        curl --retry 7 -Lo /tmp/"${COMPONENT}".zip "${DOWNSTREAM_URL}" ;
-        unzip /tmp/strimzi_downstream.zip -d strimzi ;
+        unzip /tmp/"${COMPONENT}".zip -d "${COMPONENT}" ;
-        rm /tmp/strimzi_downstream.zip ;
+        rm /tmp/"${COMPONENT}".zip ;
     fi ;
 }
-
-# TODO downstream apicurio?
-clone_apicurio()
-{
-  local APICURIO_REPO="https://github.com/Apicurio/apicurio-registry-operator.git" ;
-  local APICURIO_BRANCH="master" ;
-
-  # TODO why clone AND checkout?
-  git clone --branch "${APICURIO_BRANCH}" "${APICURIO_REPO}" apicurio && git -C apicurio checkout "${APICURIO_BRANCH}" || exit 2 ;
-}

jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh

@@ -12,6 +12,7 @@ if [ ! -f "${SECRET_PATH}" ]; then
     exit 1
 fi
 
+# TODO remove git pull and rebuild once the development is done
 git -C /testsuite/debezium stash
 git -C /testsuite/debezium pull --rebase origin DBZ-5165
 git -C /testsuite/debezium log -1
@@ -22,32 +23,44 @@ mvn clean install -DskipTests -DskipITs -f /testsuite/debezium/pom.xml
 ${OCP_PROJECTS} --project "${OCP_PROJECT_DEBEZIUM}" --create
 
 # prepare strimzi
-clone_strimzi --strimzi-repository "${STRZ_GIT_REPOSITORY}" --strimzi-branch "${STRZ_GIT_BRANCH}" --product-build "${PRODUCT_BUILD}" --strimzi-downstream "${OCP_STRIMZI_DOWNSTREAM_URL}";
+# TODO remove the defaults once it's being ran from jenkins?
+if [ -z "${STRZ_GIT_REPOSITORY}" ]; then
+  STRZ_GIT_REPOSITORY="https://github.com/strimzi/strimzi-kafka-operator.git" ;
+fi
+
+if [ -z "${STRZ_GIT_BRANCH}" ]; then
+  STRZ_GIT_BRANCH="main" ;
+fi
+
+clone_component --component strimzi --git-repository "${STRZ_GIT_REPOSITORY}" --git-branch "${STRZ_GIT_BRANCH}" --product-build "${PRODUCT_BUILD}" --downstream-url "${STRZ_DOWNSTREAM_URL}" ;
 sed -i 's/namespace: .*/namespace: '"${OCP_PROJECT_DEBEZIUM}"'/' strimzi/install/cluster-operator/*RoleBinding*.yaml ;
 oc create -f strimzi/install/cluster-operator/ -n "${OCP_PROJECT_DEBEZIUM}" ;
 
-#prepare apicurio
+# prepare apicurio if not disabled
 AVRO_PATTERN='.*!avro.*'
 if [[ ! ${GROUPS_ARG} =~ ${AVRO_PATTERN} ]]; then
-  APICURIO_RESOURCE="install/apicurio-registry-operator-1.1.0-dev.yaml"
 
-  clone_apicurio
+  # TODO remove the defaults once it's being ran from jenkins?
-  sed -i "s/namespace: apicurio-registry-operator-namespace/namespace: ${OCP_PROJECT_REGISTRY}/" apicurio/install/*.yaml
+  if [ -z "${APIC_GIT_REPOSITORY}" ]; then
+    APIC_GIT_REPOSITORY="https://github.com/Apicurio/apicurio-registry-operator.git" ;
+  fi
+
+  if [ -z "${APIC_GIT_BRANCH}" ]; then
+    APIC_GIT_BRANCH="master" ;
+  fi
+
+  if [ -z "${APICURIO_RESOURCE}" ]; then
+    APICURIO_RESOURCE="install/apicurio-registry-operator-1.1.0-dev.yaml"
+  fi
+
+  clone_component --component apicurio --git-repository "${APIC_GIT_REPOSITORY}" --git-branch "${APIC_GIT_BRANCH}" --product-build "${PRODUCT_BUILD}" --downstream-url "${APIC_DOWNSTREAM_URL}" ;
+  sed -i "s/namespace: apicurio-registry-operator-namespace/namespace: ${OCP_PROJECT_REGISTRY}/" apicurio/install/*.yaml ;
   oc create -f apicurio/${APICURIO_RESOURCE} -n "${OCP_PROJECT_REGISTRY}" ;
 fi
 
 pushd ${DEBEZIUM_LOCATION} || exit 1;
 
-if [ -z "${TEST_VERSION_KAFKA}" ]; then
+oc project "${OCP_PROJECT_DEBEZIUM}"
-  TEST_PROPERTIES="";
-else 
-  TEST_PROPERTIES="-Dversion.kafka=${TEST_VERSION_KAFKA}" ;
-fi 
-
-if [ -n "${DBZ_CONNECT_IMAGE}" ]; then
-  TEST_PROPERTIES="$TEST_PROPERTIES -Dimage.kc=${DBZ_CONNECT_IMAGE}" ;
-fi
-
 mvn install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \
                     -Docp.project.debezium="${OCP_PROJECT_DEBEZIUM}" \
                     -Docp.project.db2="${OCP_PROJECT_DB2}" \

jenkins-jobs/pipelines/ocp_system_pipeline.groovy

@@ -0,0 +1,44 @@
+pipeline {
+    agent {
+        label 'Slave'
+    }
+
+    stages {
+        stage('Prepare project') {
+            steps {
+                withCredentials([
+                        usernamePassword(credentialsId: "${OCP_CREDENTIALS}", usernameVariable: 'OCP_USERNAME', passwordVariable: 'OCP_PASSWORD'),
+                        file(credentialsId: "${PULL_SECRET}", variable: 'SECRET_PATH'),
+                ]) {
+                    oc login -u ${OCP_USERNAME} -p ${OCP_PASSWORD}
+                    oc new-project debezium-test-parent
+                    oc adm policy add-cluster-role-to-user cluster-admin system:serviceaccount:debezium-test-parent:default
+                    oc create -f ${SECRET_PATH}
+                }
+            }
+        }
+
+        stage('Run tests') {
+            steps {
+                withCredentials([
+                        file(credentialsId: "${PULL_SECRET}", variable: 'SECRET_PATH'),
+                ]) {
+                    sh '''
+                    debezium/jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh --pull-secret-name '${PULL_SECRET}' \
+                    --docker-tag "docker-test" \
+                    --project-name debezium-test \
+                    --product-build false \
+                    --strimzi-kc-build true \
+                    --dbz-connect-image "quay.io/rh_integration/test-strimzi-kafka:strz-latest-kafka-3.1.0-apc-2.2.3.Final-dbz-2.0.0-SNAPSHOT" \
+                    --artifact-server-image "quay.io/rh_integration/test-artifact-server:2.0.0-SNAPSHOT" \
+                    --apicurio-version "2.2.3.Final" \
+                    --groups-arg 'postgresql & !docker'
+                    '''
+                    oc create -f test-job-deployment.yml
+                }
+            }
+        }
+
+    }
+
+}

jenkins-jobs/pipelines/system_test_image_prepare_pipeline.groovy

@@ -39,6 +39,7 @@ pipeline {
                     docker tag testsuite:docker-test quay.io/rh_integration/dbz-testing-system:${TAG}
                     docker login -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} quay.io
                     docker push quay.io/rh_integration/dbz-testing-system:${TAG}
+                    rm debezium/jenkins-jobs/docker/debezium-testing-system/secret.yml
                 '''
                 }
             }