DBZ-5165 created job-dsl for ocp system pipeline, renamed deployment parameters

jenkins-jobs/docker/debezium-testing-system/deployment-template.sh

@@ -1,8 +1,6 @@
 #!/bin/bash
 
-PULL_SECRET_NAME="aswerf"
-
-OPTS=$(getopt -o h: --long pull-secret-name:,docker-tag:,project-name:,product-build:,strimzi-kc-build:,dbz-connect-image:,artifact-server-image:,apicurio-version:,groups-arg:,help  -n 'parse-options' -- "$@")
+OPTS=$(getopt -o h: --long filename:,pull-secret-name:,docker-tag:,project-name:,product-build:,strimzi-kc-build:,dbz-connect-image:,artifact-server-image:,apicurio-version:,groups-arg:,strz-git-repository:,strz-git-branch:,strz-downstream-url:,apic-git-repository:,apic-git-branch:,apic-downstream-url:,help  -n 'parse-options' -- "$@")
 if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi
 eval set -- "$OPTS"
 
@@ -10,6 +8,7 @@ echo "${OPTS[@]}"
 
 while true; do
     case "$1" in
+        --filename )              FILENAME=$2;                shift 2;;
         --pull-secret-name )      PULL_SECRET_NAME=$2;        shift 2;;
         --docker-tag )            DOCKER_TAG=$2;              shift 2;;
         --project-name )          PROJECT_NAME=$2;            shift 2;;
@@ -19,6 +18,12 @@ while true; do
         --artifact-server-image ) ARTIFACT_SERVER_IMAGE=$2;   shift 2;;
         --apicurio-version )      APICURIO_VERSION=$2;        shift 2;;
         --groups-arg )            GROUPS_ARG=$2;              shift 2;;
+        --strz-git-repository )   STRZ_GIT_REPOSITORY         shift 2;;
+        --strz-git-branch )       STRZ_GIT_BRANCH             shift 2;;
+        --strz-downstream-url )   STRZ_DOWNSTREAM_URL         shift 2;;
+        --apic-git-repository )   APIC_GIT_REPOSITORY         shift 2;;
+        --apic-git-branch )       APIC_GIT_BRANCH             shift 2;;
+        --apic-downstream-url )   APIC_DOWNSTREAM_URL         shift 2;;
         -h | --help )             PRINT_HELP=true             shift ;;
         -- ) shift; break ;;
         * ) break ;;
@@ -46,40 +51,52 @@ spec:
             - containerPort: 8080
               protocol: \"TCP\"
           env:
-            - name: OCP_PROJECT_DEBEZIUM
+            - name: DBZ_OCP_PROJECT_DEBEZIUM
               value: \"${PROJECT_NAME}\"
-            - name: OCP_PROJECT_DB2
+            - name: DBZ_OCP_PROJECT_DB2
               value: \"${PROJECT_NAME}-db2\"
-            - name: OCP_PROJECT_MONGO
+            - name: DBZ_OCP_PROJECT_MONGO
               value: \"${PROJECT_NAME}-mongo\"
-            - name: OCP_PROJECT_MYSQL
+            - name: DBZ_OCP_PROJECT_MYSQL
               value: \"${PROJECT_NAME}-mysql\"
-            - name: OCP_PROJECT_ORACLE
+            - name: DBZ_OCP_PROJECT_ORACLE
               value: \"${PROJECT_NAME}-oracle\"
-            - name: OCP_PROJECT_POSTGRESQL
+            - name: DBZ_OCP_PROJECT_POSTGRESQL
               value: \"${PROJECT_NAME}-postgresql\"
-            - name: OCP_PROJECT_SQLSERVER
+            - name: DBZ_OCP_PROJECT_SQLSERVER
               value: \"${PROJECT_NAME}-sqlserver\"
-            - name: OCP_PROJECT_REGISTRY
+            - name: DBZ_OCP_PROJECT_REGISTRY
               value: \"${PROJECT_NAME}-registry\"
-            - name: SECRET_PATH
+            - name: DBZ_SECRET_PATH
               value: \"/testsuite/secret.yml\"
-            - name: TEST_WAIT_SCALE
+            - name: DBZ_TEST_WAIT_SCALE
               value: \"10\"
-            - name: PRODUCT_BUILD
+            - name: DBZ_PRODUCT_BUILD
               value: \"${PRODUCT_BUILD}\"
-            - name: STRIMZI_KC_BUILD
+            - name: DBZ_STRIMZI_KC_BUILD
               value: \"${STRIMZI_KC_BUILD}\"
             - name: DBZ_CONNECT_IMAGE
               value: \"${DBZ_CONNECT_IMAGE}\"
-            - name: ARTIFACT_SERVER_IMAGE
+            - name: DBZ_ARTIFACT_SERVER_IMAGE
               value: \"${ARTIFACT_SERVER_IMAGE}\"
-            - name: APICURIO_VERSION
+            - name: DBZ_APICURIO_VERSION
               value: \"${APICURIO_VERSION}\"
-            - name: GROUPS_ARG
+            - name: DBZ_GROUPS_ARG
               value: \"${GROUPS_ARG}\"
-            - name: DELETE_PROJECTS
+            - name: DBZ_OCP_DELETE_PROJECTS
               value: \"true\"
+            - name: STRZ_GIT_REPOSITORY
+              value: \"${STRZ_GIT_REPOSITORY}\"
+            - name: STRZ_GIT_BRANCH
+              value: \"${STRZ_GIT_BRANCH}\"
+            - name: STRZ_DOWNSTREAM_URL
+              value: \"${STRZ_DOWNSTREAM_URL}\"
+            - name: APIC_GIT_REPOSITORY
+              value: \"${APIC_GIT_REPOSITORY}}\"
+            - name: APIC_GIT_BRANCH
+              value: \"${APIC_GIT_BRANCH}}\"
+            - name: APIC_DOWNSTREAM_URL
+              value: \"${APIC_DOWNSTREAM_URL}}\"
   triggers:
     - type: \"ConfigChange\"
   paused: false
@@ -88,4 +105,4 @@ spec:
 "
 
 
-printf "%s" "$output" >> test-job-deployment.yml
+printf "%s" "$output" >> "${FILENAME}.yml"

jenkins-jobs/docker/debezium-testing-system/deployment.yml

@@ -19,39 +19,39 @@ spec:
             - containerPort: 8080
               protocol: "TCP"
           env:
-            - name: OCP_PROJECT_DEBEZIUM
+            - name: DBZ_OCP_PROJECT_DEBEZIUM
               value: "debezium-test"
-            - name: OCP_PROJECT_DB2
+            - name: DBZ_OCP_PROJECT_DB2
               value: "debezium-test-db2"              
-            - name: OCP_PROJECT_MONGO
+            - name: DBZ_OCP_PROJECT_MONGO
               value: "debezium-test-mongo"
-            - name: OCP_PROJECT_MYSQL
+            - name: DBZ_OCP_PROJECT_MYSQL
               value: "debezium-test-mysql"
-            - name: OCP_PROJECT_ORACLE
+            - name: DBZ_OCP_PROJECT_ORACLE
               value: "debezium-test-oracle"
-            - name: OCP_PROJECT_POSTGRESQL
+            - name: DBZ_OCP_PROJECT_POSTGRESQL
               value: "debezium-test-postgresql"
-            - name: OCP_PROJECT_SQLSERVER
+            - name: DBZ_OCP_PROJECT_SQLSERVER
               value: "debezium-test-sqlserver"
-            - name: OCP_PROJECT_REGISTRY
+            - name: DBZ_OCP_PROJECT_REGISTRY
               value: "debezium-test-registry"
-            - name: SECRET_PATH
+            - name: DBZ_SECRET_PATH
               value: "/testsuite/secret.yml"
-            - name: TEST_WAIT_SCALE
+            - name: DBZ_TEST_WAIT_SCALE
               value: "10"
-            - name: PRODUCT_BUILD
+            - name: DBZ_PRODUCT_BUILD
               value: "false"
-            - name: STRIMZI_KC_BUILD
+            - name: DBZ_STRIMZI_KC_BUILD
               value: "true"
             - name: DBZ_CONNECT_IMAGE
               value: "quay.io/rh_integration/test-strimzi-kafka:strz-latest-kafka-3.1.0-apc-2.2.3.Final-dbz-2.0.0-SNAPSHOT"
-            - name: ARTIFACT_SERVER_IMAGE
+            - name: DBZ_ARTIFACT_SERVER_IMAGE
               value: "quay.io/rh_integration/test-artifact-server:2.0.0-SNAPSHOT"
-            - name: APICURIO_VERSION
+            - name: DBZ_APICURIO_VERSION
               value: "2.2.3.Final"
-            - name: GROUPS_ARG
+            - name: DBZ_GROUPS_ARG
               value: "postgresql & !docker"
-            - name: DELETE_PROJECTS
+            - name: DBZ_OCP_DELETE_PROJECTS
               value: "true"
             - name: STRZ_GIT_REPOSITORY
               value: ""

jenkins-jobs/docker/debezium-testing-system/secret.yml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: mmedek-pull-secret
+data:
+  .dockerconfigjson: ewogICJhdXRocyI6IHsKICAgICJxdWF5LmlvIjogewogICAgICAiYXV0aCI6ICJiVzFsWkdWck9rSlpkRFV4TkRsNU1FSnlXWE5IVjIwNFZHcGtUMHAxUW1aSlUzcGpOMGRQU0ZONWVIZGFiRWxqYjFacGJVTkliSGhOV1N0NmJtMXhWRkJwTVdkblZVdz0iLAogICAgICAiZW1haWwiOiAiIgogICAgfQogIH0KfQ==
+type: kubernetes.io/dockerconfigjson

jenkins-jobs/docker/debezium-testing-system/testsuite-deployment.sh

@@ -5,10 +5,9 @@ source /testsuite/library.sh
 
 DEBEZIUM_LOCATION="/testsuite/debezium"
 OCP_PROJECTS="${DEBEZIUM_LOCATION}/jenkins-jobs/scripts/ocp-projects.sh"
-SECRET_PATH=/testsuite/secret.yml
 
-if [ ! -f "${SECRET_PATH}" ]; then
-    echo "secret ${SECRET_PATH} does not  exist!"
+if [ ! -f "${DBZ_SECRET_PATH}" ]; then
+    echo "secret ${DBZ_SECRET_PATH} does not  exist!"
     exit 1
 fi
 
@@ -20,7 +19,7 @@ git -C /testsuite/debezium log -1
 mvn clean install -DskipTests -DskipITs -f /testsuite/debezium/pom.xml
 
 # create projects
-${OCP_PROJECTS} --project "${OCP_PROJECT_DEBEZIUM}" --create
+${OCP_PROJECTS} --project "${DBZ_OCP_PROJECT_DEBEZIUM}" --create
 
 # prepare strimzi
 # TODO remove the defaults once it's being ran from jenkins?
@@ -33,8 +32,8 @@ if [ -z "${STRZ_GIT_BRANCH}" ]; then
 fi
 
 clone_component --component strimzi --git-repository "${STRZ_GIT_REPOSITORY}" --git-branch "${STRZ_GIT_BRANCH}" --product-build "${PRODUCT_BUILD}" --downstream-url "${STRZ_DOWNSTREAM_URL}" ;
-sed -i 's/namespace: .*/namespace: '"${OCP_PROJECT_DEBEZIUM}"'/' strimzi/install/cluster-operator/*RoleBinding*.yaml ;
-oc create -f strimzi/install/cluster-operator/ -n "${OCP_PROJECT_DEBEZIUM}" ;
+sed -i 's/namespace: .*/namespace: '"${DBZ_OCP_PROJECT_DEBEZIUM}"'/' strimzi/install/cluster-operator/*RoleBinding*.yaml ;
+oc create -f strimzi/install/cluster-operator/ -n "${DBZ_OCP_PROJECT_DEBEZIUM}" ;
 
 # prepare apicurio if not disabled
 AVRO_PATTERN='.*!avro.*'
@@ -60,29 +59,28 @@ fi
 
 pushd ${DEBEZIUM_LOCATION} || exit 1;
 
-oc project "${OCP_PROJECT_DEBEZIUM}"
 mvn install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \
-                    -Docp.project.debezium="${OCP_PROJECT_DEBEZIUM}" \
-                    -Docp.project.db2="${OCP_PROJECT_DB2}" \
-                    -Docp.project.mongo="${OCP_PROJECT_MONGO}" \
-                    -Docp.project.mysql="${OCP_PROJECT_MYSQL}" \
-                    -Docp.project.oracle="${OCP_PROJECT_ORACLE}" \
-                    -Docp.project.postgresql="${OCP_PROJECT_POSTGRESQL}" \
-                    -Docp.project.sqlserver="${OCP_PROJECT_SQLSERVER}" \
-                    -Docp.project.registry="${OCP_PROJECT_REGISTRY}" \
-                    -Docp.pull.secret.paths="${SECRET_PATH}" \
-                    -Dtest.wait.scale="${TEST_WAIT_SCALE}" \
-                    -Dtest.strimzi.kc.build="${STRIMZI_KC_BUILD}" \
+                    -Docp.project.debezium="${DBZ_OCP_PROJECT_DEBEZIUM}" \
+                    -Docp.project.db2="${DBZ_OCP_PROJECT_DB2}" \
+                    -Docp.project.mongo="${DBZ_OCP_PROJECT_MONGO}" \
+                    -Docp.project.mysql="${DBZ_OCP_PROJECT_MYSQL}" \
+                    -Docp.project.oracle="${DBZ_OCP_PROJECT_ORACLE}" \
+                    -Docp.project.postgresql="${DBZ_OCP_PROJECT_POSTGRESQL}" \
+                    -Docp.project.sqlserver="${DBZ_OCP_PROJECT_SQLSERVER}" \
+                    -Docp.project.registry="${DBZ_OCP_PROJECT_REGISTRY}" \
+                    -Docp.pull.secret.paths="${DBZ_SECRET_PATH}" \
+                    -Dtest.wait.scale="${DBZ_TEST_WAIT_SCALE}" \
+                    -Dtest.strimzi.kc.build="${DBZ_STRIMZI_KC_BUILD}" \
                     -Dimage.kc="${DBZ_CONNECT_IMAGE}" \
-                    -Dimage.as="${ARTIFACT_SERVER_IMAGE}" \
-                    -Das.apicurio.version="${APICURIO_VERSION}" \
-                    -Dgroups="${GROUPS_ARG}"
+                    -Dimage.as="${DBZ_ARTIFACT_SERVER_IMAGE}" \
+                    -Das.apicurio.version="${DBZ_APICURIO_VERSION}" \
+                    -Dgroups="${DBZ_GROUPS_ARG}"
 
 popd || exit 1;
 
 cp debezium/debezium-testing/debezium-testing-system/target/failsafe-reports/*.xml /testsuite/logs
 
-if [ "${DELETE_PROJECTS}" = true ] ;
+if [ "${DBZ_OCP_DELETE_PROJECTS}" = true ] ;
 then
-  ${OCP_PROJECTS} --project "${OCP_PROJECT_DEBEZIUM}" --delete
+  ${OCP_PROJECTS} --project "${DBZ_OCP_PROJECT_DEBEZIUM}" --delete
 fi ;

jenkins-jobs/job-dsl/ocp_system_test.groovy

@@ -0,0 +1,40 @@
+pipelineJob('ocp-system-test') {
+    displayName('System Tests inside OCP')
+    description('Runs System Tests inside OCP')
+
+    parameters {
+        stringParam('MAIL_TO', 'jcechace@redhat.com')
+        booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')
+
+        stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned')
+        stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources')
+
+        stringParam('PULL_SECRET', 'rh-integration-quay-secret', 'Quay.io OpenShift secret')
+        stringParam('DOCKER_TAG', 'latest', 'Docker image tag')
+        stringParam('OCP_PROJECT_NAME', 'debezium-test', 'OCP projects name')
+        stringParam('OCP_CREDENTIALS', 'openshift-dbz-creds', 'Jenkins credentials id')
+        stringParam('OCP_URL', '', 'Ocp url')
+
+        booleanParam('TEST_APICURIO_REGISTRY', false, 'Run tests with Apicurio Registry and Avro serialization')
+
+        booleanParam('STRIMZI_KC_BUILD', false, 'Use connect image instead of artifact server')
+        stringParam('DBZ_CONNECT_IMAGE', 'quay.io/rh_integration/test-strimzi-kafka:strz-latest-kafka-3.1.0-apc-2.2.3.Final-dbz-2.0.0-SNAPSHOT', 'Debezium connect image')
+        stringParam('ARTIFACT_SERVER_IMAGE', 'quay.io/rh_integration/test-artifact-server:2.0.0-SNAPSHOT', 'Artifact server image')
+        stringParam('APICURIO_VERSION', '2.2.3.Final', 'Apicurio version')
+
+        stringParam('STRZ_GIT_REPOSITORY', 'https://github.com/strimzi/strimzi-kafka-operator.git', 'Repository from which Strimzi is cloned')
+        stringParam('STRZ_GIT_BRANCH', 'main', 'A branch/tag from which Strimzi is built')
+        stringParam('STRZ_DOWNSTREAM_URL', '', 'URL to productised strimzi sources')
+
+        stringParam('APIC_GIT_REPOSITORY', 'https://github.com/Apicurio/apicurio-registry-operator.git', 'Repository from which Apicurio is cloned')
+        stringParam('APIC_GIT_BRANCH', 'main', 'A branch/tag from which Apicurio is built')
+        stringParam('APIC_DOWNSTREAM_URL', '', 'URL to productised apicurio sources')
+    }
+
+    definition {
+        cps {
+            script(readFileFromWorkspace('jenkins-jobs/pipelines/ocp_system_pipeline.groovy'))
+            sandbox()
+        }
+    }
+}

jenkins-jobs/pipelines/ocp_system_pipeline.groovy

@@ -4,16 +4,30 @@ pipeline {
     }
 
     stages {
+        stage('Checkout - Debezium') {
+            steps {
+                checkout([
+                        $class           : 'GitSCM',
+                        branches         : [[name: "${DBZ_GIT_BRANCH}"]],
+                        userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]],
+                        extensions       : [[$class           : 'RelativeTargetDirectory',
+                                             relativeTargetDir: 'debezium']],
+                ])
+            }
+        }
+
         stage('Prepare project') {
             steps {
                 withCredentials([
                         usernamePassword(credentialsId: "${OCP_CREDENTIALS}", usernameVariable: 'OCP_USERNAME', passwordVariable: 'OCP_PASSWORD'),
                         file(credentialsId: "${PULL_SECRET}", variable: 'SECRET_PATH'),
                 ]) {
-                    oc login -u ${OCP_USERNAME} -p ${OCP_PASSWORD}
-                    oc new-project debezium-test-parent
-                    oc adm policy add-cluster-role-to-user cluster-admin system:serviceaccount:debezium-test-parent:default
-                    oc create -f ${SECRET_PATH}
+                    sh '''
+                    oc login -u "${OCP_USERNAME}" -p "${OCP_PASSWORD}" --insecure-skip-tls-verify=true "${OCP_URL}"
+                    oc new-project "${OCP_PROJECT_NAME}-parent" || oc project "${OCP_PROJECT_NAME}-parent"
+                    oc adm policy add-cluster-role-to-user cluster-admin "system:serviceaccount:${OCP_PROJECT_NAME}-parent:default"
+                    oc apply -f "${SECRET_PATH}"
+                    '''
                 }
             }
         }
@@ -24,17 +38,63 @@ pipeline {
                         file(credentialsId: "${PULL_SECRET}", variable: 'SECRET_PATH'),
                 ]) {
                     sh '''
-                    debezium/jenkins-jobs/docker/debezium-testing-system/deployment_templater.sh --pull-secret-name '${PULL_SECRET}' \
-                    --docker-tag "docker-test" \
-                    --project-name debezium-test \
+                    GROUPS_ARG="!docker"
+                    if [ ${TEST_APICURIO_REGISTRY} == false ]; then
+                        GROUPS_ARG="${GROUPS_ARG} & !avro"
+                    fi
+
+                    FILENAME="testsuite-job"
+
+                    OPTIONAL_PARAMS=""
+                    if [ ! -z  "${DBZ_CONNECT_IMAGE}"]; then
+                        OPTIONAL_PARAMS="$OPTIONAL_PARAMS --dbz-connect-image ${DBZ_CONNECT_IMAGE}"
+                    fi
+
+                    if [ ! -z  "${ARTIFACT_SERVER_IMAGE}"]; then
+                        OPTIONAL_PARAMS="$OPTIONAL_PARAMS --artifact-server-image ${ARTIFACT_SERVER_IMAGE}"
+                    fi
+
+                    if [ ! -z  "${APICURIO_VERSION}"]; then
+                        OPTIONAL_PARAMS="$OPTIONAL_PARAMS --apicurio-version ${APICURIO_VERSION}"
+                    fi
+
+                    if [ ! -z  "${STRZ_GIT_REPOSITORY}"]; then
+                        OPTIONAL_PARAMS="$OPTIONAL_PARAMS --strz-git-repository ${STRZ_GIT_REPOSITORY}"
+                    fi
+
+                    if [ ! -z  "${STRZ_GIT_BRANCH}"]; then
+                        OPTIONAL_PARAMS="$OPTIONAL_PARAMS --strz-git-branch ${STRZ_GIT_BRANCH}"
+                    fi
+
+                    if [ ! -z  "${STRZ_DOWNSTREAM_URL}"]; then
+                        OPTIONAL_PARAMS="$OPTIONAL_PARAMS --strz-downstream-url ${STRZ_DOWNSTREAM_URL}"
+                    fi
+
+                    if [ ! -z  "${APIC_GIT_REPOSITORY}"]; then
+                        OPTIONAL_PARAMS="$OPTIONAL_PARAMS --apic-git-repository ${APIC_GIT_REPOSITORY}"
+                    fi
+
+                    if [ ! -z  "${APIC_GIT_BRANCH}"]; then
+                        OPTIONAL_PARAMS="$OPTIONAL_PARAMS --apic-git-branch ${APIC_GIT_BRANCH}"
+                    fi
+
+                    if [ ! -z  "${APIC_DOWNSTREAM_URL}"]; then
+                        OPTIONAL_PARAMS="$OPTIONAL_PARAMS --apic-downstream-url ${APIC_DOWNSTREAM_URL}"
+                    fi
+
+                    cd ${WORKSPACE}/debezium
+                    jenkins-jobs/docker/debezium-testing-system/deployment-template.sh --filename "${FILENAME}" \
+                    --pull-secret-name '${PULL_SECRET}' \
+                    --docker-tag "${DOCKER_TAG}" \
+                    --project-name "${OCP_PROJECT_NAME}" \
                     --product-build false \
-                    --strimzi-kc-build true \
-                    --dbz-connect-image "quay.io/rh_integration/test-strimzi-kafka:strz-latest-kafka-3.1.0-apc-2.2.3.Final-dbz-2.0.0-SNAPSHOT" \
-                    --artifact-server-image "quay.io/rh_integration/test-artifact-server:2.0.0-SNAPSHOT" \
-                    --apicurio-version "2.2.3.Final" \
-                    --groups-arg 'postgresql & !docker'
+                    --strimzi-kc-build ${STRIMZI_KC_BUILD} \
+                    --apicurio-version "${APICURIO_VERSION}" \
+                    --groups-arg "${GROUPS_ARG}" \
+                    ${OPTIONAL_PARAMS}
+                    oc delete -f "${FILENAME}.yml" --ignore-not-found
+                    oc create -f "${FILENAME}.yml"
                     '''
-                    oc create -f test-job-deployment.yml
                 }
             }
         }