From 45b381b75ef595a8c6e2b86434a590d86c5abed0 Mon Sep 17 00:00:00 2001 From: Jiri Novotny Date: Mon, 13 Sep 2021 15:18:58 +0200 Subject: [PATCH] DBZ-3934 added jenkins job for artifact server preparation --- ..._downstream_artifact_server_prepare.groovy | 28 ++++++ ...am_artifact_server_prepare_pipeline.groovy | 58 +++++++++++++ jenkins-jobs/scripts/build-connect-image.sh | 2 +- .../scripts/build-rhel-kafka-image.sh | 2 +- jenkins-jobs/scripts/copy-images.sh | 2 +- jenkins-jobs/scripts/copy-plugins.sh | 86 +++++++++++++++++++ 6 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 jenkins-jobs/job-dsl/ocp_downstream_artifact_server_prepare.groovy create mode 100644 jenkins-jobs/pipelines/downstream_artifact_server_prepare_pipeline.groovy create mode 100755 jenkins-jobs/scripts/copy-plugins.sh diff --git a/jenkins-jobs/job-dsl/ocp_downstream_artifact_server_prepare.groovy b/jenkins-jobs/job-dsl/ocp_downstream_artifact_server_prepare.groovy new file mode 100644 index 000000000..149ca6e04 --- /dev/null +++ b/jenkins-jobs/job-dsl/ocp_downstream_artifact_server_prepare.groovy @@ -0,0 +1,28 @@ +pipelineJob('ocp-downstream-artifact-server-prepare-job') { + displayName('Artifact Server Preparation') + description('Prepares plugins file for artifact server') + + properties { + githubProjectUrl('https://github.com/debezium/debezium') + } + + parameters { +// QUAY CONFIG + stringParam('QUAY_CREDENTIALS', 'rh-integration-quay-creds', 'Quay.io credentials id') + stringParam('QUAY_ORGANISATION', '', 'Organisation where images are copied') +// DEBEZIUM CONFIG + stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned') + stringParam('DBZ_GIT_BRANCH', 'master', 'A branch/tag of Debezium sources') +// DEBEZIUM CONNECT IMAGE CONFIG + textParam('DBZ_CONNECTOR_ARCHIVE_URLS', '', 'List of URLs to productised Debezium connectors') +// EXTRA CONFIG + textParam('DBZ_EXTRA_LIBS', '', 'List of extra libraries added to connectors') + } + + definition { + cps { + script(readFileFromWorkspace('jenkins-jobs/pipelines/downstream_artifact_server_prepare_pipeline.groovy')) + sandbox() + } + } +} diff --git a/jenkins-jobs/pipelines/downstream_artifact_server_prepare_pipeline.groovy b/jenkins-jobs/pipelines/downstream_artifact_server_prepare_pipeline.groovy new file mode 100644 index 000000000..9d539acd4 --- /dev/null +++ b/jenkins-jobs/pipelines/downstream_artifact_server_prepare_pipeline.groovy @@ -0,0 +1,58 @@ +pipeline { + agent { + label 'Slave' + } + + stages { + stage('CleanWorkspace') { + steps { + cleanWs() + } + } + + stage('Checkout') { + steps { + checkout([ + $class : 'GitSCM', + branches : [[name: "${DBZ_GIT_BRANCH}"]], + userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]], + extensions : [[$class : 'RelativeTargetDirectory', + relativeTargetDir: 'debezium']], + ]) + } + } + + stage('Process connectors and extra libs') { + steps { + withCredentials([ + usernamePassword(credentialsId: "${QUAY_CREDENTIALS}", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'), + ]) { + sh ''' + set -x + cd "${WORKSPACE}/debezium" + ./jenkins-jobs/scripts/copy-plugins.sh \\ + --dir="${WORKSPACE}" \\ + --archive-urls="${DBZ_CONNECTOR_ARCHIVE_URLS}" \\ + --libs="${DBZ_EXTRA_LIBS}" \\ + --tag="${IMAGE_TAG}" \\ + --registry="quay.io" --organisation="${QUAY_ORGANISATION}" \\ + --dest-login="${QUAY_USERNAME}" \\ + --dest-pass="${QUAY_PASSWORD}" \\ + --img-output="${WORKSPACE}/published_image_dbz.txt" + ''' + } + } + } + } + + post { + always { + mail to: 'jcechace@redhat.com', subject: "Debezium artifact server preparation #${BUILD_NUMBER} finished", body: """ +${currentBuild.projectName} run ${BUILD_URL} finished with result: ${currentBuild.currentResult} +""" + } + success { + archiveArtifacts "**/published_image*.txt" + } + } +} diff --git a/jenkins-jobs/scripts/build-connect-image.sh b/jenkins-jobs/scripts/build-connect-image.sh index 6ef95652c..a77ea8da9 100755 --- a/jenkins-jobs/scripts/build-connect-image.sh +++ b/jenkins-jobs/scripts/build-connect-image.sh @@ -29,7 +29,7 @@ while true; do esac done -if [ -z "${DEST_LOGIN}" ] ; then +if [ ! -z "${DEST_LOGIN}" ] ; then docker login -u "${DEST_LOGIN}" -p "${DEST_PASS}" "${REGISTRY}" fi diff --git a/jenkins-jobs/scripts/build-rhel-kafka-image.sh b/jenkins-jobs/scripts/build-rhel-kafka-image.sh index c8ae61288..256603811 100755 --- a/jenkins-jobs/scripts/build-rhel-kafka-image.sh +++ b/jenkins-jobs/scripts/build-rhel-kafka-image.sh @@ -29,7 +29,7 @@ while true; do esac done -if [ -z "${DEST_LOGIN}" ] ; then +if [ ! -z "${DEST_LOGIN}" ] ; then docker login -u "${DEST_LOGIN}" -p "${DEST_PASS}" "${REGISTRY}" fi diff --git a/jenkins-jobs/scripts/copy-images.sh b/jenkins-jobs/scripts/copy-images.sh index 5e7fc0fee..defb636f3 100755 --- a/jenkins-jobs/scripts/copy-images.sh +++ b/jenkins-jobs/scripts/copy-images.sh @@ -25,7 +25,7 @@ while true; do esac done -if [ -z "${DEST_LOGIN}" ] ; then +if [ ! -z "${DEST_LOGIN}" ] ; then docker login -u "${DEST_LOGIN}" -p "${DEST_PASS}" "${REGISTRY}" fi diff --git a/jenkins-jobs/scripts/copy-plugins.sh b/jenkins-jobs/scripts/copy-plugins.sh new file mode 100755 index 000000000..fb314db56 --- /dev/null +++ b/jenkins-jobs/scripts/copy-plugins.sh @@ -0,0 +1,86 @@ +#! /usr/bin/env bash +set -x +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +DOCKER_FILE=${DIR}/../docker/artifact-server/Dockerfile +PLUGIN_DIR="plugins" +EXTRA_LIBS="" + +OPTS=$(getopt -o d:a:l:f:r:o:t: --long dir:,archive-urls:,libs:,dockerfile:,registry:,organisation:,tag:,dest-login:,dest-pass:,img-output: -n 'parse-options' -- "$@") +if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi +eval set -- "$OPTS" + +while true; do + case "$1" in + -d | --dir ) BUILD_DIR=$2; + PLUGIN_DIR=${BUILD_DIR}/plugins; shift; shift ;; + -a | --archive-urls ) ARCHIVE_URLS=$2; shift; shift ;; + -l | --libs ) EXTRA_LIBS=$2; shift; shift ;; + -f | --dockerfile ) DOCKER_FILE=$2; shift; shift ;; + -r | --registry ) REGISTRY=$2; shift; shift ;; + -o | --organisation ) ORGANISATION=$2; shift; shift ;; + -t | --tag ) TAG=$2; shift; shift ;; + --dest-login ) DEST_LOGIN=$2; shift; shift ;; + --dest-pass ) DEST_PASS=$2; shift; shift ;; + --img-output ) IMAGE_OUTPUT_FILE=$2; shift; shift ;; + -h | --help ) PRINT_HELP=true; shift ;; + -- ) shift; break ;; + * ) break ;; + esac +done + +if [ ! -z "${DEST_LOGIN}" ] ; then + docker login -u "${DEST_LOGIN}" -p "${DEST_PASS}" "${REGISTRY}" +fi + +echo "Creating plugin directory ${PLUGIN_DIR}" +mkdir -p "${PLUGIN_DIR}" + +pushd "${PLUGIN_DIR}" || exit +for archive in ${ARCHIVE_URLS}; do + echo "[Processing] ${archive}" + lib=$(echo ${archive} | awk -F "::" '{print $1}' | xargs) + dest=$(echo ${archive} | awk -F "::" '{print $2}' | xargs) + + if [ -z "$dest" ] ; then + curl -OJs "${lib}" + else + mkdir -p "${dest}" && pushd ${dest} && curl -OJs "${lib}" && popd || exit + fi + connectors_version=$(echo "$archive" | sed -rn 's|.*AMQ-CDC-(.*)/.*$|\1|p') +done + +for input in ${EXTRA_LIBS}; do + echo "[Processing] ${input} " + lib=$(echo ${input} | awk -F "::" '{print $1}' | xargs) + dest=$(echo ${input} | awk -F "::" '{print $2}' | xargs) + + if [ -z "$dest" ] ; then + curl -OJs "${lib}" + else + mkdir -p "${dest}" && pushd ${dest} && curl -OJs "${lib}" && popd || exit + fi + +done +popd || exit + +echo "Copying scripts to" "${BUILD_DIR}" +cp "${DIR}"/../docker/artifact-server/* "$BUILD_DIR" + +echo "Copying Dockerfile to" "${BUILD_DIR}" +cp "$DOCKER_FILE" "$BUILD_DIR" + +if [ -z "$TAG" ] ; then + image_dbz=dbz-artifact-server:dbz-${connectors_version} + target=${REGISTRY}/${ORGANISATION}/${image_dbz} +else + target=$TAG +fi + +pushd "${BUILD_DIR}" || exit +echo "[Build] Building ${image_dbz}" +docker build . -t "$target" +popd || exit + +echo "[Build] Pushing image ${target}" +docker push ${target} +[[ -z "${IMAGE_OUTPUT_FILE}" ]] || echo $target >> ${IMAGE_OUTPUT_FILE}