alexey.rassokhin/tet123
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

DBZ-6299 Remove migrated jobs and change structure

Browse Source
This commit is contained in:
Ondrej Babec 2023-04-03 12:54:45 +02:00 committed by Ondrej Babec
parent 9c8954bb25
commit 6a54b04da1
50 changed files with 52 additions and 2660 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
jenkins-jobs/job-dsl/connector_all_tests.groovy
View File

@ -1,35 +0,0 @@
pipelineJob('connector-all-tests') {
displayName('Debezium All Connectors Test')
description('Executes tests for all connectors')
parameters {
booleanParam('DB2_TEST', true, 'Run DB2 Tests')
booleanParam('MONGODB_TEST', true, 'Run MongoDB Tests')
booleanParam('MYSQL_TEST', true, 'Run MySQL Tests')
booleanParam('ORACLE_TEST', true, 'Run Oracle Tests')
booleanParam('POSTGRESQL_TEST', true, 'Run PostgreSQL Tests')
booleanParam('SQLSERVER_TEST', true, 'Run SQL Server Tests')
stringParam('REPOSITORY_CORE', 'https://github.com/debezium/debezium', 'Repository from which Debezium is built')
stringParam('BRANCH', 'main', 'A branch/tag from which Debezium is built')
// db2 specific
stringParam('REPOSITORY_DB2', 'https://github.com/debezium/debezium-connector-db2', 'Repository from which DB2 connector is built')
stringParam('SOURCE_URL', "", "URL to productised sources")
booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')
//oracle specific
stringParam('QUAY_CREDENTIALS', 'rh-integration-quay-creds', 'Quay.io credentials id')
stringParam('LABEL', "", 'Label/Debezium Version')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/connector_tests_trigger_pipeline.groovy'))
sandbox()
}
}
}

97
jenkins-jobs/job-dsl/connector_db2_matrix_test.groovy
View File

@ -1,97 +0,0 @@
// Job definition to test PostgreSQL connector against different PostgreSQL versions
matrixJob('connector-debezium-db2-matrix-test') {
displayName('Debezium DB2 Connector Test Matrix')
description('Executes tests for DB2 Connector')
label('Slave')
axes {
label("Node", "Slave")
}
properties {
githubProjectUrl('https://github.com/debezium/debezium-connector-db2')
}
parameters {
stringParam('REPOSITORY', 'https://github.com/debezium/debezium-connector-db2', 'Repository from which connector is built')
stringParam('BRANCH', 'main', 'A branch/tag from which the connector is built')
stringParam('SOURCE_URL', "", "URL to productised sources")
booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')
}
triggers {
cron('H 04 * * *')
}
wrappers {
preBuildCleanup()
timeout {
noActivity(1200)
}
credentialsBinding {
usernamePassword('QUAY_USERNAME', 'QUAY_PASSWORD', 'rh-integration-quay-creds')
string('RP_TOKEN', 'report-portal-token')
}
}
publishers {
archiveArtifacts {
pattern('**/archive.tar.gz')
}
archiveJunit('**/target/surefire-reports/*.xml')
archiveJunit('**/target/failsafe-reports/*.xml')
mailer('debezium-qe@redhat.com', false, true)
}
logRotator {
daysToKeep(7)
numToKeep(5)
}
steps {
shell('''
# Ensure WS cleaup
ls -A1 | xargs rm -rf
# Retrieve sources
if [ "$PRODUCT_BUILD" == true ] ; then
export MAVEN_OPTS="-Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true"
PROFILE_PROD="-Ppnc"
curl -OJs $SOURCE_URL && unzip debezium-*-src.zip
pushd debezium-*-src
pushd debezium-connector-db2-*
ATTRIBUTES="downstream db2"
else
git clone $REPOSITORY .
git checkout $BRANCH
ATTRIBUTES="upstream db2"
fi
# Run connector tests
mvn clean install -U -s $HOME/.m2/settings-snapshots.xml -am -fae \
-Dmaven.test.failure.ignore=true \
-Dtest.argline="-Ddebezium.test.records.waittime=5" \
-Dinsecure.repositories=WARN \
-Papicurio \
$PROFILE_PROD
RESULTS_FOLDER=final-results
RESULTS_PATH=$RESULTS_FOLDER/results
mkdir -p $RESULTS_PATH
cp target/failsafe-reports/*.xml $RESULTS_PATH
rm -rf $RESULTS_PATH/failsafe-summary.xml
tar czf archive.tar.gz $RESULTS_PATH
docker login quay.io -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD"
curl -O https://raw.githubusercontent.com/debezium/debezium/main/jenkins-jobs/scripts/report.sh
chmod +x ./report.sh
./report.sh --connector true --env-file env-file.env --results-folder $RESULTS_FOLDER --attributes "$ATTRIBUTES"
''')
}
}

98
jenkins-jobs/job-dsl/connector_mongodb_matrix_test.groovy
View File

@ -1,98 +0,0 @@
// Job definition to test MongoDB connector against different MongoDB versions
matrixJob('connector-debezium-mongodb-matrix-test') {
displayName('Debezium MongoDB Connector Test Matrix')
description('Executes tests for MongoDB Connector with MongoDB matrix')
label('Slave')
axes {
text('MONGODB_VERSION', '4.2', '4.4', '5.0', '6.0')
label("Node", "Slave")
}
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
parameters {
stringParam('REPOSITORY', 'https://github.com/debezium/debezium', 'Repository from which Debezium is built')
stringParam('BRANCH', 'main', 'A branch/tag from which Debezium is built')
stringParam('SOURCE_URL', "", "URL to productised sources")
booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')
}
triggers {
cron('H 04 * * *')
}
wrappers {
preBuildCleanup()
timeout {
noActivity(1200)
}
credentialsBinding {
usernamePassword('QUAY_USERNAME', 'QUAY_PASSWORD', 'rh-integration-quay-creds')
string('RP_TOKEN', 'report-portal-token')
}
}
publishers {
archiveArtifacts {
pattern('**/archive.tar.gz')
}
archiveJunit('**/target/surefire-reports/*.xml')
archiveJunit('**/target/failsafe-reports/*.xml')
mailer('debezium-qe@redhat.com', false, true)
}
logRotator {
daysToKeep(7)
numToKeep(5)
}
steps {
shell('''
# Ensure WS cleaup
ls -A1 | xargs rm -rf
# Retrieve sources
if [ "$PRODUCT_BUILD" == true ] ; then
export MAVEN_OPTS="-Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true"
PROFILE_PROD="-Ppnc"
curl -OJs $SOURCE_URL && unzip debezium-*-src.zip
pushd debezium-*-src
pushd $(ls | grep -P 'debezium-[^-]+.Final')
ATTRIBUTES="downstream mongoDB $MONGODB_VERSION"
else
git clone $REPOSITORY .
git checkout $BRANCH
ATTRIBUTES="upstream mongoDB $MONGODB_VERSION"
fi
# Run maven build
mvn clean install -U -s $HOME/.m2/settings-snapshots.xml -pl debezium-bom,debezium-connector-mongodb -am -fae \
-Dmaven.test.failure.ignore=true \
-Dversion.mongo.server=$MONGODB_VERSION \
-Dinsecure.repositories=WARN \
-Dmongodb.replica.size=3 \
-Papicurio \
$PROFILE_PROD
RESULTS_FOLDER=final-results
RESULTS_PATH=$RESULTS_FOLDER/results
mkdir -p $RESULTS_PATH
cp **/target/surefire-reports/*.xml $RESULTS_PATH
cp **/target/failsafe-reports/*.xml $RESULTS_PATH
rm -rf $RESULTS_PATH/failsafe-summary.xml
tar czf archive.tar.gz $RESULTS_PATH
docker login quay.io -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD"
./jenkins-jobs/scripts/report.sh --connector true --env-file env-file.env --results-folder $RESULTS_FOLDER --attributes "$ATTRIBUTES"
''')
}
}

103
jenkins-jobs/job-dsl/connector_mysql_matrix_test.groovy
View File

@ -1,103 +0,0 @@
// Job definition to test MySQL connector against different MySQL versions
matrixJob('connector-debezium-mysql-matrix-test') {
displayName('Debezium MySQL Connector Test Matrix')
description('Executes tests for MySQL Connector with MySQL matrix')
label('Slave')
axes {
text('MYSQL_VERSION', '8.0.20', '5.7')
text('PROFILE', 'none', 'assembly')
label("Node", "Slave")
}
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
parameters {
stringParam('REPOSITORY', 'https://github.com/debezium/debezium', 'Repository from which Debezium is built')
stringParam('BRANCH', 'main', 'A branch/tag from which Debezium is built')
stringParam('SOURCE_URL', "", "URL to productised sources")
booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')
}
triggers {
cron('H 04 * * *')
}
wrappers {
preBuildCleanup()
timeout {
noActivity(1200)
}
credentialsBinding {
usernamePassword('QUAY_USERNAME', 'QUAY_PASSWORD', 'rh-integration-quay-creds')
string('RP_TOKEN', 'report-portal-token')
}
}
publishers {
archiveArtifacts {
pattern('**/archive.tar.gz')
}
archiveJunit('**/target/surefire-reports/*.xml')
archiveJunit('**/target/failsafe-reports/*.xml')
mailer('debezium-qe@redhat.com', false, true)
}
logRotator {
daysToKeep(7)
numToKeep(5)
}
steps {
shell('''
# Ensure WS cleaup
ls -A1 | xargs rm -rf
# Retrieve sources
if [ "$PRODUCT_BUILD" == true ] ; then
export MAVEN_OPTS="-Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true"
PROFILE_PROD="-Ppnc"
curl -OJs $SOURCE_URL && unzip debezium-*-src.zip
pushd debezium-*-src
pushd $(ls | grep -P 'debezium-[^-]+.Final')
ATTRIBUTES="downstream MySQL $MYSQL_VERSION $PROFILE"
else
git clone $REPOSITORY .
git checkout $BRANCH
ATTRIBUTES="upstream MySQL $MYSQL_VERSION $PROFILE"
fi
# Run maven build
mvn clean install -U -s $HOME/.m2/settings-snapshots.xml -pl debezium-bom,debezium-connector-mysql -am -fae \
-Dmaven.test.failure.ignore=true \
-Dversion.mysql.server=$MYSQL_VERSION \
-Dmysql.port=4301 \
-Dmysql.replica.port=4301 \
-Dmysql.gtid.port=4302 \
-Dmysql.gtid.replica.port=4303 \
-P$PROFILE \
-Papicurio \
-Dinsecure.repositories=WARN \
$PROFILE_PROD
RESULTS_FOLDER=final-results
RESULTS_PATH=$RESULTS_FOLDER/results
mkdir -p $RESULTS_PATH
cp **/target/surefire-reports/*.xml $RESULTS_PATH
cp **/target/failsafe-reports/*.xml $RESULTS_PATH
rm -rf $RESULTS_PATH/failsafe-summary.xml
tar czf archive.tar.gz $RESULTS_PATH
docker login quay.io -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD"
./jenkins-jobs/scripts/report.sh --connector true --env-file env-file.env --results-folder $RESULTS_FOLDER --attributes "$ATTRIBUTES"
''')
}
}

137
jenkins-jobs/job-dsl/connector_oracle_matrix.groovy
View File

@ -1,137 +0,0 @@
matrixJob('connector-debezium-oracle-matrix-test') {
displayName('Debezium Oracle Connector Test Matrix')
description('Executes tests for Oracle connector with Oracle matrix')
label('Slave')
axes {
text('ORACLE_VERSION', '21.3.0', '19.3.0', '19.3.0-noncdb', '12.2.0.1')
label("Node", "NodeXL")
}
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
parameters {
stringParam('REPOSITORY', 'https://github.com/debezium/debezium', 'Repository from which Debezium is built')
stringParam('BRANCH', 'main', 'A branch/tag from which Debezium is built')
// QUAY CONFIG
stringParam('QUAY_CREDENTIALS', 'rh-integration-quay-creds', 'Quay.io credentials id')
// PRODUCT CONFIG
stringParam('SOURCE_URL', "", "URL to productised sources")
booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')
}
triggers {
cron('H 04 * * *')
}
wrappers {
preBuildCleanup()
timeout {
noActivity(1200)
}
credentialsBinding{
usernamePassword('QUAY_USERNAME', 'QUAY_PASSWORD', 'rh-integration-quay-creds')
string('RP_TOKEN', 'report-portal-token')
}
}
publishers {
archiveArtifacts {
pattern('**/archive.tar.gz')
}
archiveJunit('**/target/surefire-reports/*.xml')
archiveJunit('**/target/failsafe-reports/*.xml')
mailer('debezium-qe@redhat.com', false, true)
}
logRotator {
daysToKeep(7)
numToKeep(5)
}
steps {
shell ('''
# Ensure WS cleaup
ls -A1 | xargs rm -rf
set -x
# Retrieve sources
if [ "$PRODUCT_BUILD" == true ] ; then
export MAVEN_OPTS="-Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true"
PROFILE_PROD="-Ppnc"
curl -OJs $SOURCE_URL && unzip debezium-*-src.zip
pushd debezium-*-src
pushd $(ls | grep -P 'debezium-[^-]+.Final')
ATTRIBUTES="downstream Oracle $ORACLE_VERSION"
else
git clone $REPOSITORY .
git checkout $BRANCH
ATTRIBUTES="upstream Oracle $ORACLE_VERSION"
fi
# Run database
docker login -u=${QUAY_USERNAME} -p=${QUAY_PASSWORD} quay.io
docker run --name oracledb -d -p 1521:1521 quay.io/rh_integration/dbz-oracle:${ORACLE_VERSION}
until docker inspect oracledb | grep '"Status": "healthy"' 2>/dev/null
do
echo "waiting for database container to start"
sleep 10
done
# Prepare tests
ORACLE_ARTIFACT_VERSION=$(mvn -s $HOME/.m2/settings-snapshots.xml -q -DforceStdout help:evaluate -Dexpression=version.oracle.driver ${PROFILE_PROD})
ORACLE_ARTIFACT_DIR="${HOME}/oracle-libs/${ORACLE_ARTIFACT_VERSION}.0"
pushd ${ORACLE_ARTIFACT_DIR}
mvn install:install-file -DgroupId=com.oracle.instantclient -DartifactId=ojdbc8 -Dversion=${ORACLE_ARTIFACT_VERSION} -Dpackaging=jar -Dfile=ojdbc8.jar
mvn install:install-file -DgroupId=com.oracle.instantclient -DartifactId=xstreams -Dversion=${ORACLE_ARTIFACT_VERSION} -Dpackaging=jar -Dfile=xstreams.jar
popd
# Run tests
if [[ ${ORACLE_VERSION} =~ .*noncdb$ ]] ; then
MVN_PROP_PDB_NAME='-Ddatabase.pdb.name='
MVN_PROP_DATABASE_NAME='-Ddatabase.dbname=ORCLCDB'
MVN_PROP_USER_NAME='dbzuser'
else
MVN_PROP_USER_NAME='c##dbzuser'
fi
mvn clean install -U -s $HOME/.m2/settings-snapshots.xml -pl debezium-connector-oracle -am -fae \\
-Poracle-tests \\
-Ddatabase.hostname=0.0.0.0 \\
-Ddatabase.admin.hostname=0.0.0.0 \\
-Dinstantclient.dir=${HOME}/oracle-libs \\
-Dmaven.test.failure.ignore=true \\
-Dinsecure.repositories=WARN \\
-Ddatabase.user=${MVN_PROP_USER_NAME} \\
${MVN_PROP_PDB_NAME} \\
${MVN_PROP_DATABASE_NAME} \\
-Papicurio \\
${PROFILE_PROD}
# Cleanup
docker stop $(docker ps -a -q) || true
docker rm $(docker ps -a -q) || true
RESULTS_FOLDER=final-results
RESULTS_PATH=$RESULTS_FOLDER/results
mkdir -p $RESULTS_PATH
cp **/target/surefire-reports/*.xml $RESULTS_PATH
cp **/target/failsafe-reports/*.xml $RESULTS_PATH
rm -rf $RESULTS_PATH/failsafe-summary.xml
tar czf archive.tar.gz $RESULTS_PATH
docker login quay.io -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD"
./jenkins-jobs/scripts/report.sh --connector true --env-file env-file.env --results-folder $RESULTS_FOLDER --attributes "$ATTRIBUTES"
''')
}
}

117
jenkins-jobs/job-dsl/connector_postgresql_matrix_test.groovy
View File

@ -1,117 +0,0 @@
// Job definition to test PostgreSQL connector against different PostgreSQL versions
matrixJob('connector-debezium-postgresql-matrix-test') {
displayName('Debezium PostgreSQL Connector Test Matrix')
description('Executes tests for PostgreSQL Connector with PostgreSQL matrix')
label('Slave')
combinationFilter('''
DECODER_PLUGIN == "decoderbufs" ||
(DECODER_PLUGIN == "pgoutput" && (POSTGRES_VERSION == "10" || POSTGRES_VERSION == "11")) ||
POSTGRES_VERSION == "12" ||
POSTGRES_VERSION == "13" ||
POSTGRES_VERSION == "14" ||
POSTGRES_VERSION == "15"
''')
axes {
text('POSTGRES_VERSION', '10', '11', '12', '13', '14', '15')
text('DECODER_PLUGIN', 'decoderbufs', 'pgoutput')
label("Node", "Slave")
}
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
parameters {
stringParam('REPOSITORY', 'https://github.com/debezium/debezium', 'Repository from which Debezium is built')
stringParam('BRANCH', 'main', 'A branch/tag from which Debezium is built')
stringParam('SOURCE_URL', "", "URL to productised sources")
booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')
}
triggers {
cron('H 04 * * *')
}
wrappers {
preBuildCleanup()
timeout {
noActivity(3600)
}
credentialsBinding {
usernamePassword('QUAY_USERNAME', 'QUAY_PASSWORD', 'rh-integration-quay-creds')
string('RP_TOKEN', 'report-portal-token')
}
}
publishers {
archiveArtifacts {
pattern('**/archive.tar.gz')
}
archiveJunit('**/target/surefire-reports/*.xml')
archiveJunit('**/target/failsafe-reports/*.xml')
mailer('debezium-qe@redhat.com', false, true)
}
logRotator {
daysToKeep(7)
numToKeep(5)
}
steps {
shell('''
# Ensure WS cleaup
ls -A1 | xargs rm -rf
# Retrieve sources
if [ "$PRODUCT_BUILD" == true ] ; then
export MAVEN_OPTS="-Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true"
PROFILE_PROD="-Ppnc"
curl -OJs $SOURCE_URL && unzip debezium-*-src.zip
pushd debezium-*-src
pushd $(ls | grep -P 'debezium-[^-]+.Final')
ATTRIBUTES="downstream PostgreSQL $POSTGRES_VERSION $DECODER_PLUGIN"
else
git clone $REPOSITORY .
git checkout $BRANCH
ATTRIBUTES="upstream PostgreSQL $POSTGRES_VERSION $DECODER_PLUGIN"
fi
# Setup pg config for Alpine distributions
if [[ $POSTGRES_VERSION =~ alpine$ ]] ; then
MAVEN_ARGS = "-Dpostgres.config.file=/usr/local/share/postgresql/postgresql.conf.sample"
else
MAVEN_ARGS="-Dnone"
fi
# Run maven build
mvn clean install -U -s $HOME/.m2/settings-snapshots.xml -pl debezium-bom,debezium-connector-postgres -am -fae \
-Dmaven.test.failure.ignore=true \
-Dpostgres.port=55432 \
-Dversion.postgres.server=$POSTGRES_VERSION \
-Ddecoder.plugin.name=$DECODER_PLUGIN \
-Dtest.argline="-Ddebezium.test.records.waittime=5" \
-Dinsecure.repositories=WARN \
-Papicurio \
$PROFILE_PROD \
$MAVEN_ARGS
RESULTS_FOLDER=final-results
RESULTS_PATH=$RESULTS_FOLDER/results
mkdir -p $RESULTS_PATH
cp **/target/surefire-reports/*.xml $RESULTS_PATH
cp **/target/failsafe-reports/*.xml $RESULTS_PATH
rm -rf $RESULTS_PATH/failsafe-summary.xml
tar czf archive.tar.gz $RESULTS_PATH
docker login quay.io -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD"
./jenkins-jobs/scripts/report.sh --connector true --env-file env-file.env --results-folder $RESULTS_FOLDER --attributes "$ATTRIBUTES"
''')
}
}

107
jenkins-jobs/job-dsl/connector_sql_server_matrix_test.groovy
View File

@ -1,107 +0,0 @@
// Job definition to test SQL Server connector against different SQL Server versions
matrixJob('connector-debezium-sqlserver-matrix-test') {
displayName('Debezium SQL Server Connector Test Matrix')
description('Executes tests for SQL Server Connector with SQL Server matrix')
label('Slave')
axes {
text('SQL_SERVER_VERSION', '2017', '2019')
label("Node", "Slave")
}
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
parameters {
stringParam('REPOSITORY', 'https://github.com/debezium/debezium', 'Repository from which Debezium is built')
stringParam('BRANCH', 'main', 'A branch/tag from which Debezium is built')
stringParam('SOURCE_URL', "", "URL to productised sources")
booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')
}
scm {
git('$REPOSITORY', '$BRANCH')
}
triggers {
cron('H 04 * * *')
}
wrappers {
preBuildCleanup()
timeout {
noActivity(1200)
}
credentialsBinding {
usernamePassword('QUAY_USERNAME', 'QUAY_PASSWORD', 'rh-integration-quay-creds')
string('RP_TOKEN', 'report-portal-token')
}
}
publishers {
archiveArtifacts {
pattern('**/archive.tar.gz')
}
archiveJunit('**/target/surefire-reports/*.xml')
archiveJunit('**/target/failsafe-reports/*.xml')
mailer('debezium-qe@redhat.com', false, true)
}
logRotator {
daysToKeep(7)
numToKeep(5)
}
steps {
shell('''
# Ensure WS cleaup
ls -A1 | xargs rm -rf
# Retrieve sources
if [ "$PRODUCT_BUILD" == true ] ; then
export MAVEN_OPTS="-Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true"
PROFILE_PROD="-Ppnc"
curl -OJs $SOURCE_URL && unzip debezium-*-src.zip
pushd debezium-*-src
pushd $(ls | grep -P 'debezium-[^-]+.Final')
ATTRIBUTES="downstream SQLserver $SQL_SERVER_VERSION"
else
git clone $REPOSITORY .
git checkout $BRANCH
ATTRIBUTES="upstream SQLserver $SQL_SERVER_VERSION"
fi
# Select image
case $SQL_SERVER_VERSION in
"2017") DATABASE_IMAGE="mcr.microsoft.com/mssql/server:2017-latest" ;;
"2019") DATABASE_IMAGE="mcr.microsoft.com/mssql/server:2019-latest" ;;
*) status=$status ;;
esac
# Run maven build
mvn clean install -U -s $HOME/.m2/settings-snapshots.xml -pl debezium-bom,debezium-connector-sqlserver -am -fae \
-Dmaven.test.failure.ignore=true \
-Ddocker.db=$DATABASE_IMAGE \
-Dinsecure.repositories=WARN \
-Papicurio \
$PROFILE_PROD
RESULTS_FOLDER=final-results
RESULTS_PATH=$RESULTS_FOLDER/results
mkdir -p $RESULTS_PATH
cp **/target/surefire-reports/*.xml $RESULTS_PATH
cp **/target/failsafe-reports/*.xml $RESULTS_PATH
rm -rf $RESULTS_PATH/failsafe-summary.xml
tar czf archive.tar.gz $RESULTS_PATH
docker login quay.io -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD"
./jenkins-jobs/scripts/report.sh --connector true --env-file env-file.env --results-folder $RESULTS_FOLDER --attributes "$ATTRIBUTES"
''')
}
}

36
jenkins-jobs/job-dsl/node_snapshot_build.groovy
View File

@ -1,36 +0,0 @@
pipelineJob('node-snapshot-build') {
displayName('Jenkins node snapshot preparation ')
description('Updates jenkins node snapshot')
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
logRotator {
numToKeep(10)
}
parameters {
// CREDENTIALS
stringParam('GITLAB_CREDENTIALS', 'gitlab-debeziumci-ssh', 'QE gitlab credentials id')
stringParam('ANSIBLE_VAULT_PASSWORD', 'ansible-vault-password', 'Password for ansible vault in used ansible playbook')
// OPENSTACK INSTANCE AND SNAPSHOT CONFIG
stringParam('CLOUD_NAME', 'openstack', 'Name of openstack cloud')
stringParam('SNAPSHOT_NAME', 'debezium-jenkins-node-centos8', 'Name of created snapshot')
stringParam('BASE_IMAGE', 'CentOS-Stream-8-x86_64-GenericCloud', 'Base image for created snapshot')
stringParam('INSTANCE_NAME', 'Ansible_temporary_instance', 'Name of created instance in Openstack')
stringParam('INSTANCE_USER', 'centos', 'User used to connect to the instance')
stringParam('KEYPAIR', 'jenkins', 'Keypair used to connect to the instance')
// ANSIBLE REPO
stringParam('ANS_GIT_SECRET', 'jenkins-node-git-repo', 'ID of secret containing repo from which ansible resources are cloned')
stringParam('ANS_GIT_REPOSITORY', '', 'Repository from which ansible resources are cloned. Use SSH format')
stringParam('ANS_GIT_BRANCH', 'master', 'A branch/tag of ansible sources')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/node_snapshot_build_pipeline.groovy'))
sandbox()
}
}
}

23
jenkins-jobs/job-dsl/node_snapshot_history_cleanup.groovy
View File

@ -1,23 +0,0 @@
pipelineJob('node-snapshot-history-cleanup') {
displayName('Jenkins node snapshot history cleanup')
description('Delete old snapshots from openstack')
logRotator {
numToKeep(10)
}
parameters {
stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium', 'Debezium repository where the ansible script is located')
stringParam('DBZ_GIT_BRANCH', '*/main', 'A branch/tag where the ansible script is located')
stringParam('SNAPSHOT_NAME', 'debezium-jenkins-node-centos8', 'Name of the snapshot')
stringParam('OPENSTACK_AUTH', 'psi-clouds-yaml', 'yaml with openstack authentication')
stringParam('CLOUD_NAME', 'openstack', 'Name of openstack cloud')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/node_snapshot_history_cleanup_pipeline.groovy'))
sandbox()
}
}
}

26
jenkins-jobs/job-dsl/ocp_aro_deployment.groovy
View File

@ -1,26 +0,0 @@
pipelineJob('ocp-aro-deployment') {
displayName('OCP ARO deployment')
description('Deploys Debezium OCP ARO on Azure cloud')
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
logRotator {
numToKeep 10
}
parameters {
//cluster config
stringParam('CLUSTER_NAME', "cluster", "Name of the ARO cluster")
stringParam('DOMAIN', "dbz", "Domain for ARO cluster installation")
stringParam('RESOURCE_GROUP', "debezium-ARO", 'name of ARO resource group')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/aro_deployment_pipeline.groovy'))
sandbox()
}
}
}

24
jenkins-jobs/job-dsl/ocp_aro_teardown.groovy
View File

@ -1,24 +0,0 @@
pipelineJob('ocp-aro-teardown') {
displayName('OCP ARO teardown')
description('Tears downs Debezium OCP ARO on Azure cloud')
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
logRotator {
numToKeep 10
}
parameters {
stringParam('CLUSTER_NAME', "cluster", "Name of the ARO cluster")
stringParam('RESOURCE_GROUP', "debezium-ARO", 'name of ARO resource group')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/aro_teardown_pipeline.groovy'))
sandbox()
}
}
}

26
jenkins-jobs/job-dsl/ocp_cluster_deployment.groovy
View File

@ -1,26 +0,0 @@
pipelineJob('ocp-cluster-deployment') {
displayName('OCP on AWS deployment')
description('Deploys/Removes Debezium OCP on AWS EC2 cloud using OSIA')
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
logRotator {
numToKeep(10)
}
parameters {
stringParam('CLUSTER_NAME', 'cluster', 'Name of OCP cluster')
stringParam('INSTALLER_VERSION', 'latest-4.10', 'Version of OCP installer')
stringParam('CLOUD', 'openstack', 'Cloud')
booleanParam('REMOVE_CLUSTER', false, 'If true, instead of cluster deployment removes said cluster')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/cluster_deployment_pipeline.groovy'))
sandbox()
}
}
}

36
jenkins-jobs/job-dsl/ocp_downstream_artifact_server_prepare.groovy
View File

@ -1,36 +0,0 @@
pipelineJob('ocp-downstream-artifact-server-prepare-job') {
displayName('Artifact Server Preparation - Downstream')
description('Prepares plugins file for artifact server')
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
logRotator {
numToKeep(10)
}
parameters {
stringParam('MAIL_TO', 'debezium-qe@redhat.com')
// QUAY CONFIG
stringParam('QUAY_CREDENTIALS', 'rh-integration-quay-creds', 'Quay.io credentials id')
stringParam('QUAY_ORGANISATION', '', 'Organisation where images are copied')
// DEBEZIUM CONFIG
stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned')
stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources')
// DEBEZIUM CONNECT IMAGE CONFIG
textParam('DBZ_CONNECTOR_ARCHIVE_URLS', '', 'List of URLs to productised Debezium connectors')
// EXTRA CONFIG
textParam('DBZ_EXTRA_LIBS', '', 'List of extra libraries added to connectors')
// IMAGE NAME
booleanParam('AUTO_TAG', true, 'Use automatically generated tag')
textParam('EXTRA_IMAGE_TAGS', 'latest', 'List of extra texts tags for multiple images')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/downstream_artifact_server_prepare_pipeline.groovy'))
sandbox()
}
}
}

38
jenkins-jobs/job-dsl/ocp_downstream_strimzi_prepare.groovy
View File

@ -1,38 +0,0 @@
pipelineJob('ocp-downstream-strimzi-prepare-job') {
displayName('Strimzi Preparation - Downstream')
description('Prepares images and deployment descriptor for AMQ Streams')
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
logRotator {
numToKeep(10)
}
parameters {
stringParam('MAIL_TO', 'debezium-qe@redhat.com')
// QUAY CONFIG
stringParam('QUAY_CREDENTIALS', 'rh-integration-quay-creds', 'Quay.io credentials id')
stringParam('QUAY_ORGANISATION', '', 'Organisation where images are copied')
// STRIMZI CONFIG
stringParam('STRZ_RESOURCES_ARCHIVE_URL', '', 'URL to productised strimzi sources')
stringParam('STRZ_RESOURCES_DEPLOYMENT_DESCRIPTOR', '060-Deployment-strimzi-cluster-operator.yaml', 'Descriptor for cluster-operator deployment')
textParam('STRZ_IMAGES', '', 'List of productised strimzi images')
// DEBEZIUM CONFIG
stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned')
stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources')
// DEBEZIUM CONNECT IMAGE CONFIG
booleanParam('DBZ_CONNECT_BUILD', true, 'Also build debezium images')
textParam('DBZ_CONNECTOR_ARCHIVE_URLS', '', 'List of URLs to productised Debezium connectors')
// EXTRA CONFIG
textParam('DBZ_EXTRA_LIBS', '', 'List of extra libraries added to connectors')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/downstream_prepare_pipeline.groovy'))
sandbox()
}
}
}

56
jenkins-jobs/job-dsl/ocp_downstream_trigger_all_prepare.groovy
View File

@ -1,56 +0,0 @@
pipelineJob('ocp-downstream-trigger-all-prepare') {
displayName('Trigger Preparations - Downstream')
description('Triggers downstream preparation jobs')
parameters {
stringParam('LABEL', '', 'Optional label for the preparation jobs')
booleanParam('EXECUTE_AS', false, 'Execute downstream Artifact Server prepare')
booleanParam('EXECUTE_STRIMZI', false, 'Execute AMQ Stream Deployment Preparation')
booleanParam('EXECUTE_APICURIO', false, 'Execute Apicurio deployment preparation')
booleanParam('EXECUTE_RHEL', false, 'Execute AMQ Streams on RHEL Preparation')
stringParam('MAIL_TO', 'debezium-qe@redhat.com')
// QUAY CONFIG
stringParam('QUAY_CREDENTIALS', 'rh-integration-quay-creds', 'Quay.io credentials id')
stringParam('QUAY_ORGANISATION', '', 'Organisation where images are copied')
// DEBEZIUM CONFIG
stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned')
stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources')
stringParam('DBZ_GIT_REPOSITORY_DB2', 'https://github.com/debezium/debezium-connector-db2.git', 'Repository from which Debezium DB2 sources are cloned')
stringParam('DBZ_GIT_BRANCH_DB2', 'main', 'A branch/tag of Debezium DB2 sources')
// OCP STRIMZI
stringParam('STRZ_RESOURCES_ARCHIVE_URL', '', 'OCP Downstream strimzi - URL to productised strimzi sources')
stringParam('STRZ_RESOURCES_DEPLOYMENT_DESCRIPTOR', '060-Deployment-strimzi-cluster-operator.yaml', 'OCP Downstream strimzi - Descriptor for cluster-operator deployment')
textParam('STRZ_IMAGES', '', 'OCP Downstream strimzi - List of productised strimzi images')
// DEBEZIUM CONNECT IMAGE CONFIG
booleanParam('STRZ_DBZ_CONNECT_BUILD', true, 'OCP Downstream strimzi - Also build debezium images')
textParam('DBZ_CONNECTOR_ARCHIVE_URLS', '', 'OCP Downstream strimzi, Artifact Server - List of URLs to productised Debezium connectors')
textParam('STRZ_DBZ_EXTRA_LIBS', '', 'Downstream Strimzi, RHEL - List of extra libraries added to connectors')
// APICURIO
stringParam('APIC_RESOURCES_ARCHIVE_URL', '', 'URL to productised apicurio sources')
stringParam('APIC_RESOURCES_DEPLOYMENT_DESCRIPTOR', 'install.yaml', 'Descriptor for deployment')
booleanParam('PUSH_IMAGES', true, 'Apicurio - Push images to quay.io')
textParam('APIC_IMAGES', '', 'List of productised apicurio images')
// RHEL
booleanParam('AUTO_TAG', true, 'RHEL, Artifact Server - Use automatically generated tag')
textParam('EXTRA_IMAGE_TAGS', 'latest', 'RHEL - List of extra texts tags for multiple images')
stringParam('RHEL_IMAGE', 'registry.access.redhat.com/ubi8:latest', 'Base RHEL image')
stringParam('KAFKA_URL', '', 'RHEL - AMQ streams kafka')
// ARTIFACT SERVER
textParam('AS_DBZ_EXTRA_LIBS', '', 'Artifact Server - List of extra libraries added to connectors')
textParam('AS_EXTRA_IMAGE_TAGS', 'latest', 'Artifact Server - List of extra texts tags for multiple images')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/downstream_preparations_trigger_pipeline.groovy'))
sandbox()
}
}
}

39
jenkins-jobs/job-dsl/ocp_system_test.groovy
View File

@ -1,39 +0,0 @@
pipelineJob('ocp-system-test') {
displayName('System Tests inside OCP')
description('Runs System Tests inside OCP')
parameters {
stringParam('MAIL_TO', 'debezium-qe@redhat.com')
booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')
stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned')
stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources')
stringParam('PULL_SECRET', 'rh-integration-quay-secret', 'Quay.io OpenShift secret')
stringParam('DOCKER_TAG', 'latest', 'Docker image tag')
stringParam('OCP_CREDENTIALS', 'openshift-dbz-creds', 'Jenkins credentials id')
stringParam('OCP_URL', '', 'Ocp url')
booleanParam('TEST_APICURIO_REGISTRY', false, 'Run tests with Apicurio Registry and Avro serialization')
booleanParam('STRIMZI_KC_BUILD', false, 'True -> use artifact server, false -> dbz connect image')
stringParam('DBZ_CONNECT_IMAGE', '', 'Debezium connect image')
stringParam('ARTIFACT_SERVER_IMAGE', '', 'Artifact server image')
stringParam('APICURIO_VERSION', '2.2.3.Final', 'Apicurio version')
stringParam('KAFKA_VERSION', '', 'Kafka version')
stringParam('STRIMZI_PREPARE_BUILD_NUMBER', '', 'Downstream preparation build to obtain AMQ streams operator from.' +
' Leave empty to install operator from ocp marketplace')
stringParam('STRZ_CHANNEL', 'stable', 'Update channel for Strimzi operator')
stringParam('APIC_CHANNEL', '2.x', 'Update channel for Apicurio operator')
stringParam('TEST_TAGS', '', 'Which test tags to run (empty for all)')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/ocp_system_pipeline.groovy'))
sandbox()
}
}
}

25
jenkins-jobs/job-dsl/ocp_system_test_image_prepare.groovy
View File

@ -1,25 +0,0 @@
pipelineJob('ocp-debezium-testing-system-image-prepare') {
displayName('Debezium TestSuite Image Prepare')
description('Creates a testsuite docker image and uploads it to quay')
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
logRotator {
numToKeep(10)
}
parameters {
stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned into docker image')
stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources')
stringParam('TAG', 'latest', 'Docker image tag')
stringParam('QUAY_CREDENTIALS', 'rh-integration-quay-creds', 'Quay.io credentials id')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/system_test_image_pipeline.groovy'))
}
}
}

39
jenkins-jobs/job-dsl/ocp_upstream_artifact_server_prepare.groovy
View File

@ -1,39 +0,0 @@
pipelineJob('ocp-upstream-artifact-server-prepare-job') {
displayName('Artifact Server Preparation - Upstream')
description('Prepares plugins file for artifact server')
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
logRotator {
numToKeep(10)
}
parameters {
stringParam('MAIL_TO', 'debezium-qe@redhat.com')
// QUAY CONFIG
stringParam('QUAY_CREDENTIALS', 'debezium-quay-creds', 'Quay.io credentials id')
stringParam('QUAY_ORGANISATION', 'debezium', 'Organisation where images are copied')
// DEBEZIUM CONFIG
stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned')
stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources')
stringParam('DBZ_GIT_REPOSITORY_DB2', 'https://github.com/debezium/debezium-connector-db2.git', 'Repository from which Debezium DB2 sources are cloned')
stringParam('DBZ_GIT_BRANCH_DB2', 'main', 'A branch/tag of Debezium DB2 sources')
// IMAGE NAME
booleanParam('AUTO_TAG', true, 'Use automatically generated tag')
textParam('EXTRA_IMAGE_TAGS', 'latest', 'List of extra texts tags for multiple images')
// COMPONENT VERSIONS
stringParam('APICURIO_VERSION', '2.1.0.Final', 'Service registry bits version')
// ORACLE INCLUSION
booleanParam('ORACLE_INCLUDED', false, 'Should Oracle connector be included in image')
stringParam('PRIVATE_QUAY_CREDENTIALS', 'rh-integration-quay-creds', 'Quay.io credentials id to private repo')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/upstream_artifact_server_prepare_pipeline.groovy'))
sandbox()
}
}
}

10
jenkins-jobs/job-dsl/deploy_debezium_tool_images.groovy → jenkins-jobs/job-dsl/release/deploy_debezium_tool_images.groovy
View File

@ -1,4 +1,10 @@
pipelineJob('release-deploy-debezium-tool-images') {
folder("release") {
description("This folder contains all jobs used by developers for upstream release and all relevant stuff")
displayName("Release")
}
pipelineJob('release/release-deploy-debezium-tool-images') {
displayName('Debezium Deploy Tool Images')
description('Build and deploy debezium tool images to the registry')
@ -24,7 +30,7 @@ pipelineJob('release-deploy-debezium-tool-images') {
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/build_debezium_tool_images_pipeline.groovy'))
script(readFileFromWorkspace('jenkins-jobs/pipelines/release/build_debezium_tool_images_pipeline.groovy'))
}
}
}

9
jenkins-jobs/job-dsl/deploy_docker_images.groovy → jenkins-jobs/job-dsl/release/deploy_docker_images.groovy
View File

@ -1,4 +1,9 @@
pipelineJob('release-deploy-container-images') {
folder("release") {
description("This folder contains all jobs used by developers for upstream release and all relevant stuff")
displayName("Release")
}
pipelineJob('release/release-deploy-container-images') {
displayName('Debezium Deploy Container Images')
description('Build and deploy Container images to the registry')
@ -28,7 +33,7 @@ pipelineJob('release-deploy-container-images') {
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/build_debezium_images_pipeline.groovy'))
script(readFileFromWorkspace('jenkins-jobs/pipelines/release/build_debezium_images_pipeline.groovy'))
}
}
}

7
jenkins-jobs/job-dsl/maven_artifact_check.groovy → jenkins-jobs/job-dsl/release/maven_artifact_check.groovy
View File

@ -1,4 +1,9 @@
freeStyleJob('prepare-maven-artifact-check') {
folder("release") {
description("This folder contains all jobs used by developers for upstream release and all relevant stuff")
displayName("Release")
}
freeStyleJob('release/prepare-maven-artifact-check') {
displayName('Maven artifact check')
description('Verify integrity of maven artifact')
label('Slave')

6
jenkins-jobs/job-dsl/release_debezium_nightly_image.groovy → jenkins-jobs/job-dsl/release/release_debezium_nightly_image.groovy
View File

@ -1,6 +1,10 @@
// Job definition to test PostgreSQL connector against different PostgreSQL versions
folder("release") {
description("This folder contains all jobs used by developers for upstream release and all relevant stuff")
displayName("Release")
}
freeStyleJob('release-debezium-nightly-image') {
freeStyleJob('release/release-debezium-nightly-image') {
displayName('Debezium Nightly Image')
description('Build and deploy nightly image to the registry')

9
jenkins-jobs/job-dsl/release_debezium_stage_prod.groovy → jenkins-jobs/job-dsl/release/release_debezium_stage_prod.groovy
View File

@ -1,4 +1,9 @@
pipelineJob('release-debezium-stage-prod') {
folder("release") {
description("This folder contains all jobs used by developers for upstream release and all relevant stuff")
displayName("Release")
}
pipelineJob('release/release-debezium-stage-prod') {
displayName('Debezium Build Product Artifacts')
description('Builds and uploads product artifacts into a stage location')
@ -28,7 +33,7 @@ pipelineJob('release-debezium-stage-prod') {
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/prod-release-pipeline.groovy'))
script(readFileFromWorkspace('jenkins-jobs/pipelines/release/prod-release-pipeline.groovy'))
}
}
}

9
jenkins-jobs/job-dsl/release_deploy_snapshots.groovy → jenkins-jobs/job-dsl/release/release_deploy_snapshots.groovy
View File

@ -1,4 +1,9 @@
pipelineJob('release-deploy_snapshots_pipeline') {
folder("release") {
description("This folder contains all jobs used by developers for upstream release and all relevant stuff")
displayName("Release")
}
pipelineJob('release/release-deploy_snapshots_pipeline') {
displayName('Debezium Deploy Snapshots')
description('Deploy -SNAPSHOT versions to Maven Central')
@ -29,7 +34,7 @@ pipelineJob('release-deploy_snapshots_pipeline') {
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/deploy_snapshots_pipeline.groovy'))
script(readFileFromWorkspace('jenkins-jobs/pipelines/release/deploy_snapshots_pipeline.groovy'))
}
}
}

9
jenkins-jobs/job-dsl/release_upstream.groovy → jenkins-jobs/job-dsl/release/release_upstream.groovy
View File

@ -1,4 +1,9 @@
pipelineJob('release-debezium-upstream') {
folder("release") {
description("This folder contains all jobs used by developers for upstream release and all relevant stuff")
displayName("Release")
}
pipelineJob('release/release-debezium-upstream') {
displayName('Debezium Release')
description('Builds Debezium and deploys into Maven Central and Docker Hub')
@ -38,7 +43,7 @@ pipelineJob('release-debezium-upstream') {
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/release-pipeline.groovy'))
script(readFileFromWorkspace('jenkins-jobs/pipelines/release/release-pipeline.groovy'))
}
}
}

6
jenkins-jobs/job-dsl/tools_source_clear.groovy → jenkins-jobs/job-dsl/release/tools_source_clear.groovy
View File

@ -1,6 +1,10 @@
// Job definition to test PostgreSQL connector against different PostgreSQL versions
folder("release") {
description("This folder contains all jobs used by developers for upstream release and all relevant stuff")
displayName("Release")
}
freeStyleJob('tools-debezium-source-clear') {
freeStyleJob('release/tools-debezium-source-clear') {
displayName('Vulnerability scan')
description('Executes SourceClear vulnerability scan of Debezium sources and binaries')

54
jenkins-jobs/job-dsl/system_test.groovy
View File

@ -1,54 +0,0 @@
pipelineJob('ocp-debezium-testing-system') {
displayName('Debezium System-level TestSuite')
description('Executes tests for OpenShift & Strimzi compatibility verification')
properties {
githubProjectUrl('https://github.com/debezium/debezium')
}
logRotator {
numToKeep(10)
}
parameters {
stringParam('MAIL_TO', 'debezium-qe@redhat.com')
booleanParam('PRODUCT_BUILD', false, 'Is this a productised build?')
booleanParam('TEST_APICURIO_REGISTRY', false, 'Run tests with Apicurio Registry and Avro serialization')
// OCP CONFIG
stringParam('OCP_URL', '', 'OpenShift admin API url')
stringParam('OCP_CREDENTIALS', 'openshift-dbz-creds', 'Jenkins credentials id')
// QUAY CONFIG
stringParam('QUAY_CREDENTIALS', 'debezium-quay-creds', 'Quay.io credentials id')
// PULL SECRET
stringParam('PULL_SECRET', 'rh-integration-quay-secret', 'Quay.io OpenShift secret')
// DEBEZIUM CONFIG
stringParam('DBZ_GIT_REPOSITORY', 'https://github.com/debezium/debezium.git', 'Repository from which Debezium sources are cloned')
stringParam('DBZ_GIT_BRANCH', 'main', 'A branch/tag of Debezium sources')
stringParam('DBZ_GIT_REPOSITORY_DB2', 'https://github.com/debezium/debezium-connector-db2.git', 'Repository from which Debezium DB2 sources are cloned')
stringParam('DBZ_GIT_BRANCH_DB2', 'main', 'A branch/tag of Debezium DB2 sources')
// OPERATORS CONFIG
stringParam('STRIMZI_PREPARE_BUILD_NUMBER', '', 'Downstream preparation build to obtain AMQ streams operator from.' +
' Leave empty to install operator from ocp marketplace')
stringParam('STRIMZI_OPERATOR_CHANNEL', 'stable', 'Update channel for Strimzi operator')
stringParam('APICURIO_OPERATOR_CHANNEL', '2.x', 'Update channel for Apicurio operator')
// Images config
stringParam('IMAGE_DBZ_AS', '', "Debezium artifact server image (usable with Strimzi's build mechanism")
stringParam('IMAGE_CONNECT_STRZ', '', 'Kafka Connect Strimzi Image with DBZ plugins.')
stringParam('IMAGE_CONNECT_RHEL', '', 'Kafka Connect RHEL Image with DBZ plugins.')
// TEST CONFIG
stringParam('TEST_WAIT_SCALE', '1', 'Wait time scaling factor')
stringParam('TEST_TAGS', '', 'Which test tags to run (empty for all)')
// Artifact Versions
stringParam('TEST_VERSION_KAFKA', '', 'Kafka version')
stringParam('AS_VERSION_APICURIO', '', 'Service registry bits version')
stringParam('AS_VERSION_DEBEZIUM', '', 'Debezium bits version')
// Maven
booleanParam('DEBUG_MODE', false, 'Enable remote debugger')
}
definition {
cps {
script(readFileFromWorkspace('jenkins-jobs/pipelines/system_pipeline.groovy'))
}
}
}

30
jenkins-jobs/job-dsl/view_dashboard.groovy
View File

@ -1,30 +0,0 @@
dashboardView('Dashboard') {
description('Debezium job dashboard')
jobs {
regex(/^connector-.*$/)
name('ocp-debezium-openshift-test')
}
columns {
status()
weather()
name()
lastSuccess()
lastFailure()
lastDuration()
buildButton()
}
topPortlets {
jenkinsJobsList()
}
leftPortlets {
testStatisticsChart()
}
rightPortlets {
testTrendChart()
}
bottomPortlets {
buildStatistics()
}
}

15
jenkins-jobs/job-dsl/view_list_connector.groovy
View File

@ -1,15 +0,0 @@
listView('Debezium-Connector-Tests') {
description('Upstream connector jobs')
jobs {
regex(/^connector-.*$/)
}
columns {
status()
weather()
name()
lastSuccess()
lastFailure()
lastDuration()
buildButton()
}
}

18
jenkins-jobs/job-dsl/view_list_jenkins.groovy
View File

@ -1,18 +0,0 @@
listView('Jenkins') {
description('Jenkins tools')
jobs {
name('job-configurator')
name('MonitoringTest')
name('node-snapshot-build')
name('node-snapshot-history-cleanup')
}
columns {
status()
weather()
name()
lastSuccess()
lastFailure()
lastDuration()
buildButton()
}
}

15
jenkins-jobs/job-dsl/view_list_release.groovy
View File

@ -1,15 +0,0 @@
listView('Debezium-Release-Automation') {
description('Debezium release jobs')
jobs {
regex(/^release-.*$/)
}
columns {
status()
weather()
name()
lastSuccess()
lastFailure()
lastDuration()
buildButton()
}
}

15
jenkins-jobs/job-dsl/view_list_system_tests.groovy
View File

@ -1,15 +0,0 @@
listView('Debezium-System-Tests') {
description('OpenShift certification jobs')
jobs {
regex(/^ocp-.*$|^rhel-.*$|^system-.*$|^prepare-.*$/)
}
columns {
status()
weather()
name()
lastSuccess()
lastFailure()
lastDuration()
buildButton()
}
}

15
jenkins-jobs/job-dsl/view_list_tools.groovy
View File

@ -1,15 +0,0 @@
listView('Debezium-Tools') {
description('Tooling jobs')
jobs {
regex(/^tools-.*$/)
}
columns {
status()
weather()
name()
lastSuccess()
lastFailure()
lastDuration()
buildButton()
}
}

141
jenkins-jobs/pipelines/aro_deployment_pipeline.groovy
View File

@ -1,141 +0,0 @@
pipeline {
agent {
label 'debezium-ci-tools'
}
environment {
ARO_TEMPLATE_PATH = "${env.WORKSPACE}/ARO-dbz/aro-configuration/template.json"
ARO_PARAMETERS_PATH = "${env.WORKSPACE}/ARO-dbz/aro-configuration/parameters.json"
ARO_GIT_SECRET = "ocp-deployment-repo"
ARO_GIT_BRANCH = "main"
GITLAB_CREDENTIALS = "gitlab-debeziumci-ssh"
SR_CREDENTIALS = "aro-service-account"
PULL_SECRET = "ocp-pull-secret-json"
ANSIBLE_VAULT_PASSWORD = "ansible-vault-password"
}
stages {
stage("Checkout ARO configuration repo") {
steps {
container("debezium-ci-tools") {
script {
withCredentials([
string(credentialsId: "${ARO_GIT_SECRET}", variable: 'TMP_ARO_GIT_REPOSITORY')
]) { env.ARO_GIT_REPOSITORY = TMP_ARO_GIT_REPOSITORY }
}
checkout([
$class : 'GitSCM',
branches : [[name: "${env.ARO_GIT_BRANCH}"]],
userRemoteConfigs : [[url: "${env.ARO_GIT_REPOSITORY}",
credentialsId: "${GITLAB_CREDENTIALS}"]],
extensions : [[$class: 'CleanCheckout'],
[$class: 'RelativeTargetDirectory',
relativeTargetDir: 'ARO-dbz']] +
[[$class: 'CloneOption', noTags: false, depth: 1, reference: '', shallow: true]],
submoduleCfg : [],
doGenerateSubmoduleConfigurations: false,
])
}
}
}
stage('Login to Azure') {
steps {
container("debezium-ci-tools") {
withCredentials([
usernamePassword(credentialsId: "${SR_CREDENTIALS}", passwordVariable: 'password', usernameVariable: 'username')
]) {
sh('az login --service-principal -u ${username} -p ${password} --tenant 520cf09d-78ff-44ed-a731-abd623e73b09')
}
}
}
}
stage('Install cluster') {
steps {
container("debezium-ci-tools") {
script {
withCredentials([
usernamePassword(credentialsId: "${SR_CREDENTIALS}", passwordVariable: 'password', usernameVariable: 'username'),
file(credentialsId: "${PULL_SECRET}", variable: 'secret_path')
]) {
sh '''
az group create -l eastus -n ${RESOURCE_GROUP}
az deployment group create \
--name ${CLUSTER_NAME} \
--template-file ${ARO_TEMPLATE_PATH} \
--parameters ${ARO_PARAMETERS_PATH} \
--resource-group ${RESOURCE_GROUP} \
--parameters aadClientSecret=${password} pullSecret="$(cat ${secret_path})" \
clusterName=${CLUSTER_NAME} domain=${DOMAIN} aadClientId=${username}
'''
}
}
}
}
}
stage('Get cluster description') {
steps {
container("debezium-ci-tools") {
script {
sh("az aro show --name ${env.CLUSTER_NAME} --resource-group ${env.RESOURCE_GROUP} > aro_info.txt")
}
}
}
}
stage('Get admin credentials and URLs') {
steps {
container("debezium-ci-tools") {
script {
env.ADMIN_USER = "kubeadmin"
env.ADMIN_PASS = sh(script: "az aro list-credentials --name ${env.CLUSTER_NAME} " +
"--resource-group ${env.RESOURCE_GROUP} | jq -r '.kubeadminPassword'", returnStdout: true).trim()
env.API_URL = sh(script: "cat aro_info.txt | jq -r '.apiserverProfile.url'", returnStdout: true).trim()
env.CONSOLE_URL = sh(script: "cat aro_info.txt | jq -r '.consoleProfile.url'", returnStdout: true).trim()
println("[INFO] username: ${env.ADMIN_USER}")
println("[INFO] password: ${env.ADMIN_PASS}")
println("[INFO] API url: ${env.API_URL}")
println("[INFO] Console url: ${env.CONSOLE_URL}")
}
}
}
}
stage('Setup Debezium credentials') {
steps {
container("debezium-ci-tools") {
withCredentials([
string(credentialsId: "${env.ANSIBLE_VAULT_PASSWORD}", variable: 'ANSIBLE_PASSWORD')
]) {
sh '''
set -ex
cd "${WORKSPACE}/ARO-dbz/secrets"
echo "${ANSIBLE_PASSWORD}" > ../password.txt
ansible-vault decrypt --vault-password-file ../password.txt *
cd ..
mv ./secrets/* ./
oc login ${API_URL} -u ${ADMIN_USER} -p ${ADMIN_PASS} --insecure-skip-tls-verify=true
oc create secret generic htpass-secret --from-file=htpasswd=ocp-users.htpasswd -n openshift-config
oc apply -f htpasswd.cr.yaml -n openshift-config
oc adm policy add-cluster-role-to-user cluster-admin debezium
'''
}
}
}
}
}
post {
failure {
build job: 'ocp-aro-teardown', parameters: [
string(name: 'CLUSTER_NAME', value: params.CLUSTER_NAME),
string(name: 'RESOURCE_GROUP', value: params.RESOURCE_GROUP),
]
}
always {
script {
mail to: 'debezium-qe@redhat.com', subject: "ARO cluster deployment #${env.BUILD_NUMBER} finished with ${currentBuild.currentResult}",
body: """
${currentBuild.projectName} run ${env.BUILD_URL} finished with result: ${currentBuild.currentResult}
"""
archiveArtifacts "**/aro_info.txt"
}
}
}
}

45
jenkins-jobs/pipelines/aro_teardown_pipeline.groovy
View File

@ -1,45 +0,0 @@
pipeline {
agent {
label 'debezium-ci-tools'
}
environment {
ARO_TEMPLATE_PATH = "${env.WORKSPACE}/ARO-dbz/aro-configuration/template.json"
ARO_PARAMETERS_PATH = "${env.WORKSPACE}/ARO-dbz/aro-configuration/parameters.json"
SR_CREDENTIALS = "aro-service-account"
}
stages {
stage('Login to Azure') {
steps {
container("debezium-ci-tools") {
withCredentials([
usernamePassword(credentialsId: "${SR_CREDENTIALS}", passwordVariable: 'password', usernameVariable: 'username')
]) {
sh('az login --service-principal -u ${username} -p ${password} --tenant 520cf09d-78ff-44ed-a731-abd623e73b09')
}
}
}
}
stage('Delete cluster') {
steps {
container("debezium-ci-tools") {
script {
sh("az aro delete --name ${env.CLUSTER_NAME} --resource-group ${env.RESOURCE_GROUP} -y")
sh("az network vnet delete --name dbz-ARO-vnet --resource-group ${env.RESOURCE_GROUP}")
sh("az network watcher configure --resource-group NetworkWatcherRG --locations eastus --enabled false")
}
}
}
}
}
post {
always {
script {
mail to: 'debezium-qe@redhat.com', subject: "ARO cluster teardown #${env.BUILD_NUMBER} finished with ${currentBuild.currentResult}",
body: """
${currentBuild.projectName} run ${env.BUILD_URL} finished with result: ${currentBuild.currentResult}
"""
}
}
}
}

122
jenkins-jobs/pipelines/cluster_deployment_pipeline.groovy
View File

@ -1,122 +0,0 @@
pipeline {
environment {
OSIA_GIT_SECRET = "ocp-deployment-repo"
OSIA_GIT_BRANCH = "persistence"
ANSIBLE_VAULT_PASSWORD = "ansible-vault-password"
GITLAB_CREDENTIALS = "gitlab-debeziumci-ssh"
}
agent {
label 'debezium-ci-tools'
}
stages {
stage('Checkout and prepare git') {
steps {
container("debezium-ci-tools") {
script {
withCredentials([
string(credentialsId: "${env.OSIA_GIT_SECRET}", variable: 'OSIA_GIT_REPOSITORY')
]) {
checkout([
$class : 'GitSCM',
branches : [[name: "${env.OSIA_GIT_BRANCH}"]],
userRemoteConfigs: [[url: "${OSIA_GIT_REPOSITORY}",
credentialsId: "${env.GITLAB_CREDENTIALS}"]],
extensions : [[$class: 'CleanCheckout'],
[$class: 'RelativeTargetDirectory',
relativeTargetDir: 'OSIA-dbz']] +
[[$class: 'CloneOption', noTags: false, depth: 1, reference: '', shallow: true]],
submoduleCfg : [],
doGenerateSubmoduleConfigurations: false,
])
}
}
script {
withCredentials([
sshUserPrivateKey(credentialsId: "${env.GITLAB_CREDENTIALS}", keyFileVariable: 'TMP_SSH_KEY_FILE')
]){
sh '''
cd "${WORKSPACE}/OSIA-dbz/"
ls
git status
git config --local user.name "Debezium CI"
git config --local user.email "debezium-qe@redhat.com"
cp ${TMP_SSH_KEY_FILE} SSH_KEY_FILE
git config --add --local core.sshCommand "ssh -i $(readlink -f SSH_KEY_FILE)"
git checkout ${OSIA_GIT_BRANCH}
'''
}
}
}
}
}
stage("Setup AWS credentials") {
steps {
container("debezium-ci-tools") {
withCredentials([string(credentialsId: "awscredentials", variable: "AWS_SHARED_CREDENTIALS_FILE")]) {
sh(script: "mkdir -p /home/jenkins/.aws")
sh(script: "echo -e \"${AWS_SHARED_CREDENTIALS_FILE}\" > /home/jenkins/.aws/credentials")
}
}
}
}
stage('Deploy_cluster') {
when() {
expression { !params.REMOVE_CLUSTER }
}
steps {
container("debezium-ci-tools") {
withCredentials([
string(credentialsId: "${env.ANSIBLE_VAULT_PASSWORD}", variable: 'ANSIBLE_PASSWORD')
]) {
sh '''
set -ex
cd "${WORKSPACE}/OSIA-dbz/secrets"
echo "${ANSIBLE_PASSWORD}" > ../password.txt
ansible-vault decrypt --vault-password-file ../password.txt *
cd ..
mv ./secrets/* ./
if [[ $CLOUD == "openstack" ]] ; then export CLOUD_ENV=" --cloud-env psi" ; else export CLOUD_ENV="" ; fi
if [[ $CLOUD == "openstack" ]] ; then export DNS_PROVIDER="--dns-provider route53" ; else export DNS_PROVIDER="" ; fi
export AWS_SHARED_CREDENTIALS_FILE=/home/jenkins/.aws/credentials && osia install --cluster-name ${CLUSTER_NAME} --cloud ${CLOUD} --installer-version ${INSTALLER_VERSION} ${DNS_PROVIDER} ${CLOUD_ENV}
export KUBECONFIG="${WORKSPACE}/OSIA-dbz/${CLUSTER_NAME}/auth/kubeconfig"
oc create secret generic htpass-secret --from-file=htpasswd=ocp-users.htpasswd -n openshift-config
oc apply -f htpasswd.cr.yaml -n openshift-config
oc adm policy add-cluster-role-to-user cluster-admin debezium
'''
}
}
}
}
stage('Remove_cluster') {
when() {
expression { params.REMOVE_CLUSTER }
}
steps {
container("debezium-ci-tools") {
withCredentials([
string(credentialsId: "${env.ANSIBLE_VAULT_PASSWORD}", variable: 'ANSIBLE_PASSWORD')
]) {
sh '''
set -ex
cd "${WORKSPACE}/OSIA-dbz/secrets"
echo "${ANSIBLE_PASSWORD}" > ../password.txt
ansible-vault decrypt --vault-password-file ../password.txt *
cd ..
mv ./secrets/* ./
export AWS_SHARED_CREDENTIALS_FILE=/home/jenkins/.aws/credentials && osia clean --cluster-name ${CLUSTER_NAME} --installer-version ${INSTALLER_VERSION}
'''
}
}
}
}
}
post {
always {
mail to: 'debezium-qe@redhat.com', subject: "OCP cluster deployment/removal #${env.BUILD_NUMBER} finished with ${currentBuild.currentResult}", body: """
${currentBuild.projectName} run ${env.BUILD_URL} finished with result: ${currentBuild.currentResult}
"""
}
}
}

143
jenkins-jobs/pipelines/connector_tests_trigger_pipeline.groovy
View File

@ -1,143 +0,0 @@
pipeline {
agent {
label 'Slave'
}
stages {
stage('Start') {
parallel {
stage('Invoke_db2') {
when {
expression { params.DB2_TEST }
}
steps {
build job: 'connector-debezium-db2-matrix-test', parameters: [
string(name: 'REPOSITORY_DB2', value: params.REPOSITORY_DB2),
string(name: 'BRANCH', value: params.BRANCH),
string(name: 'SOURCE_URL', value: params.SOURCE_URL),
booleanParam(name: 'PRODUCT_BUILD', value: params.PRODUCT_BUILD)
]
}
}
stage('Invoke_mongodb') {
when {
expression { params.MONGODB_TEST }
}
steps {
build job: 'connector-debezium-mongodb-matrix-test', parameters: [
string(name: 'REPOSITORY_CORE', value: params.REPOSITORY_CORE),
string(name: 'BRANCH', value: params.BRANCH),
string(name: 'SOURCE_URL', value: params.SOURCE_URL),
booleanParam(name: 'PRODUCT_BUILD', value: params.PRODUCT_BUILD)
]
}
}
stage('Invoke_mysql') {
when {
expression { params.MYSQL_TEST }
}
steps {
build job: 'connector-debezium-mysql-matrix-test', parameters: [
string(name: 'REPOSITORY_CORE', value: params.REPOSITORY_CORE),
string(name: 'BRANCH', value: params.BRANCH),
string(name: 'SOURCE_URL', value: params.SOURCE_URL),
booleanParam(name: 'PRODUCT_BUILD', value: params.PRODUCT_BUILD),
]
}
}
stage('Invoke_oracle') {
when {
expression { params.ORACLE_TEST }
}
steps {
build job: 'connector-debezium-oracle-matrix-test', parameters: [
string(name: 'QUAY_CREDENTIALS', value: params.QUAY_CREDENTIALS),
string(name: 'REPOSITORY_CORE', value: params.REPOSITORY_CORE),
string(name: 'BRANCH', value: params.BRANCH),
string(name: 'SOURCE_URL', value: params.SOURCE_URL),
booleanParam(name: 'PRODUCT_BUILD', value: params.PRODUCT_BUILD)
]
}
}
stage('Invoke_postgresql') {
when {
expression { params.POSTGRESQL_TEST }
}
steps {
build job: 'connector-debezium-postgresql-matrix-test', parameters: [
string(name: 'REPOSITORY_CORE', value: params.REPOSITORY_CORE),
string(name: 'BRANCH', value: params.BRANCH),
string(name: 'SOURCE_URL', value: params.SOURCE_URL),
booleanParam(name: 'PRODUCT_BUILD', value: params.PRODUCT_BUILD)
]
}
}
stage('Invoke_sqlserver') {
when {
expression { params.SQLSERVER_TEST }
}
steps {
build job: 'connector-debezium-sqlserver-matrix-test', parameters: [
string(name: 'REPOSITORY_CORE', value: params.REPOSITORY_CORE),
string(name: 'BRANCH', value: params.BRANCH),
string(name: 'SOURCE_URL', value: params.SOURCE_URL),
booleanParam(name: 'PRODUCT_BUILD', value: params.PRODUCT_BUILD)
]
}
}
}
}
}
post {
always {
script {
def version = ""
for (db in ['db2', 'mongodb', 'mysql', 'oracle', 'postgresql', 'sqlserver']) {
if (!params["${db.toUpperCase()}_TEST"]) {
continue
}
def jobName = "connector-debezium-${db}-matrix-test"
def build = jenkins.model.Jenkins.instance.getItem(jobName).getLastCompletedBuild()
if (!build) {
println "No build of ${jobName} found!"
return
}
label = "#${build.number} parent: #${currentBuild.number}"
if (params.LABEL) {
label += " ${params.LABEL}"
}
// if label param is not set and product build, try parsing label from source url/branch
else if (params.PRODUCT_BUILD) {
def versionMatch = params.SOURCE_URL =~ /.*\/debezium-(.+)-src.zip$/
if (!version && versionMatch && versionMatch[0][1]) {
version = versionMatch[0][1].toString()
label += " version: ${version}"
} else {
println "Debezium version of product build couldn't be parsed from SOURCE_URL"
continue
}
}
// set label
build.displayName = label
}
// set parent job label
if (params.LABEL) {
currentBuild.displayName = "#${currentBuild.number} ${params.LABEL}"
} else if (version) {
currentBuild.displayName = "#${currentBuild.number} version: ${version}"
}
// if not a product build and no custom label set, keep default parent build name
}
}
}
}

60
jenkins-jobs/pipelines/downstream_artifact_server_prepare_pipeline.groovy
View File

@ -1,60 +0,0 @@
pipeline {
agent {
label 'Slave'
}
stages {
stage('CleanWorkspace') {
steps {
cleanWs()
}
}
stage('Checkout') {
steps {
checkout([
$class : 'GitSCM',
branches : [[name: "${DBZ_GIT_BRANCH}"]],
userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]],
extensions : [[$class : 'RelativeTargetDirectory',
relativeTargetDir: 'debezium']],
])
}
}
stage('Process connectors and extra libs') {
steps {
withCredentials([
usernamePassword(credentialsId: "${QUAY_CREDENTIALS}", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'),
]) {
sh '''
set -x
cd "${WORKSPACE}/debezium"
./jenkins-jobs/scripts/copy-plugins.sh \\
--dir="${WORKSPACE}" \\
--archive-urls="${DBZ_CONNECTOR_ARCHIVE_URLS}" \\
--libs="${DBZ_EXTRA_LIBS}" \\
--tags="${EXTRA_IMAGE_TAGS}" \\
--auto-tag="${AUTO_TAG}" \\
--registry="quay.io" --organisation="${QUAY_ORGANISATION}" \\
--dest-login="${QUAY_USERNAME}" \\
--dest-pass="${QUAY_PASSWORD}" \\
--img-output="${WORKSPACE}/published_image_dbz.txt"
'''
zip(archive: true, zipFile: 'artifact-server-all.zip', glob: 'published_image_dbz.txt')
}
}
}
}
post {
always {
mail to: params.MAIL_TO, subject: "Debezium artifact server preparation #${env.BUILD_NUMBER} finished with ${currentBuild.currentResult}", body: """
${currentBuild.projectName} run ${env.BUILD_URL} finished with result: ${currentBuild.currentResult}
"""
}
success {
archiveArtifacts "**/published_image*.txt"
}
}
}

118
jenkins-jobs/pipelines/downstream_preparations_trigger_pipeline.groovy
View File

@ -1,118 +0,0 @@
pipeline {
agent {
label 'Slave'
}
stages {
stage('Start') {
parallel {
stage('Invoke_downstream_as') {
when {
expression { params.EXECUTE_AS }
}
steps {
build job: 'ocp-downstream-artifact-server-prepare-job', parameters: [
string(name: MAIL_TO, value: params.MAIL_TO),
string(name: 'QUAY_CREDENTIALS', value: params.QUAY_CREDENTIALS),
string(name: 'QUAY_ORGANISATION', value: params.QUAY_ORGANISATION),
string(name: 'DBZ_GIT_REPOSITORY', value: params.DBZ_GIT_REPOSITORY),
string(name: 'DBZ_GIT_BRANCH', value: params.DBZ_GIT_BRANCH),
text(name: 'DBZ_EXTRA_LIBS', value: params.AS_DBZ_EXTRA_LIBS),
text(name: 'EXTRA_IMAGE_TAGS', value: params.AS_EXTRA_IMAGE_TAGS),
booleanParam(name: 'AUTO_TAG', value: params.AUTO_TAG),
text(name: 'DBZ_CONNECTOR_ARCHIVE_URLS', value: params.DBZ_CONNECTOR_ARCHIVE_URLS),
]
copyArtifacts(projectName: 'ocp-downstream-artifact-server-prepare-job', selector: lastCompleted())
}
}
stage('Invoke_downstream_strimzi') {
when {
expression { params.EXECUTE_STRIMZI }
}
steps {
build job: 'ocp-downstream-strimzi-prepare-job', parameters: [
string(name: MAIL_TO, value: params.MAIL_TO),
string(name: 'QUAY_CREDENTIALS', value: params.QUAY_CREDENTIALS),
string(name: 'QUAY_ORGANISATION', value: params.QUAY_ORGANISATION),
string(name: 'STRZ_RESOURCES_ARCHIVE_URL', value: params.STRZ_RESOURCES_ARCHIVE_URL),
string(name: 'STRZ_RESOURCES_DEPLOYMENT_DESCRIPTOR', value: params.STRZ_RESOURCES_DEPLOYMENT_DESCRIPTOR),
text(name: 'STRZ_IMAGES', value: params.STRZ_IMAGES),
string(name: 'DBZ_GIT_REPOSITORY', value: params.DBZ_GIT_REPOSITORY),
string(name: 'DBZ_GIT_BRANCH', value: params.DBZ_GIT_BRANCH),
booleanParam(name: 'DBZ_CONNECT_BUILD', value: params.STRZ_DBZ_CONNECT_BUILD),
text(name: 'DBZ_CONNECTOR_ARCHIVE_URLS', value: params.DBZ_CONNECTOR_ARCHIVE_URLS),
text(name: 'DBZ_EXTRA_LIBS', value: params.STRZ_DBZ_EXTRA_LIBS),
]
copyArtifacts(projectName: 'ocp-downstream-strimzi-prepare-job', selector: lastCompleted())
}
}
stage('Invoke_rhel') {
when {
expression { params.EXECUTE_RHEL }
}
steps {
build job: 'rhel-downstream-prepare-job', parameters: [
string(name: MAIL_TO, value: params.MAIL_TO),
string(name: 'QUAY_CREDENTIALS', value: params.QUAY_CREDENTIALS),
string(name: 'QUAY_ORGANISATION', value: params.QUAY_ORGANISATION),
string(name: 'RHEL_IMAGE', value: params.RHEL_IMAGE),
string(name: 'KAFKA_URL', value: params.KAFKA_URL),
string(name: 'DBZ_GIT_REPOSITORY', value: params.DBZ_GIT_REPOSITORY),
string(name: 'DBZ_GIT_BRANCH', value: params.DBZ_GIT_BRANCH),
booleanParam(name: 'AUTO_TAG', value: params.AUTO_TAG),
text(name: 'EXTRA_IMAGE_TAGS', value: params.EXTRA_IMAGE_TAGS),
text(name: 'DBZ_CONNECTOR_ARCHIVE_URLS', value: params.DBZ_CONNECTOR_ARCHIVE_URLS),
text(name: 'DBZ_EXTRA_LIBS', value: params.STRZ_DBZ_EXTRA_LIBS),
]
copyArtifacts(projectName: 'rhel-downstream-prepare-job', selector: lastCompleted())
}
}
}
}
}
post {
always {
script {
def jobMap = [:] as TreeMap
jobMap.put("ocp-downstream-artifact-server-prepare-job", params.EXECUTE_AS)
jobMap.put("ocp-downstream-strimzi-prepare-job", params.EXECUTE_STRIMZI)
jobMap.put("rhel-downstream-prepare-job", params.EXECUTE_RHEL)
jobMap.each { entry ->
if (!entry.value) {
return
}
def build = jenkins.model.Jenkins.get().getItem("${entry.key}").getLastCompletedBuild()
if (!build) {
println "No build of ${entry.key} found!"
return
}
// if no label is set, add branch name
def label = "#${build.number} parent: #${currentBuild.number}"
if (params.LABEL) {
label += " ${params.LABEL}"
}
// set label
build.displayName = "${label}"
}
// set parent job label
if (params.LABEL) {
currentBuild.displayName = "#${currentBuild.number} ${params.LABEL}"
} else {
// if no label is set, add branch name
currentBuild.displayName = "#${currentBuild.number}"
}
archiveArtifacts "**/*.zip"
}
}
}
}

106
jenkins-jobs/pipelines/downstream_prepare_pipeline.groovy
View File

@ -1,106 +0,0 @@
pipeline {
agent {
label 'Slave'
}
stages {
stage('CleanWorkspace') {
steps {
cleanWs()
}
}
stage('Checkout') {
steps {
checkout([
$class : 'GitSCM',
branches : [[name: "${DBZ_GIT_BRANCH}"]],
userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]],
extensions : [[$class : 'RelativeTargetDirectory',
relativeTargetDir: 'debezium']],
])
script {
env.STRZ_RESOURCES = "${env.WORKSPACE}/strimzi/install/cluster-operator"
}
sh '''
set -x
curl -OJs ${STRZ_RESOURCES_ARCHIVE_URL} && unzip amq-streams-*-install-examples.zip -d strimzi
'''
}
}
stage('Copy Images & Process Resource') {
steps {
withCredentials([
usernamePassword(credentialsId: "${QUAY_CREDENTIALS}", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'),
]) {
sh '''
set -x
cd "${WORKSPACE}/debezium"
./jenkins-jobs/scripts/copy-images.sh \\
--dir="${STRZ_RESOURCES}" \\
--images="${STRZ_IMAGES}" \\
--registry="quay.io" --organisation="${QUAY_ORGANISATION}" \\
--dest-login="${QUAY_USERNAME}" \\
--dest-pass="${QUAY_PASSWORD}" \\
--deployment-desc="${STRZ_RESOURCES_DEPLOYMENT_DESCRIPTOR}" \\
--img-output="${WORKSPACE}/published_images.txt"
'''
zip(archive: true, zipFile: 'amq-streams-install-examples.zip', dir: 'strimzi')
}
}
}
stage('Build Debezium Connect Image') {
when {
expression { params.DBZ_CONNECT_BUILD }
}
steps {
withCredentials([
usernamePassword(credentialsId: "${QUAY_CREDENTIALS}", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'),
]) {
sh '''
set -x
cd "${WORKSPACE}/debezium"
./jenkins-jobs/scripts/build-connect-image.sh \\
--dir="${WORKSPACE}" \\
--archive-urls="${DBZ_CONNECTOR_ARCHIVE_URLS}" \\
--libs="${DBZ_EXTRA_LIBS}" \\
--images="${STRZ_IMAGES}" \\
--registry="quay.io" --organisation="${QUAY_ORGANISATION}" \\
--dest-login="${QUAY_USERNAME}" \\
--dest-pass="${QUAY_PASSWORD}" \\
--img-output="${WORKSPACE}/published_images_dbz.txt"
'''
}
}
}
stage('Create main artefact') {
steps {
sh '''
set -x
mkdir "${WORKSPACE}/amq-streams-all"
cp -R "${WORKSPACE}/strimzi" "${WORKSPACE}/amq-streams-all/strimzi"
cp "${WORKSPACE}/published_images.txt" "${WORKSPACE}/amq-streams-all/amq-streams-published-images.txt"
cp "${WORKSPACE}/published_images_dbz.txt" \\
"${WORKSPACE}/amq-streams-all/amq-streams-published-images-dbz.txt" || :
'''
zip(archive: true, zipFile: 'amq-streams-all.zip', dir: 'amq-streams-all')
}
}
}
post {
always {
mail to: params.MAIL_TO, subject: "Debezium OpenShift test run #${env.BUILD_NUMBER} finished with ${currentBuild.currentResult}", body: """
${currentBuild.projectName} run ${env.BUILD_URL} finished with result: ${currentBuild.currentResult}
"""
}
success {
archiveArtifacts "**/published_images*.txt"
}
}
}

71
jenkins-jobs/pipelines/node_snapshot_build_pipeline.groovy
View File

@ -1,71 +0,0 @@
pipeline {
agent {
label 'Core'
}
stages {
stage('CleanWorkspace') {
steps {
cleanWs()
}
}
stage('Checkout') {
steps {
script {
if (ANS_GIT_REPOSITORY.isEmpty()) {
withCredentials([
string(credentialsId: "${ANS_GIT_SECRET}", variable: 'TMP_ANS_GIT_REPOSITORY')
]){env.ANS_GIT_REPOSITORY = TMP_ANS_GIT_REPOSITORY}
}
}
checkout([
$class : 'GitSCM',
branches : [[name: "${ANS_GIT_BRANCH}"]],
userRemoteConfigs: [[url: "${ANS_GIT_REPOSITORY}",
credentialsId: "${GITLAB_CREDENTIALS}"]],
extensions : [[$class : 'RelativeTargetDirectory',
relativeTargetDir: 'ci-jenkins-node']],
])
}
}
stage('Build image snapshot') {
steps {
withCredentials([
string(credentialsId: "${ANSIBLE_VAULT_PASSWORD}", variable: 'ANSIBLE_PASSWORD')
]) {
sh '''
set -x
cd "${WORKSPACE}/ci-jenkins-node"
echo "${ANSIBLE_PASSWORD}" > password.txt
ansible-vault decrypt --vault-password-file ./password.txt clouds.yaml
sudo cp roles/os_node_snapshot/files/CA-RH-NEW.crt /etc/pki/ca-trust/source/anchors/CA-RH-NEW.crt
sudo update-ca-trust
ansible-galaxy collection install -r requirements.yml
ansible-galaxy install -r requirements.yml
export ANSIBLE_HOST_KEY_CHECKING=false
ansible-playbook create_jenkins_node_snapshot.yml --vault-password-file ./password.txt \\
--extra-vars \\
"snapshot_name="${SNAPSHOT_NAME}" \\
os_base_image="${BASE_IMAGE}" \\
os_name="${INSTANCE_NAME}" \\
slave_user="${INSTANCE_USER}" os_keypair="${KEYPAIR}" \\
ssh_keypair_path="~/.ssh/id_rsa" \\
git_repo="${ANS_GIT_REPOSITORY}" \\
os_cloud="${CLOUD_NAME}" \\
git_branch="${ANS_GIT_BRANCH}""
'''
}
}
}
}
post {
always {
mail to: 'debezium-qe@redhat.com', subject: "Jenkins node image snapshot #${env.BUILD_NUMBER} finished with ${currentBuild.currentResult}", body: """
${currentBuild.projectName} run ${env.BUILD_URL} finished with result: ${currentBuild.currentResult}
"""
}
}
}

48
jenkins-jobs/pipelines/node_snapshot_history_cleanup_pipeline.groovy
View File

@ -1,48 +0,0 @@
pipeline {
agent {
label 'Core'
}
stages {
stage('CleanWorkspace') {
steps {
cleanWs()
}
}
stage('Checkout') {
steps {
checkout([
$class : 'GitSCM',
branches : [[name: "${DBZ_GIT_BRANCH}"]],
userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]],
extensions : [[$class : 'RelativeTargetDirectory',
relativeTargetDir: 'debezium']],
])
}
}
stage('Delete old snapshots') {
steps {
withCredentials([
file(credentialsId: "$OPENSTACK_AUTH", variable: 'OS_AUTH')
]) {
sh '''
cp ${OS_AUTH} ./clouds.yaml
cp ${WORKSPACE}/debezium/jenkins-jobs/scripts/cleanup_images_ansible.yml ./cleanup_images_ansible.yml
ansible-galaxy collection install openstack.cloud
ansible-playbook cleanup_images_ansible.yml --extra-vars "cloud_name="${CLOUD_NAME}" snapshot_name="${SNAPSHOT_NAME}""
'''
}
}
}
}
post {
always {
mail to: 'debezium-qe@redhat.com', subject: "Jenkins node image snapshot history cleanup #${env.BUILD_NUMBER} finished with ${currentBuild.currentResult}", body: """
${currentBuild.projectName} run ${env.BUILD_URL} finished with result: ${currentBuild.currentResult}
"""
}
}
}

184
jenkins-jobs/pipelines/ocp_system_pipeline.groovy
View File

@ -1,184 +0,0 @@
pipeline {
agent {
label 'Slave'
}
environment {
DEBEZIUM_LOCATION = "${WORKSPACE}/debezium"
OCP_PROJECT_NAME = "debezium-ocp-${BUILD_NUMBER}"
OCP_PROJECT_DEBEZIUM_TESTSUITE = "${OCP_PROJECT_NAME}-testsuite"
}
stages {
stage('Clean up ws') {
steps {
cleanWs()
}
}
stage('Checkout - Debezium') {
steps {
checkout([
$class : 'GitSCM',
branches : [[name: "${DBZ_GIT_BRANCH}"]],
userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]],
extensions : [[$class : 'RelativeTargetDirectory',
relativeTargetDir: 'debezium']],
])
}
}
stage('Configure namespaces') {
steps {
withCredentials([
usernamePassword(credentialsId: "${OCP_CREDENTIALS}", usernameVariable: 'OCP_USERNAME', passwordVariable: 'OCP_PASSWORD'),
file(credentialsId: "${params.PULL_SECRET}", variable: 'SECRET_PATH'),
]) {
sh '''
oc login -u "${OCP_USERNAME}" -p "${OCP_PASSWORD}" --insecure-skip-tls-verify=true "${OCP_URL}"
# create testsuite project and secret
oc new-project "${OCP_PROJECT_DEBEZIUM_TESTSUITE}"
oc adm policy add-cluster-role-to-user cluster-admin "system:serviceaccount:${OCP_PROJECT_DEBEZIUM_TESTSUITE}:default"
oc apply -f "${SECRET_PATH}"
'''
}
}
}
stage('Checkout Downstream AMQ Streams') {
when {
expression { params.PRODUCT_BUILD && params.STRIMZI_PREPARE_BUILD_NUMBER }
}
steps {
copyArtifacts projectName: 'ocp-downstream-strimzi-prepare-job',
filter: 'amq-streams-install-examples.zip',
selector: specific(params.STRIMZI_PREPARE_BUILD_NUMBER)
unzip zipFile: 'amq-streams-install-examples.zip', dir: 'strimzi'
}
}
stage('Prepare Downstream AMQ Streams') {
when {
expression { params.PRODUCT_BUILD && params.STRIMZI_PREPARE_BUILD_NUMBER }
}
steps {
script {
env.STRZ_RESOURCES = "${env.WORKSPACE}/strimzi/install/cluster-operator"
env.OCP_ENV_FILE = "${WORKSPACE}/debezium-${BUILD_NUMBER}.ocp.env"
}
withCredentials([
usernamePassword(credentialsId: "${OCP_CREDENTIALS}", usernameVariable: 'OCP_USERNAME', passwordVariable: 'OCP_PASSWORD'),
]) {
sh '''
set -x
cd "${DEBEZIUM_LOCATION}"
oc login ${OCP_URL} -u "${OCP_USERNAME}" --password="${OCP_PASSWORD}" --insecure-skip-tls-verify=true >/dev/null
./jenkins-jobs/scripts/ocp-projects.sh --create --project "${OCP_PROJECT_NAME}" --envfile "${OCP_ENV_FILE}"
source "${OCP_ENV_FILE}"
sed -i "s/namespace: .*/namespace: ${OCP_PROJECT_NAME}/" ${WORKSPACE}/strimzi/install/cluster-operator/*RoleBinding*.yaml
oc delete -f ${STRZ_RESOURCES} -n ${OCP_PROJECT_NAME} --ignore-not-found
oc create -f ${STRZ_RESOURCES} -n ${OCP_PROJECT_NAME}
'''
}
}
}
stage('Run testsuite') {
steps {
withCredentials([
usernamePassword(credentialsId: "${OCP_CREDENTIALS}", usernameVariable: 'OCP_USERNAME', passwordVariable: 'OCP_PASSWORD'),
file(credentialsId: "${params.PULL_SECRET}", variable: 'SECRET_PATH'),
]) {
script{
env.PREPARE_STRIMZI = params.STRIMZI_PREPARE_BUILD_NUMBER ? "false" : "true"
env.TEST_TAG_EXPRESSION = params.TEST_TAGS
if (!params.TEST_APICURIO_REGISTRY) {
env.TEST_TAG_EXPRESSION = [env.TEST_TAG_EXPRESSION, "!avro"].findAll().join(" & ")
}
env.TEST_TAG_EXPRESSION = [env.TEST_TAG_EXPRESSION, "!docker"].findAll().join(" & ")
}
sh '''
cd ${DEBEZIUM_LOCATION}
POD_DESCRIPTION="testsuite.yml"
PULL_SECRET_NAME=$(cat ${SECRET_PATH} | grep name | awk '{print $2;}')
LOG_LOCATION_IN_POD=/root/testsuite/testsuite_log
jenkins-jobs/docker/debezium-testing-system/deployment-template.sh --filename "${POD_DESCRIPTION}" \
--pull-secret-name "${PULL_SECRET_NAME}" \
--docker-tag "${DOCKER_TAG}" \
--project-name "${OCP_PROJECT_NAME}" \
--product-build "${PRODUCT_BUILD}" \
--strimzi-kc-build ${STRIMZI_KC_BUILD} \
--apicurio-version "${APICURIO_VERSION}" \
--kafka-version "${KAFKA_VERSION}" \
--groups-arg "${TEST_TAG_EXPRESSION}" \
--dbz-connect-image "${DBZ_CONNECT_IMAGE}" \
--artifact-server-image "${ARTIFACT_SERVER_IMAGE}" \
--dbz-git-repository "${DBZ_GIT_REPOSITORY}" \
--dbz-git-branch "${DBZ_GIT_BRANCH}" \
--testsuite-log "${LOG_LOCATION_IN_POD}" \
--strimzi-channel "${STRZ_CHANNEL}" \
--apicurio-channel "${APIC_CHANNEL}" \
--prepare-strimzi "${PREPARE_STRIMZI}"
oc project "${OCP_PROJECT_DEBEZIUM_TESTSUITE}"
oc create -f "${POD_DESCRIPTION}"
pod_name=$(oc get pods | tail -1 | awk '{print $1;}')
{
oc wait --timeout=10h --for=condition=Ready pod/${pod_name}
# copy log and test results
mkdir ${WORKSPACE}/testsuite_artifacts
oc rsync ${pod_name}:${LOG_LOCATION_IN_POD} ${WORKSPACE}/testsuite_artifacts
oc rsync ${pod_name}:/root/testsuite/artifacts.zip ${WORKSPACE}/testsuite_artifacts || oc delete pod ${pod_name}
oc delete pod ${pod_name}
} &
# wait for container to start and print logs
for i in {1..100}; do
sleep 2
oc logs -f ${pod_name} && break
done
'''
}
}
}
}
post {
always {
sh '''
cd ${DEBEZIUM_LOCATION}
./jenkins-jobs/scripts/ocp-projects.sh --delete --testsuite --project ${OCP_PROJECT_NAME}
'''
archiveArtifacts "**/testsuite_artifacts/*"
withCredentials([
usernamePassword(credentialsId: "rh-integration-quay-creds", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'),
string(credentialsId: "report-portal-token", variable: 'RP_TOKEN'),
]) {
sh '''
if [ "${PRODUCT_BUILD}" == true ] ; then
export ATTRIBUTES="downstream ocp"
else
export ATTRIBUTES="upstream ocp"
fi
cd ${WORKSPACE}/testsuite_artifacts
mkdir results
unzip artifacts.zip -d results
RESULTS_FOLDER="."
rm -rf ${RESULTS_FOLDER}/failsafe-summary.xml
docker login quay.io -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD"
${DEBEZIUM_LOCATION}/jenkins-jobs/scripts/report.sh --connector false --env-file env-file.env --results-folder ${RESULTS_FOLDER} --attributes "${ATTRIBUTES}"
'''
}
}
}
}

0
jenkins-jobs/pipelines/build_debezium_images_pipeline.groovy → jenkins-jobs/pipelines/release/build_debezium_images_pipeline.groovy
View File

0
jenkins-jobs/pipelines/build_debezium_tool_images_pipeline.groovy → jenkins-jobs/pipelines/release/build_debezium_tool_images_pipeline.groovy
View File

0
jenkins-jobs/pipelines/deploy_snapshots_pipeline.groovy → jenkins-jobs/pipelines/release/deploy_snapshots_pipeline.groovy
View File

0
jenkins-jobs/pipelines/prod-release-pipeline.groovy → jenkins-jobs/pipelines/release/prod-release-pipeline.groovy
View File

0
jenkins-jobs/pipelines/release-pipeline.groovy → jenkins-jobs/pipelines/release/release-pipeline.groovy
View File

241
jenkins-jobs/pipelines/system_pipeline.groovy
View File

@ -1,241 +0,0 @@
pipeline {
agent {
label 'NodeXL'
}
stages {
stage('Checkout - Debezium') {
steps {
checkout([
$class : 'GitSCM',
branches : [[name: "${DBZ_GIT_BRANCH}"]],
userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]],
extensions : [[$class : 'RelativeTargetDirectory',
relativeTargetDir: 'debezium']],
])
}
}
stage('Checkout - Debezium DB2 connector') {
when {
expression { !params.PRODUCT_BUILD }
}
steps {
checkout([
$class : 'GitSCM',
branches : [[name: "${DBZ_GIT_BRANCH_DB2}"]],
userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY_DB2}"]],
extensions : [[$class : 'RelativeTargetDirectory',
relativeTargetDir: 'debezium-connector-db2']],
])
}
}
stage('Configure') {
steps {
script {
if (!params.PRODUCT_BUILD && params.STRIMZI_PREPARE_BUILD_NUMBER) {
error("Using productised strimzi archive in upstream build")
}
env.OCP_ENV_FILE = "${WORKSPACE}/debezium-${BUILD_NUMBER}.ocp.env"
env.MVN_PROFILE_PROD = params.PRODUCT_BUILD ? "-Pproduct" : ""
env.PREPARE_STRIMZI_OPERATOR = params.STRIMZI_PREPARE_BUILD_NUMBER ? "-Dprepare.strimzi=false" : ""
env.OCP_PROJECT_DEBEZIUM = "debezium-${currentBuild.number}"
// Use strimzi build mechanism unless pre-built KC image is provided
env.TEST_CONNECT_STRZ_BUILD = params.IMAGE_CONNECT_STRZ ? false : true
// Configure images if provided
env.MVN_IMAGE_CONNECT_STRZ = params.IMAGE_CONNECT_STRZ ? "-Dimage.kc=${params.IMAGE_CONNECT_STRZ}" : ""
env.MVN_IMAGE_CONNECT_RHEL = params.IMAGE_CONNECT_RHEL ? "-Ddocker.image.kc=${params.IMAGE_CONNECT_RHEL}" : ""
env.MVN_IMAGE_DBZ_AS = params.IMAGE_DBZ_AS ? "-Dimage.as=${params.IMAGE_DBZ_AS}" : ""
// Test tag configuration
env.TEST_TAG_EXPRESSION = params.TEST_TAGS
if (!params.TEST_APICURIO_REGISTRY) {
env.TEST_TAG_EXPRESSION = [env.TEST_TAG_EXPRESSION, "!avro"].findAll().join(" & ")
}
// Version configuration
env.MVN_VERSION_KAFKA = params.TEST_VERSION_KAFKA ? "-Dversion.kafka=${params.TEST_VERSION_KAFKA}" : ""
env.MVN_VERSION_AS_DEBEZIUM = params.AS_VERSION_DEBEZIUM ? "-Das.debezium.version=${params.AS_VERSION_DEBEZIUM}" : ""
env.MVN_VERSION_AS_APICURIO = params.AS_VERSION_APICURIO ? "-Das.apicurio.version=${params.AS_VERSION_APICURIO}" : ""
}
}
}
stage('Checkout Downstream AMQ Streams') {
when {
expression { params.PRODUCT_BUILD && params.STRIMZI_PREPARE_BUILD_NUMBER }
}
steps {
copyArtifacts projectName: 'ocp-downstream-strimzi-prepare-job',
filter: 'amq-streams-install-examples.zip',
selector: specific(params.STRIMZI_PREPARE_BUILD_NUMBER)
unzip zipFile: 'amq-streams-install-examples.zip', dir: 'strimzi'
}
}
stage('Prepare Downstream AMQ Streams') {
when {
expression { params.PRODUCT_BUILD && params.STRIMZI_PREPARE_BUILD_NUMBER }
}
steps {
script {
env.STRZ_RESOURCES = "${env.WORKSPACE}/strimzi/install/cluster-operator"
}
withCredentials([
usernamePassword(credentialsId: "${OCP_CREDENTIALS}", usernameVariable: 'OCP_USERNAME', passwordVariable: 'OCP_PASSWORD'),
usernamePassword(credentialsId: "${QUAY_CREDENTIALS}", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'),
]) {
sh '''
set -x
cd "${WORKSPACE}/debezium"
oc login ${OCP_URL} -u "${OCP_USERNAME}" --password="${OCP_PASSWORD}" --insecure-skip-tls-verify=true >/dev/null
./jenkins-jobs/scripts/ocp-projects.sh --create -t "${BUILD_NUMBER}" --envfile "${OCP_ENV_FILE}"
source "${OCP_ENV_FILE}"
sed -i "s/namespace: .*/namespace: ${OCP_PROJECT_DEBEZIUM}/" ${WORKSPACE}/strimzi/install/cluster-operator/*RoleBinding*.yaml
oc delete -f ${STRZ_RESOURCES} -n ${OCP_PROJECT_DEBEZIUM} --ignore-not-found
oc create -f ${STRZ_RESOURCES} -n ${OCP_PROJECT_DEBEZIUM}
'''
}
}
}
stage('Build') {
steps {
withCredentials([
usernamePassword(credentialsId: "${OCP_CREDENTIALS}", usernameVariable: 'OCP_USERNAME', passwordVariable: 'OCP_PASSWORD'),
usernamePassword(credentialsId: "${QUAY_CREDENTIALS}", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'),
]) {
sh '''
set -x
docker login -u=${QUAY_USERNAME} -p=${QUAY_PASSWORD} quay.io
oc login ${OCP_URL} -u "${OCP_USERNAME}" --password="${OCP_PASSWORD}" --insecure-skip-tls-verify=true >/dev/null
'''
sh '''
set -x
cd ${WORKSPACE}/debezium
ORACLE_ARTIFACT_VERSION=$(mvn -q -DforceStdout help:evaluate -Dexpression=version.oracle.driver)
ORACLE_ARTIFACT_DIR="${HOME}/oracle-libs/${ORACLE_ARTIFACT_VERSION}.0"
cd ${ORACLE_ARTIFACT_DIR}
mvn install:install-file -DgroupId=com.oracle.instantclient -DartifactId=ojdbc8 -Dversion=${ORACLE_ARTIFACT_VERSION} -Dpackaging=jar -Dfile=ojdbc8.jar
mvn install:install-file -DgroupId=com.oracle.instantclient -DartifactId=xstreams -Dversion=${ORACLE_ARTIFACT_VERSION} -Dpackaging=jar -Dfile=xstreams.jar
'''
}
sh '''
set -x
cd "${WORKSPACE}/debezium"
mvn clean install -DskipTests -DskipITs
'''
}
}
stage('Build -- Upstream') {
when {
expression { !params.PRODUCT_BUILD }
}
steps {
// Build DB2 Connector
sh '''
set -x
cd ${WORKSPACE}/debezium-connector-db2
mvn clean install -DskipTests -DskipITs -Passembly
'''
// Build Oracle connector
sh '''
set -x
cd ${WORKSPACE}/debezium
mvn install -Passembly,oracle-all -DskipTests -DskipITs
'''
}
}
stage('Enable debug') {
when {
expression { params.DEBUG_MODE }
}
steps {
script {
env.MAVEN_OPTS = "-DforkCount=0 -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=*:5005"
}
}
}
stage('Test') {
steps {
withCredentials([
usernamePassword(credentialsId: "${OCP_CREDENTIALS}", usernameVariable: 'OCP_USERNAME', passwordVariable: 'OCP_PASSWORD'),
file(credentialsId: "${params.PULL_SECRET}", variable: 'SECRET_PATH'),
]) {
sh '''
set -x
cd ${WORKSPACE}/debezium
mvn install -pl debezium-testing/debezium-testing-system -PsystemITs,oracleITs \\
${MVN_PROFILE_PROD} \\
${PREPARE_STRIMZI_OPERATOR} \\
-Docp.project.debezium="${OCP_PROJECT_DEBEZIUM}" \\
-Docp.username="${OCP_USERNAME}" \\
-Docp.password="${OCP_PASSWORD}" \\
-Docp.url="${OCP_URL}" \\
-Docp.pull.secret.paths="${SECRET_PATH}" \\
-Dstrimzi.kc.build=${TEST_CONNECT_STRZ_BUILD} \\
-Dtest.wait.scale="${TEST_WAIT_SCALE}" \\
${MVN_IMAGE_CONNECT_STRZ} \\
${MVN_IMAGE_CONNECT_RHEL} \\
${MVN_IMAGE_DBZ_AS} \\
${MVN_VERSION_KAFKA} \\
${MVN_VERSION_AS_DEBEZIUM} \\
${MVN_VERSION_AS_APICURIO} \\
-Dstrimzi.operator.channel=${STRIMZI_OPERATOR_CHANNEL} \\
-Dapicurio.operator.channel=${APICURIO_OPERATOR_CHANNEL} \\
-Dgroups="${TEST_TAG_EXPRESSION}"
'''
}
}
}
}
post {
always {
archiveArtifacts '**/target/failsafe-reports/*.xml'
junit '**/target/failsafe-reports/*.xml'
mail to: params.MAIL_TO, subject: "Debezium OpenShift test run #${env.BUILD_NUMBER} finished with ${currentBuild.currentResult}", body: """
OpenShift interoperability test run ${env.BUILD_URL} finished with result: ${currentBuild.currentResult}
"""
withCredentials([
usernamePassword(credentialsId: "rh-integration-quay-creds", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'),
string(credentialsId: "report-portal-token", variable: 'RP_TOKEN'),
]) {
sh '''
if [ "$PRODUCT_BUILD" == true ] ; then
export ATTRIBUTES="downstream"
else
export ATTRIBUTES="upstream"
fi
RESULTS_FOLDER=final-results
RESULTS_PATH=$RESULTS_FOLDER/results
mkdir -p $RESULTS_PATH
cp debezium/debezium-testing/debezium-testing-system/target/failsafe-reports/*.xml $RESULTS_PATH
rm -rf $RESULTS_PATH/failsafe-summary.xml
docker login quay.io -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD"
./debezium/jenkins-jobs/scripts/report.sh --connector false --env-file env-file.env --results-folder $RESULTS_FOLDER --attributes $ATTRIBUTES
'''
}
}
}
}

144
jenkins-jobs/pipelines/upstream_artifact_server_prepare_pipeline.groovy
View File

@ -1,144 +0,0 @@
pipeline {
agent {
label 'Slave'
}
stages {
stage('CleanWorkspace') {
steps {
cleanWs()
}
}
stage('Checkout') {
steps {
checkout([
$class : 'GitSCM',
branches : [[name: "${DBZ_GIT_BRANCH}"]],
userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY}"]],
extensions : [[$class : 'RelativeTargetDirectory',
relativeTargetDir: 'debezium']],
])
}
}
stage('Checkout - Debezium DB2') {
steps {
checkout([
$class : 'GitSCM',
branches : [[name: "${DBZ_GIT_BRANCH_DB2}"]],
userRemoteConfigs: [[url: "${DBZ_GIT_REPOSITORY_DB2}"]],
extensions : [[$class : 'RelativeTargetDirectory',
relativeTargetDir: 'debezium-connector-db2']],
])
}
}
stage('Configure') {
steps {
script {
// Configure images if provided
env.IMAGE_TAG_SUFFIX = "${BUILD_NUMBER}"
// Apicurio version
env.APICURIO_ARTIFACT_VERSION = "${APICURIO_VERSION}"
}
withCredentials([
usernamePassword(credentialsId: "${QUAY_CREDENTIALS}", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'),
]) {
sh '''
set -x
cd ${WORKSPACE}/debezium
ORACLE_ARTIFACT_VERSION=$( mvn -q -DforceStdout help:evaluate -Dexpression=version.oracle.driver)
ORACLE_ARTIFACT_DIR="${HOME}/oracle-libs/${ORACLE_ARTIFACT_VERSION}.0"
mvn org.apache.maven.plugins:maven-dependency-plugin:2.8:get \\
-Dartifact=io.apicurio:apicurio-registry-distro-connect-converter:${APICURIO_ARTIFACT_VERSION}:zip \\
-Dmaven.repo.local=${WORKSPACE}/debezium/local-maven-repo
cd ${ORACLE_ARTIFACT_DIR}
mvn install:install-file -DgroupId=com.oracle.instantclient -DartifactId=ojdbc8 \\
-Dversion=${ORACLE_ARTIFACT_VERSION} -Dpackaging=jar -Dfile=ojdbc8.jar \\
-Dmaven.repo.local=${WORKSPACE}/debezium/local-maven-repo
mvn install:install-file -DgroupId=com.oracle.instantclient -DartifactId=xstreams \\
-Dversion=${ORACLE_ARTIFACT_VERSION} -Dpackaging=jar -Dfile=xstreams.jar \\
-Dmaven.repo.local=${WORKSPACE}/debezium/local-maven-repo
'''
}
}
}
stage('Build debezium') {
steps {
// Build core & parent
sh '''
set -x
cd "${WORKSPACE}/debezium"
mvn clean install -DskipTests -DskipITs -Dmaven.repo.local=local-maven-repo
'''
// Build Oracle connector
sh '''
set -x
cd ${WORKSPACE}/debezium
mvn install -Passembly,oracle-all -DskipTests -DskipITs -Dmaven.repo.local=local-maven-repo
'''
// Build DB2 Connector
sh '''
set -x
cd ${WORKSPACE}/debezium-connector-db2
mvn clean install -DskipTests -DskipITs -Passembly -Dmaven.repo.local=${WORKSPACE}/debezium/local-maven-repo
'''
}
}
stage('Login to private Quay repository'){
when(){
expression { params.ORACLE_INCLUDED }
}
steps {
withCredentials([
usernamePassword(credentialsId: "${PRIVATE_QUAY_CREDENTIALS}", usernameVariable: 'PRIVATE_QUAY_USERNAME', passwordVariable: 'PRIVATE_QUAY_PASSWORD'),
]){
sh '''
set -x
docker login -u ${PRIVATE_QUAY_USERNAME} -p ${PRIVATE_QUAY_PASSWORD} quay.io
'''
}
}
}
stage('Build and push image') {
steps {
withCredentials([
usernamePassword(credentialsId: "${QUAY_CREDENTIALS}", usernameVariable: 'QUAY_USERNAME', passwordVariable: 'QUAY_PASSWORD'),
]) {
sh '''
set -x
cd "${WORKSPACE}/debezium"
./jenkins-jobs/scripts/upstream-artifact-server-prepare.sh \\
--dir="${WORKSPACE}" \\
--tags="${EXTRA_IMAGE_TAGS}" \\
--auto-tag="${AUTO_TAG}" \\
--registry="quay.io" --organisation="${QUAY_ORGANISATION}" \\
--dest-login="${QUAY_USERNAME}" \\
--dest-pass="${QUAY_PASSWORD}" \\
--img-output="${WORKSPACE}/published_image_dbz.txt" \\
--oracle-included="${ORACLE_INCLUDED}" \\
--maven-repo=${WORKSPACE}/debezium/local-maven-repo
'''
}
}
}
}
post {
always {
mail to: params.MAIL_TO, subject: "Debezium upstream artifact server preparation #${env.BUILD_NUMBER} finished with ${currentBuild.currentResult}", body: """
${currentBuild.projectName} run ${env.BUILD_URL} finished with result: ${currentBuild.currentResult}
"""
}
success {
archiveArtifacts "**/published_image*.txt"
}
}
}