diff --git a/.github/workflows/debezium-workflow.yml b/.github/workflows/debezium-workflow.yml new file mode 100644 index 000000000..7ea02b602 --- /dev/null +++ b/.github/workflows/debezium-workflow.yml @@ -0,0 +1,1113 @@ +name: Build Debezium + +on: + push: + branches: + - main + - 1.* + - 2.* + pull_request: + branches: + - main + - 1.* + - 2.* + +env: + MAVEN_FULL_BUILD_PROJECTS: "\\!debezium-microbenchmark-oracle" + +jobs: + # This job is responsible for inspecting the changes in the repository and setting specific output variables + # that subsequent jobs can use to determine whether a particular job should be skipped or included. + file_changes: + name: "Detect repository changes" + runs-on: ubuntu-latest + outputs: + common-changed: ${{ steps.changed-files-common.outputs.any_changed }} + mongodb-changed: ${{ steps.changed-files-mongodb.outputs.any_changed }} + mysql-changed: ${{ steps.changed-files-mysql.outputs.any_changed }} + postgresql-changed: ${{ steps.changed-files-postgresql.outputs.any_changed }} + oracle-changed: ${{ steps.changed-files-oracle.outputs.any_changed }} + sqlserver-changed: ${{ steps.changed-files-sqlserver.outputs.any_changed }} + outbox-changed: ${{ steps.changed-files-outbox.outputs.any_changed }} + rest-extension-changed: ${{ steps.changed-files-rest-extension.outputs.any_changed }} + schema-generator-changed: ${{ steps.changed-files-schema-generator.outputs.any_changed }} + debezium-testing-changed: ${{ steps.changed-files-debezium-testing.outputs.any_changed }} + mysql-ddl-parser-changed: ${{ steps.changed-files-mysql-ddl-parser.outputs.any_changed }} + oracle-ddl-parser-changed: ${{ steps.changed-files-oracle-ddl-parser.outputs.any_changed }} + documentation-only-changed: ${{ steps.changed-files-documentation.outputs.only_changed}} + storage-only-changed: ${{ steps.changed-files-storage.outputs.only_changed}} + steps: + - name: Checkout Action + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get modified files (Common) + id: changed-files-common + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + support/checkstyle/** + debezium-api/** + debezium-assembly-descriptors/** + debezium-core/** + debezium-embedded/** + debezium-revapi/** + debezium-ide-configs/** + debezium-parent/pom.xml + debezium-bom/pom.xml + debezium-storage/** + pom.xml + .github/workflows/debezium-workflow.yml + + - name: Get modified files (MongoDB) + id: changed-files-mongodb + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-connector-mongodb/** + + - name: Get modified files (MySQL) + id: changed-files-mysql + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-connector-mysql/** + + - name: Get modified files (PostgreSQL) + id: changed-files-postgresql + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-connector-postgres/** + + - name: Get modified files (Oracle) + id: changed-files-oracle + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-connector-oracle/** + + - name: Get modified files (SQL Server) + id: changed-files-sqlserver + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-connector-sqlserver/** + + - name: Get modified files (Quarkus Outbox) + id: changed-files-outbox + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-quarkus-outbox/** + + - name: Get modified files (REST Extension) + id: changed-files-rest-extension + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-connect-rest-extension/** + support/checkstyle/** + debezium-parent/pom.xml + debezium-bom/pom.xml + pom.xml + .github/workflows/debezium-workflow.yml + + - name: Get modified files (Schema Generator) + id: changed-files-schema-generator + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-schema-generator/** + support/checkstyle/** + debezium-parent/pom.xml + debezium-bom/pom.xml + pom.xml + .github/workflows/debezium-workflow.yml + + - name: Get modified files (Debezium Testing) + id: changed-files-debezium-testing + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-testing/** + + - name: Get modified files (MySQL DDL parser) + id: changed-files-mysql-ddl-parser + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-ddl-parser/src/main/antlr4/io/debezium/ddl/parser/mysql/** + debezium-ddl-parser/src/main/java/io/debezium/antlr/** + debezium-ddl-parser/src/test/resources/mysql/examples/** + + - name: Get modified files (Oracle DDL parser) + id: changed-files-oracle-ddl-parser + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-ddl-parser/src/main/antlr4/io/debezium/ddl/parser/oracle/** + debezium-ddl-parser/src/main/java/io/debezium/antlr/** + debezium-ddl-parser/src/main/java/io/debezium/ddl/parser/oracle/** + debezium-ddl-parser/src/test/resources/oracle/examples/** + + - name: Get modified files (Documentation) + id: changed-files-documentation + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + documentation/** + + - name: Get modified files (Storage) + id: changed-files-storage + uses: tj-actions/changed-files@v42.0.5 + with: + files: | + debezium-storage/** + + build_cache: + name: "Dependency Cache" + needs: [file_changes] + if: ${{ needs.file_changes.outputs.documentation-only-changed == 'false' }} + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + id: maven-cache-check + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + # This step is responsible for pulling down artifacts + # Unfortunately due to the nature of how some of the maven workflows work, the only reliable way + # to guarantee fully seeding the maven cache is to run a full build. This step does not execute + # tests, formatting, checkstyle, nor import sorts. + # + # This check is conditioned so that if the cache-key was not found, we will execute this step. + # If the cache-key was found, this means we cannot update the cache and therefore we should + # not need to explicitly run this step. This should improve response of format/checkstyle + # errors to users faster. + # + # This job also explicitly excludes the "debezium-microbenchmark-oracle" module temporarily. + # There is a dependency on xstream.jar for this module that should be fixed and made to not + # be required so that the module can be built on GitHub Actions. + - name: Download dependencies + if: steps.maven-cache-check.outputs.cache-hit != 'true' + run: > + ./mvnw -B -ntp clean install + -pl ${{ env.MAVEN_FULL_BUILD_PROJECTS }} + -Dformat.skip=true + -Dcheckstyle.skip=true + -Dorg.slf4j.simpleLogger.showDateTime=true + -Dorg.slf4j.simpleLogger.dateTimeFormat="YYYY-MM-DD HH:mm:ss,SSS" + -DskipTests=true + -DskipITs=true + + check_style: + name: "Checkstyle and Formatting" + needs: [build_cache] + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + id: maven-cache-check + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + # Since we do a complete build conditionally above, we should always either be able to rely on the + # build step to have installed all needed artifacts or the artifacts that exist from a prior run + # in order to satisfy dependencies for the format, checkstyle, and import sort checks. If the + # dependencies change, a new cache should be generated and a full build step executed, so this + # should always work successfully. + - name: Checkstyle, Formatting, and Import Order Checks + run: > + ./mvnw -B -ntp process-sources checkstyle:checkstyle + -pl ${{ env.MAVEN_FULL_BUILD_PROJECTS }} + -Dformat.formatter.goal=validate + -Dformat.imports.goal=check + -Dorg.slf4j.simpleLogger.showDateTime=true + -Dorg.slf4j.simpleLogger.dateTimeFormat="YYYY-MM-DD HH:mm:ss,SSS" + + build_mongodb: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mongodb-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }} + name: "MongoDB" + runs-on: ubuntu-latest + strategy: + matrix: + version-mongo-server: ["5.0", "6.0", "7.0"] + fail-fast: false + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Debezium Connector MongoDB (Mode ${{ matrix.capture-mode }}) + run: > + ./mvnw clean install -B -pl debezium-connector-mongodb -am -Passembly + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dversion.mongo.server=${{ matrix.version-mongo-server }} + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -DfailFlakyTests=false + + build_mysql_8_0: + needs: [ check_style, file_changes ] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mysql-changed == 'true' || needs.file_changes.outputs.mysql-ddl-parser-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }} + strategy: + fail-fast: false + matrix: + profile: [ 'mysql-ci', 'mysql-ci-gtids', 'mysql-ci-percona', 'mysql-ci-ssl' ] + name: "MySQL 8.0 (${{ matrix.profile }})" + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Debezium Connector MySQL + run: > + ./mvnw clean install -B -pl debezium-connector-mysql -am -P${{ matrix.profile }} + -Dversion.mysql.server=8.0 + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -DfailFlakyTests=false + + build_mysql: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mysql-changed == 'true' || needs.file_changes.outputs.mysql-ddl-parser-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }} + strategy: + fail-fast: false + matrix: + profile: [ 'mysql-ci', 'mysql-ci-gtids', 'mysql-ci-percona', 'mysql-ci-ssl' ] + name: "MySQL (${{ matrix.profile }})" + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Debezium Connector MySQL + run: > + ./mvnw clean install -B -pl debezium-connector-mysql -am -P${{ matrix.profile }} + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -DfailFlakyTests=false + + build_mariadb: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mysql-changed == 'true' || needs.file_changes.outputs.mysql-ddl-parser-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }} + strategy: + fail-fast: false + matrix: + profile: [ 'mysql-ci', 'mysql-ci-gtids' ] + name: "MariaDB (${{ matrix.profile }})" + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Debezium Connector MySQL for MariaDB + # We specifically exclude the assembly profile for now as the MariaDB integration is still + # ongoing for GTID and therefore this simply runs the tests like non-assembly builds but + # utilizes the MariaDB database rather than MySQL at this time. It also gives us a chance + # to see how changes react to MariaDB as they're contributed. + run: > + ./mvnw clean install -B -pl debezium-connector-mysql -am -Pmariadb,${{ matrix.profile }} + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -DfailFlakyTests=false + + build_postgresql: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.postgresql-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }} + name: "PostgreSQL" + runs-on: ubuntu-latest + strategy: + matrix: + postgres-plugin: ["assembly", "assembly,postgres-16,pgoutput-decoder"] + fail-fast: false + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Debezium Connector PostgreSQL (${{ matrix.postgres-plugin }}) + run: > + ./mvnw clean install -B -pl debezium-connector-postgres -am -P${{ matrix.postgres-plugin }} + -Ddebezium.test.records.waittime=5 + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -DfailFlakyTests=false + + build_oracle: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.oracle-changed == 'true' || needs.file_changes.outputs.oracle-ddl-parser-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }} + name: "Oracle" + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + # We explicitly do not run the integration tests in this step as it's designed for pull requests only + # Integration tests can only be run against pushes to the repository for secret access + - name: Build Debezium Connector Oracle (PR) + if: ${{ github.event_name == 'pull_request' }} + run: > + ./mvnw clean install -B -pl debezium-connector-oracle -am + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -DfailFlakyTests=false + + build_sqlserver: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.sqlserver-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }} + name: "SQL Server" + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Debezium Connector SQL Server + run: > + ./mvnw clean install -B -pl debezium-connector-sqlserver -am -Passembly + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -Ddebezium.test.records.waittime=10 + -DfailFlakyTests=false + + build_outbox: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.outbox-changed == 'true' }} + name: "Quarkus Outbox" + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Quarkus Outbox + run: > + ./mvnw clean install -B -pl :debezium-quarkus-outbox -am -amd -Passembly + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + build_rest_extension: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.rest-extension-changed == 'true' }} + name: "REST Extension" + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Connect REST Extension + run: > + ./mvnw clean install -B -pl debezium-connect-rest-extension -am + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + build_schema_generator: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.schema-generator-changed == 'true' }} + name: "Schema Generator" + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Schema Generator + run: > + ./mvnw clean install -B -pl debezium-schema-generator -am + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + build_debezium_testing: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.debezium-testing-changed == 'true' }} + name: "Debezium Testing" + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Debezium Testing + run: > + ./mvnw clean install -B -pl debezium-testing,debezium-testing/debezium-testing-testcontainers -am -Passembly + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -DfailFlakyTests=false + -Dstartup.timeout.seconds=120 + + build_storage: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' || needs.file_changes.outputs.mysql-changed == 'true' || needs.file_changes.outputs.mysql-ddl-parser-changed == 'true' || needs.file_changes.outputs.storage-changed == 'true' }} + name: "Storage" + runs-on: ubuntu-latest + steps: + - name: Checkout Action + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('**/pom.xml') }} + + - name: Build Debezium (Core) + run: > + ./mvnw clean install + -pl debezium-assembly-descriptors,debezium-bom,debezium-core,debezium-embedded,:debezium-ide-configs,:debezium-checkstyle,:debezium-revapi + -am + -DskipTests=true + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + - name: Build Debezium Connector MySQL + run: > + ./mvnw clean install -B -pl debezium-connector-mysql -Passembly + -DskipTests=true + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + - name: Build Debezium Storage + run: > + ./mvnw clean install -B -f debezium-storage/pom.xml -Passembly + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + build_cassandra: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' }} + name: "Cassandra" + runs-on: ubuntu-latest + steps: + - name: Checkout Action (Core) + uses: actions/checkout@v4 + with: + path: core + + - name: Checkout Action (Cassandra) + uses: actions/checkout@v4 + with: + repository: debezium/debezium-connector-cassandra + path: cassandra + + # We explicitly use only the hash of the POM files from the core repository by default + # For this build, we do not care if there are or are not changes in the sibling repository since this + # job will only ever fire if there are changes in the common paths identified in the files_changed job. + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Build Debezium (Core) + run: > + ./core/mvnw clean install -f core/pom.xml + -pl debezium-assembly-descriptors,debezium-bom,debezium-core,debezium-embedded,:debezium-ide-configs,:debezium-checkstyle,:debezium-revapi + -am + -DskipTests=true + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + - name: Install dse dependency artifacts (for Debezium Connector Cassandra) + working-directory: cassandra + run: sh ./install-artifacts.sh + + - name: Build Debezium Connector Cassandra + run: > + ./core/mvnw clean install -f cassandra/pom.xml -Passembly + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + build_db2: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' }} + name: "Db2" + runs-on: ubuntu-latest + steps: + - name: Checkout Action (Core) + uses: actions/checkout@v4 + with: + path: core + + - name: Checkout Action (Db2) + uses: actions/checkout@v4 + with: + repository: debezium/debezium-connector-db2 + path: db2 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + # We explicitly use only the hash of the POM files from the core repository by default + # For this build, we do not care if there are or are not changes in the sibling repository since this + # job will only ever fire if there are changes in the common paths identified in the files_changed job. + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + + - name: Build Debezium (Core) + run: > + ./core/mvnw clean install -f core/pom.xml + -pl debezium-assembly-descriptors,debezium-bom,debezium-core,debezium-embedded,:debezium-ide-configs,:debezium-checkstyle,:debezium-revapi + -am + -DskipTests=true + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + - name: Build Debezium Connector Db2 + run: > + ./core/mvnw clean install -f db2/pom.xml -Passembly + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -Dfail.flaky.tests=false + + build_informix: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' }} + name: "Informix" + runs-on: ubuntu-latest + steps: + - name: Checkout Action (Core) + uses: actions/checkout@v4 + with: + path: core + + - name: Checkout Action (Db2) + uses: actions/checkout@v4 + with: + repository: debezium/debezium-connector-informix + path: informix + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + # We explicitly use only the hash of the POM files from the core repository by default + # For this build, we do not care if there are or are not changes in the sibling repository since this + # job will only ever fire if there are changes in the common paths identified in the files_changed job. + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + + - name: Build Debezium (Core) + run: > + ./core/mvnw clean install -f core/pom.xml + -pl debezium-assembly-descriptors,debezium-bom,debezium-core,debezium-embedded,:debezium-ide-configs,:debezium-checkstyle,:debezium-revapi + -am + -DskipTests=true + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + - name: Build Debezium Connector Informix + run: > + ./informix/mvnw clean install -f informix/pom.xml -Passembly,informix14 + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -Ddebezium.test.records.waittime=5 + -Ddebezium.test.records.waittime.after.nulls=5 + -DfailFlakyTests=false + + build_vitess: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' }} + name: "Vitess" + runs-on: ubuntu-latest + steps: + - name: Checkout Action (Core) + uses: actions/checkout@v4 + with: + path: core + + - name: Checkout Action (Vitess) + uses: actions/checkout@v4 + with: + repository: debezium/debezium-connector-vitess + path: vitess + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + # We explicitly use only the hash of the POM files from the core repository by default + # For this build, we do not care if there are or are not changes in the sibling repository since this + # job will only ever fire if there are changes in the common paths identified in the files_changed job. + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + + - name: Build Debezium (Core) + run: > + ./core/mvnw clean install -f core/pom.xml + -pl debezium-assembly-descriptors,debezium-bom,debezium-core,debezium-embedded,:debezium-ide-configs,:debezium-checkstyle,:debezium-revapi + -am + -DskipTests=true + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + - name: Build Debezium Connector Vitess + run: > + ./core/mvnw clean install -f vitess/pom.xml -Passembly + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + build_spanner: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' }} + name: "Spanner" + runs-on: ubuntu-latest + steps: + - name: Checkout Action (Core) + uses: actions/checkout@v4 + with: + path: core + + - name: Checkout Action (Spanner) + uses: actions/checkout@v4 + with: + repository: debezium/debezium-connector-spanner + path: spanner + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + # We explicitly use only the hash of the POM files from the core repository by default + # For this build, we do not care if there are or are not changes in the sibling repository since this + # job will only ever fire if there are changes in the common paths identified in the files_changed job. + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + + - name: Build Debezium (Core) + run: > + ./core/mvnw clean install -f core/pom.xml + -pl debezium-assembly-descriptors,debezium-bom,debezium-core,debezium-embedded,:debezium-ide-configs,:debezium-checkstyle,:debezium-revapi + -am + -DskipTests=true + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + - name: Build Debezium Connector Spanner + run: > + ./core/mvnw clean install -f spanner/pom.xml -Passembly + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + build_jdbc: + needs: [ check_style, file_changes ] + if: ${{ needs.file_changes.outputs.common-changed == 'true'}} + name: "JDBC" + runs-on: ubuntu-latest + steps: + - name: Checkout Action (Core) + uses: actions/checkout@v4 + with: + path: core + + - name: Checkout Action (JDBC) + uses: actions/checkout@v4 + with: + repository: debezium/debezium-connector-jdbc + path: jdbc + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + # We explicitly use only the hash of the POM files from the core repository by default + # For this build, we do not care if there are or are not changes in the sibling repository since this + # job will only ever fire if there are changes in the common paths identified in the files_changed job. + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + + - name: Build Debezium (Core) + run: > + ./core/mvnw clean install -f core/pom.xml + -pl debezium-assembly-descriptors,debezium-bom,debezium-core,debezium-embedded,:debezium-ide-configs,:debezium-checkstyle,:debezium-revapi + -am + -DskipTests=true + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + - name: Build Debezium Connector JDBC + run: > + ./core/mvnw clean install -f jdbc/pom.xml -Passembly + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + build_debezium_server: + needs: [check_style, file_changes] + if: ${{ needs.file_changes.outputs.common-changed == 'true' }} + name: "Debezium Server" + runs-on: ubuntu-latest + steps: + - name: Checkout Action (Core) + uses: actions/checkout@v4 + with: + path: core + + - name: Checkout Action (Debezium Server) + uses: actions/checkout@v4 + with: + repository: debezium/debezium-server + path: server + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + # We explicitly use only the hash of the POM files from the core repository by default + # For this build, we do not care if there are or are not changes in the sibling repository since this + # job will only ever fire if there are changes in the common paths identified in the files_changed job. + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + restore-keys: | + maven-debezium-test-build-${{ hashFiles('core/**/pom.xml') }} + + - name: Build Debezium (Core) + run: > + ./core/mvnw clean install -f core/pom.xml + -DskipTests=true + -DskipITs=true + -Dcheckstyle.skip=true + -Dformat.skip=true + -Drevapi.skip + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + + - name: Build Debezium Server + run: > + ./core/mvnw clean install -fae -f server/pom.xml -Passembly + -Dcheckstyle.skip=true + -Dformat.skip=true + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -DskipNonCore diff --git a/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbReplicaSet.java b/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbReplicaSet.java index 99494be2d..95453b726 100644 --- a/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbReplicaSet.java +++ b/debezium-testing/debezium-testing-testcontainers/src/main/java/io/debezium/testing/testcontainers/MongoDbReplicaSet.java @@ -6,7 +6,6 @@ package io.debezium.testing.testcontainers; import static io.debezium.testing.testcontainers.util.DockerUtils.logDockerDesktopBanner; -import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.joining; import static org.awaitility.Awaitility.await; @@ -45,7 +44,9 @@ public class MongoDbReplicaSet implements MongoDbDeployment { private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbReplicaSet.class); - + private static final String STARTUP_TIMEOUT = System.getProperty("startup.timeout.seconds"); + private static final long DEFAULT_STARTUP_TIMEOUT = 60; + private final long STARTUP_TIMEOUT_SECONDS = STARTUP_TIMEOUT != null ? Long.parseLong(STARTUP_TIMEOUT) : DEFAULT_STARTUP_TIMEOUT; private final String name; private final int memberCount; private final boolean configServer; @@ -342,7 +343,7 @@ public Container.ExecResult execMongoScript(MountableFile file, String container public void awaitReplicaPrimary() { await() - .atMost(1, MINUTES) + .atMost(Duration.ofSeconds(STARTUP_TIMEOUT_SECONDS)) .pollDelay(1, SECONDS) .ignoreException(IllegalStateException.class) .until(() -> tryPrimary().isPresent());