DBZ-6324 Provide @Flaky annotation for CI unstable tests

debezium-core/src/test/java/io/debezium/junit/ConditionalFail.java

@@ -9,8 +9,11 @@
 import java.util.function.Supplier;
 
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * JUnit rule that inspects the presence of the {@link ShouldFailWhen} annotation on a test method.
@@ -21,12 +24,26 @@
  */
 public class ConditionalFail extends AnnotationBasedTestRule {
 
+    private static final Logger FLAKY_LOGGER = LoggerFactory.getLogger(Flaky.class);
+
+    private static final String JIRA_BASE_URL = "https://issues.jboss.org/browse/";
+
     @Override
     public Statement apply(final Statement base, final Description description) {
         final ShouldFailWhen conditionClass = hasAnnotation(description, ShouldFailWhen.class);
-        if (conditionClass == null) {
-            return base;
+        if (conditionClass != null) {
+            return failOnCondition(base, description, conditionClass);
         }
+
+        final Flaky flakyClass = hasAnnotation(description, Flaky.class);
+        if (flakyClass != null) {
+            return ignoreFlakyFailure(base, description, flakyClass);
+        }
+        return base;
+    }
+
+    private Statement failOnCondition(final Statement base, final Description description,
+                                      final ShouldFailWhen conditionClass) {
         try {
             Supplier<Boolean> condition = conditionClass.value().getDeclaredConstructor().newInstance();
             return new Statement() {
@@ -55,4 +72,31 @@ else if (failure != null) {
             throw new IllegalStateException(e);
         }
     }
+
+    private Statement ignoreFlakyFailure(final Statement base, final Description description,
+                                         final Flaky flakyClass) {
+
+        final String flakyFailuresProperty = System.getProperty(Flaky.IGNORE_FLAKY_FAILURES_PROPERTY);
+        if (flakyFailuresProperty == null || !Boolean.valueOf(flakyFailuresProperty)) {
+            return base;
+        }
+
+        return new Statement() {
+            @Override
+            public void evaluate() throws Throwable {
+                try {
+                    base.evaluate();
+                }
+                catch (final Throwable t) {
+                    FLAKY_LOGGER.error("Ignored failure for {}, tracked with {}", description, issueUrl(flakyClass.value()), t);
+                    // Marks test as skipped
+                    Assume.assumeTrue(String.format("Flaky test %s#%s failed", description.getTestClass().getSimpleName(), description.getMethodName()), false);
+                }
+            }
+        };
+    }
+
+    private String issueUrl(String jiraId) {
+        return JIRA_BASE_URL + jiraId;
+    }
 }

debezium-core/src/test/java/io/debezium/junit/Flaky.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Debezium Authors.
+ *
+ * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
+ */
+package io.debezium.junit;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation used together with the {@link ConditionalFail} JUnit rule, that allows ignoring
+ * failures of tests know to be unstable on CI, using the {@code ignoreFlakyFailures} system property.
+ *
+ * @author Jiri Pechanec
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.METHOD, ElementType.TYPE })
+public @interface Flaky {
+
+    String IGNORE_FLAKY_FAILURES_PROPERTY = "ignoreFlakyFailures";
+
+    /**
+     * The Jira id of the issue tracking the failing test
+     */
+    String value();
+}