From 2b5370de1ae63515996e126c15f89835e237078f Mon Sep 17 00:00:00 2001
From: Jiri Pechanec <jpechane@redhat.com>
Date: Mon, 19 Mar 2018 19:48:36 +0100
Subject: [PATCH] DBZ-667 Sceintific numbers parsing

---
 .../connector/mysql/MySqlDdlParserTest.java   | 12 +++++++
 .../io/debezium/relational/ddl/DdlParser.java |  5 +--
 .../java/io/debezium/text/TokenStream.java    | 33 +++++++++++++++++--
 3 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/debezium-connector-mysql/src/test/java/io/debezium/connector/mysql/MySqlDdlParserTest.java b/debezium-connector-mysql/src/test/java/io/debezium/connector/mysql/MySqlDdlParserTest.java
index e0e896c64..e43d2adad 100644
--- a/debezium-connector-mysql/src/test/java/io/debezium/connector/mysql/MySqlDdlParserTest.java
+++ b/debezium-connector-mysql/src/test/java/io/debezium/connector/mysql/MySqlDdlParserTest.java
@@ -811,6 +811,18 @@ public void shouldParseAndIgnoreCreateFunction() {
         listener.forEach(this::printEvent);
     }
 
+    @FixFor("DBZ-667")
+    @Test
+    public void shouldParseScientificNotationNumber() {
+        String ddl = "CREATE TABLE t (id INT NOT NULL, myvalue DOUBLE DEFAULT 1E-10, PRIMARY KEY (`id`));"
+                + "CREATE TABLE t (id INT NOT NULL, myvalue DOUBLE DEFAULT 1.3E-10, PRIMARY KEY (`id`));"
+                + "CREATE TABLE t (id INT NOT NULL, myvalue DOUBLE DEFAULT 1.3E+10, PRIMARY KEY (`id`));"
+                + "CREATE TABLE t (id INT NOT NULL, myvalue DOUBLE DEFAULT 3E10, PRIMARY KEY (`id`));"
+                + "CREATE TABLE t (id INT NOT NULL, myvalue DOUBLE DEFAULT 1.3e10, PRIMARY KEY (`id`))";
+        parser.parse(ddl, tables);
+        assertThat(tables.size()).isEqualTo(1);
+    }
+
     @FixFor("DBZ-162")
     @Test
     public void shouldParseAlterTableWithNewlineFeeds() {
diff --git a/debezium-core/src/main/java/io/debezium/relational/ddl/DdlParser.java b/debezium-core/src/main/java/io/debezium/relational/ddl/DdlParser.java
index 6e3d6abcd..3e639182b 100644
--- a/debezium-core/src/main/java/io/debezium/relational/ddl/DdlParser.java
+++ b/debezium-core/src/main/java/io/debezium/relational/ddl/DdlParser.java
@@ -754,15 +754,16 @@ protected Object parseNumericLiteral(Marker start, boolean signed) {
             sb.append(tokens.consumeInteger());
         }
         if (tokens.canConsume('.')) {
+            sb.append('.');
             sb.append(tokens.consumeInteger());
             decimal = true;
         }
-        if (!tokens.canConsume('E')) {
+        if (!tokens.canConsumeAnyOf("E", "e")) {
             if (decimal) return Double.parseDouble(sb.toString());
             return Integer.parseInt(sb.toString());
         }
         sb.append('E');
-        if (tokens.matches("+", "-")) {
+        if (tokens.matchesAnyOf("+", "-")) {
             sb.append(tokens.consumeAnyOf("+", "-"));
         }
         sb.append(tokens.consumeInteger());
diff --git a/debezium-core/src/main/java/io/debezium/text/TokenStream.java b/debezium-core/src/main/java/io/debezium/text/TokenStream.java
index b7d4f55bb..77d9f09b4 100644
--- a/debezium-core/src/main/java/io/debezium/text/TokenStream.java
+++ b/debezium-core/src/main/java/io/debezium/text/TokenStream.java
@@ -17,6 +17,7 @@
 import io.debezium.annotation.Immutable;
 import io.debezium.annotation.NotThreadSafe;
 import io.debezium.function.BooleanConsumer;
+import io.debezium.relational.ddl.DdlTokenizer;
 import io.debezium.util.Strings;
 
 /**
@@ -572,10 +573,22 @@ public Position nextPosition() {
     public int consumeInteger() throws ParsingException, IllegalStateException {
         if (completed) throwNoMoreContent();
         // Get the value from the current token ...
-        String value = currentToken().value();
+        String value = currentToken().value().toUpperCase();
         try {
+            List<Token> newTokens = new ArrayList<>();
+            int ePos = value.indexOf("E");
+            // Scientific format, need to identify mantissa and exponent and put it back to stream
+            if (ePos != -1) {
+                String mantissa = value.substring(0, ePos);
+                newTokens.add(new CaseInsensitiveToken(currentToken().startIndex() + ePos, currentToken().startIndex() + ePos + 1, DdlTokenizer.WORD, currentToken().position()));
+                // Number is in format xxxEyyy
+                if (ePos != value.length() - 1) {
+                    newTokens.add(new CaseInsensitiveToken(currentToken().startIndex() + ePos + 1, currentToken().endIndex(), DdlTokenizer.WORD, currentToken().position()));
+                }
+                value = mantissa;
+            }
             int result = Integer.parseInt(value);
-            moveToNextToken();
+            moveToNextToken(newTokens);
             return result;
         } catch (NumberFormatException e) {
             Position position = currentToken().position();
@@ -1795,7 +1808,17 @@ public String toString() {
         return sb.toString();
     }
 
-    private void moveToNextToken() {
+    private void moveToNextToken(List<Token> newTokens) {
+        if (newTokens != null && !newTokens.isEmpty()) {
+            for (Token t: newTokens) {
+                tokenIterator.add(t);
+            }
+            for (int i = 0; i < newTokens.size() - 1; i++) {
+                tokenIterator.previous();
+            }
+            currentToken = newTokens.get(0);
+            return;
+        }
         // And move the currentToken to the next token ...
         if (!tokenIterator.hasNext()) {
             completed = true;
@@ -1805,6 +1828,10 @@ private void moveToNextToken() {
         }
     }
 
+    private void moveToNextToken() {
+        moveToNextToken(null);
+    }
+
     /**
      * Get the current token.
      *