alexey.rassokhin/tet123
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

DBZ-4641 Fix Oracle Loader, HDFS, and Hive grammar rules

Browse Source
This commit is contained in:
Chris Cranford 2022-02-21 10:49:46 -05:00 committed by Gunnar Morling
parent 2074767ebc
commit 3e01770380
3 changed files with 61 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
debezium-ddl-parser/src/main/antlr4/io/debezium/ddl/parser/oracle/generated/PlSqlLexer.g4
View File

@ -2471,26 +2471,3 @@ fragment SIMPLE_LETTER : [A-Z];
fragment FLOAT_FRAGMENT : UNSIGNED_INTEGER* '.'? UNSIGNED_INTEGER+;
fragment NEWLINE : '\r'? '\n';
fragment SPACE : [ \t];
// ORACLE HDFS & HIVE rules
COM_ORACLE_HDFS_HIVE_BIGDATA_COLMAP : 'com.oracle.bigdata.colMap';
COM_ORACLE_HDFS_HIVE_BIGDATA_ERROROPT : 'com.oracle.bigdata.errorOpt';
COM_ORACLE_HDFS_HIVE_BIGDATA_FIELDS : 'com.oracle.bigdata.fields';
COM_ORACLE_HDFS_HIVE_BIGDATA_FILEFORMAT : 'com.oracle.bigdata.fileFormat';
COM_ORACLE_HDFS_HIVE_BIGDATA_EXEC : 'com.oracle.bigdata.exec';
COM_ORACLE_HDFS_HIVE_BIGDATA_QC : 'com.oracle.bigdata.qc';
COM_ORACLE_HDFS_HIVE_BIGDATA_OVERFLOW : 'com.oracle.bigdata.overflow';
COM_ORACLE_HDFS_HIVE_BIGDATA_ROWFORMAT : 'com.oracle.bigdata.rowFormat';
COM_ORACLE_HDFS_HIVE_BIGDATA_TABLENAME : 'com.oracle.bigdata.tableName';
ET_COL : '"col"';
ET_FIELD : '"fields"';
ET_ACTION : '"action"';
ET_REJECT : '"reject"';
ET_SETNULL : '"setnull"';
ET_REPLACE : '"replace"';
ET_VALUE : '"value"';
ET_TRUNCATE : '"truncate"';
ET_ERROR : '"error"';

92
debezium-ddl-parser/src/main/antlr4/io/debezium/ddl/parser/oracle/generated/PlSqlParser.g4
View File

@ -2183,57 +2183,55 @@ et_oracle_datapump
;
et_oracle_hdfs_hive
: COM_ORACLE_HDFS_HIVE_BIGDATA_COLMAP ( '=' | ':' ) (
colmap_entry | ('[' colmap_entry ( ',' colmap_entry)* ']')) et_oracle_hdfs_hive*
| COM_ORACLE_HDFS_HIVE_BIGDATA_ERROROPT ( '=' | ':' ) (
error_element | ('[' error_element ( ',' error_element)* ']' )) et_oracle_hdfs_hive*
| COM_ORACLE_HDFS_HIVE_BIGDATA_FIELDS ( '=' | ':' ) (
field_spec et_data_type
( COMMENT col_comment )? (',' COMMENT col_comment )* ) et_oracle_hdfs_hive*
| COM_ORACLE_HDFS_HIVE_BIGDATA_FILEFORMAT ( '=' | ':' ) (
SEQUENCEFILE
| TEXTFILE
| RCFILE
| ORC
| PARQUET
| INPUTFORMAT quoted_string OUTPUTFORMAT quoted_string ) et_oracle_hdfs_hive*
| COM_ORACLE_HDFS_HIVE_BIGDATA_EXEC ( '=' | ':' ) (
et_directory_spec? et_file_spec ) et_oracle_hdfs_hive*
| COM_ORACLE_HDFS_HIVE_BIGDATA_QC ( '=' | ':' ) (
et_directory_spec? et_file_spec ) et_oracle_hdfs_hive*
| COM_ORACLE_HDFS_HIVE_BIGDATA_OVERFLOW ( '=' | ':' ) (
overflow_element | ('[' overflow_element ( ',' overflow_element)* ']')) et_oracle_hdfs_hive*
| COM_ORACLE_HDFS_HIVE_BIGDATA_ROWFORMAT ( '=' | ':' ) (
DELIMITED?
( FIELDS TERMINATED BY CHARACTER ( ESCAPED BY CHARACTER )
| COLLECTION ITEMS TERMINATED BY CHARACTER
| MAP KEYS TERMINATED BY CHARACTER
| LINES TERMINATED BY CHARACTER
| NULL_ DEFINED AS CHARACTER
: et_oracle_hdfs_hive_parameter_name ('=' | ':')
(
// com.oracle.bigdata.tablename
tableview_name
// com.oracle.bigdata.colmap
// com.oracle.bigdata.erroropt
// com.oracle.bigdata.overflow
| et_oracle_hdfs_hive_parameter_map
| '[' et_oracle_hdfs_hive_parameter_map (',' et_oracle_hdfs_hive_parameter_map)* ']'
// com.oracle.bigdata.fields
| field_spec et_data_type ( COMMENT col_comment )?(',' COMMENT col_comment)*
// com.oracle.bigdata.fileformat
| SEQUENCEFILE
| TEXTFILE
| RCFILE
| ORC
| PARQUET
| INPUTFORMAT quoted_string OUTPUTFORMAT quoted_string
// com.oracle.bigdata.exec
// co.oracle.bigdata.qc
| et_directory_spec? et_file_spec
// com.oracle.bigdata.rowformat
| DELIMITED?
(
FIELDS TERMINATED BY CHARACTER ( ESCAPED BY CHARACTER )
| COLLECTION ITEMS TERMINATED BY CHARACTER
| MAP KEYS TERMINATED BY CHARACTER
| LINES TERMINATED BY CHARACTER
| NULL_ DEFINED AS CHARACTER
)
| SERDE quoted_string ( WITH SERDEPROPERTIES ( quoted_string '=' quoted_string ( ',' quoted_string '=' quoted_string )* ) )?
)
| SERDE quoted_string ( WITH SERDEPROPERTIES ( quoted_string '=' quoted_string ( ',' quoted_string '=' quoted_string)*))? ) et_oracle_hdfs_hive*
| COM_ORACLE_HDFS_HIVE_BIGDATA_TABLENAME ( '=' | ':' ) (
tableview_name ) et_oracle_hdfs_hive*
et_oracle_hdfs_hive?
;
colmap_entry
: LEFT_CURLY_PAREN ET_COL BINDVAR ',' ET_FIELD BINDVAR RIGHT_CURLY_PAREN
et_oracle_hdfs_hive_parameter_map
: LEFT_CURLY_PAREN ( et_oracle_hdfs_hive_parameter_mapentry (',' et_oracle_hdfs_hive_parameter_mapentry)* ) RIGHT_CURLY_PAREN
;
error_element
: LEFT_CURLY_PAREN ET_ACTION ':' ( ET_REJECT | ET_SETNULL | ET_REPLACE) (',' ET_VALUE BINDVAR )
( ',' ET_COL ':'? ( BINDVAR | '[' quoted_string ( ',' quoted_string )* ']' ))? RIGHT_CURLY_PAREN
et_oracle_hdfs_hive_parameter_mapentry
: quoted_string BINDVAR
| quoted_string ':' '[' quoted_string (',' quoted_string)* ']'
| '[' quoted_string (',' quoted_string)* ']'
;
col_comment
: quoted_string
;
overflow_element
: LEFT_CURLY_PAREN ET_ACTION ':' ( ET_TRUNCATE | ET_ERROR )
( ',' ET_COL ':'? ( BINDVAR | '[' quoted_string ( ',' quoted_string )* ']' ))? RIGHT_CURLY_PAREN
;
et_data_type
: ( et_primitive_type
ARRAY '<' et_data_type '>'
@ -2330,11 +2328,15 @@ field_definitions
;
et_field_list
: ( field_spec et_pos_spec? et_datatype_spec? et_init_spec? et_LLS_spec? ) ( ',' et_field_list )*
: field_spec (et_pos_spec? et_datatype_spec? et_init_spec? et_LLS_spec?) (',' et_field_list)*
;
et_pos_spec
: POSITION? '(' (( UNSIGNED_INTEGER | '*' | ( '+' | '-' ) (UNSIGNED_INTEGER | '*' | '-' )) ( ':' | '-' ) ( UNSIGNED_INTEGER | UNSIGNED_INTEGER )) ')'
: POSITION?
'('
( '*'? ('+'|'-')? UNSIGNED_INTEGER? )
( BINDVAR | ( ':' ('+'|'-')? UNSIGNED_INTEGER ) )
')'
;
et_datatype_spec
@ -2348,7 +2350,7 @@ et_datatype_spec
| BINARY_FLOAT EXTERNAL? UNSIGNED_INTEGER? et_delim_spec?
| BINARY_DOUBLE
| RAW UNSIGNED_INTEGER?
| CHAR EXTERNAL? UNSIGNED_INTEGER? et_delim_spec? et_trim_spec? et_date_format_spec?
| CHAR EXTERNAL? ( '(' UNSIGNED_INTEGER ')' )? et_delim_spec? et_trim_spec? et_date_format_spec?
| ( VARCHAR | VARRAW | VARCHARC | VARRAWC ) ('('( UNSIGNED_INTEGER ',')? UNSIGNED_INTEGER ')')
)
;
@ -4816,6 +4818,10 @@ paren_column_list
: LEFT_PAREN column_list RIGHT_PAREN
;
et_oracle_hdfs_hive_parameter_name
: id_expression ('.' id_expression)*
;
// PL/SQL Specs
// NOTE: In reality this applies to aggregate functions only

24
debezium-ddl-parser/src/test/resources/oracle/examples/ddl_create.sql
View File

@ -40,10 +40,10 @@ CREATE TABLE emp_load (first_name CHAR(15), last_name CHAR(20), resume CHAR(2000
FIELDS (first_name VARCHAR(2,12), last_name VARCHAR(2,20), resume VARCHAR(4,10000), picture VARRAW(4,100000)))
LOCATION ('info.dat'));
--CREATE TABLE emp_load (first_name CHAR(15), last_name CHAR(20), year_of_birth INT, phone CHAR(12), area_code CHAR(3), exchange CHAR(3), extension CHAR(4))
-- ORGANIZATION EXTERNAL ( TYPE ORACLE_LOADER DEFAULT DIRECTORY ext_tab_dir
-- ACCESS PARAMETERS ( FIELDS RTRIM (first_name (1:15) CHAR(15), last_name (*:+20), year_of_birth (36:39), phone (40:52), area_code (*-12: +3), exchange (*+1: +3), extension (*+1: +4)))
-- LOCATION ('info.dat'));
CREATE TABLE emp_load (first_name CHAR(15), last_name CHAR(20), year_of_birth INT, phone CHAR(12), area_code CHAR(3), exchange CHAR(3), extension CHAR(4))
ORGANIZATION EXTERNAL ( TYPE ORACLE_LOADER DEFAULT DIRECTORY ext_tab_dir
ACCESS PARAMETERS ( FIELDS RTRIM (first_name (1:15) CHAR(15), last_name (*:+20), year_of_birth (36:39), phone (40:52), area_code (*-12: +3), exchange (*+1: +3), extension (*+1: +4)))
LOCATION ('info.dat'));
CREATE TABLE emp_load (first_name CHAR(15), last_name CHAR(20), year_of_birth CHAR(4))
ORGANIZATION EXTERNAL ( TYPE ORACLE_LOADER DEFAULT DIRECTORY ext_tab_dir
@ -86,14 +86,14 @@ CREATE TABLE emp_load (first_name CHAR(15), last_name CHAR(20), year_of_birth CH
ACCESS PARAMETERS (RECORDS FIXED 20 FIELDS (first_name CHAR(7), last_name CHAR(8), year_of_birth CHAR(4)))
LOCATION ('info.dat'));
--CREATE TABLE CUSTOMER_TABLE (cust_num VARCHAR2(10), order_num VARCHAR2(20), order_date DATE, item_cnt NUMBER, description VARCHAR2(100), order_total NUMBER(8,2))
-- ORGANIZATION EXTERNAL ( TYPE ORACLE_HIVE
-- ACCESS PARAMETERS (
-- com.oracle.bigdata.tableName: order_db.order_summary
-- com.oracle.bigdata.colMap: {"col":"ITEM_CNT", "field":"order_line_item_count"}
-- com.oracle.bigdata.overflow: {"action":"ERROR", "col":"DESCRIPTION"}
-- com.oracle.bigdata.errorOpt: [{"action":"replace", "value":"INV_NUM" , "col":["CUST_NUM","ORDER_NUM"]} , {"action":"reject", "col":"ORDER_TOTAL"}]
-- ));
CREATE TABLE CUSTOMER_TABLE (cust_num VARCHAR2(10), order_num VARCHAR2(20), order_date DATE, item_cnt NUMBER, description VARCHAR2(100), order_total NUMBER(8,2))
ORGANIZATION EXTERNAL ( TYPE ORACLE_HIVE
ACCESS PARAMETERS (
com.oracle.bigdata.tableName: order_db.order_summary
com.oracle.bigdata.colMap: {"col":"ITEM_CNT", "field":"order_line_item_count"}
com.oracle.bigdata.overflow: {"action":"ERROR", "col":"DESCRIPTION"}
com.oracle.bigdata.errorOpt: [{"action":"replace", "value":"INV_NUM" , "col":["CUST_NUM","ORDER_NUM"]} , {"action":"reject", "col":"ORDER_TOTAL"}]
));
-- Create index
create index hr.name on hr.table (id,data) tablespace ts;
create unique index idx_eshp_auction_file_history_id on eshp_auction_file_history(history_id);