From a553bde8774a55c462936d04bba942dd2a6e5a59 Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Mon, 30 Oct 2023 11:25:17 +0000 Subject: [PATCH 01/62] bump snapshot --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 4 ++-- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 19 insertions(+), 19 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 51ff6fb06..df38e7267 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 28de0db21..cb803d1ba 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index d05e592f5..44dd1fe8c 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT CloudSQL MySQL plugin @@ -45,7 +45,7 @@ io.cdap.plugin mysql-plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 2f974e854..8faab79ba 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 0ecbfb445..683dd2f43 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index a43bcb92e..39c3fcd52 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index dbcd46d47..b823356d8 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 8a8dcd1c4..d8a78cd4d 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 0e9a09e02..7ece99f31 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 5c50a857e..c9dbaf035 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 45e2b9c03..f5fc81a93 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index f691a15f2..bb5caf4c0 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 900e430fe..f7b559439 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index e0ed7ff50..b21152fcd 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index a6b40960c..201cc804d 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 7f3e6f14c..8f086e482 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 86b40a38e..26ce2e396 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index fa770a19a..c351a9d96 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT teradata-plugin From f949fbe4a58ca5db5d256ab2c600a4888cc9be86 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Wed, 1 Nov 2023 17:50:59 +0530 Subject: [PATCH 02/62] cdap e2e framework snapshot version fix --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 201cc804d..74932ee90 100644 --- a/pom.xml +++ b/pom.xml @@ -723,7 +723,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.3.0-SNAPSHOT + 0.4.0-SNAPSHOT test From 1ea0d68adb1b7eac1dafb48b754c61f4f5ab095f Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Wed, 20 Sep 2023 17:09:15 +0530 Subject: [PATCH 03/62] Redshift source and connector plugin added. Redshift source and connector plugin added. --- .../docs/Redshift-batchsource.md | 102 ++++++ .../docs/Redshift-connector.md | 26 ++ .../icons/Redshift-batchsource.png | Bin 0 -> 6145 bytes amazon-redshift-plugin/pom.xml | 139 ++++++++ .../amazon/redshift/RedshiftConnector.java | 117 ++++++ .../redshift/RedshiftConnectorConfig.java | 87 +++++ .../amazon/redshift/RedshiftConstants.java | 27 ++ .../amazon/redshift/RedshiftDBRecord.java | 129 +++++++ .../amazon/redshift/RedshiftSchemaReader.java | 117 ++++++ .../amazon/redshift/RedshiftSource.java | 128 +++++++ .../redshift/RedshiftConnectorTest.java | 39 ++ .../redshift/RedshiftConnectorUnitTest.java | 55 +++ .../redshift/RedshiftDBRecordUnitTest.java | 155 ++++++++ .../RedshiftFailedConnectionTest.java | 38 ++ .../redshift/RedshiftPluginTestBase.java | 218 ++++++++++++ .../redshift/RedshiftPluginTestSuite.java | 31 ++ .../redshift/RedshiftSourceTestRun.java | 332 ++++++++++++++++++ .../widgets/Redshift-batchsource.json | 240 +++++++++++++ .../widgets/Redshift-connector.json | 75 ++++ pom.xml | 1 + 20 files changed, 2056 insertions(+) create mode 100644 amazon-redshift-plugin/docs/Redshift-batchsource.md create mode 100644 amazon-redshift-plugin/docs/Redshift-connector.md create mode 100644 amazon-redshift-plugin/icons/Redshift-batchsource.png create mode 100644 amazon-redshift-plugin/pom.xml create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java create mode 100644 amazon-redshift-plugin/widgets/Redshift-batchsource.json create mode 100644 amazon-redshift-plugin/widgets/Redshift-connector.json diff --git a/amazon-redshift-plugin/docs/Redshift-batchsource.md b/amazon-redshift-plugin/docs/Redshift-batchsource.md new file mode 100644 index 000000000..38873b15a --- /dev/null +++ b/amazon-redshift-plugin/docs/Redshift-batchsource.md @@ -0,0 +1,102 @@ +# Amazon Redshift Batch Source + +Description +----------- +Reads from an Amazon Redshift database using a configurable SQL query. +Outputs one record for each row returned by the query. + + +Use Case +-------- +The source is used whenever you need to read from an Amazon Redshift database. For example, you may want +to create daily snapshots of a database table by using this source and writing to +a TimePartitionedFileSet. + + +Properties +---------- +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**JDBC Driver name:** Name of the JDBC driver to use. + +**Host:** Host URL of the current master instance of Redshift cluster. + +**Port:** Port that Redshift master instance is listening to. + +**Database:** Redshift database name. + +**Import Query:** The SELECT query to use to import data from the specified table. +You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should +contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. +The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. +The '$CONDITIONS' string is not required if numSplits is set to one. + +**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. + +**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. + +**Number of Splits to Generate:** Number of splits to generate. + +**Username:** User identity for connecting to the specified database. + +**Password:** Password to use to connect to the specified database. + +**Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments +will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. + +**Schema:** The schema of records output by the source. This will be used in place of whatever schema comes +back from the query. However, it must match the schema that comes back from the query, +except it can mark fields as nullable and can contain a subset of the fields. + +**Fetch Size:** The number of rows to fetch at a time per split. Larger fetch size can result in faster import, +with the tradeoff of higher memory usage. + +Example +------ +Suppose you want to read data from an Amazon Redshift database named "prod" that is running on +"redshift.xyz.eu-central-1.redshift.amazonaws.com", port 5439, as "sa" user with "Test11" password. +Ensure that the driver for Redshift is installed (you can also provide driver name for some specific driver, +otherwise "redshift" will be used), then configure the plugin with:then configure plugin with: + +``` +Reference Name: "src1" +Driver Name: "redshift" +Host: "redshift.xyz.eu-central-1.redshift.amazonaws.com" +Port: 5439 +Database: "prod" +Import Query: "select id, name, email, phone from users;" +Number of Splits to Generate: 1 +Username: "sa" +Password: "Test11" +``` + +Data Types Mapping +------------------ + +Mapping of Redshift types to CDAP schema: + +| Redshift Data Type | CDAP Schema Data Type | Comment | +|-----------------------------------------------------|-----------------------|----------------------------------| +| bigint | long | | +| boolean | boolean | | +| character | string | | +| character varying | string | | +| double precision | double | | +| integer | int | | +| numeric(precision, scale)/decimal(precision, scale) | decimal | | +| numeric(with 0 precision) | string | | +| real | float | | +| smallint | int | | +| smallserial | int | | +| text | string | | +| date | date | | +| time [ (p) ] [ without time zone ] | time | | +| time [ (p) ] with time zone | string | | +| timestamp [ (p) ] [ without time zone ] | timestamp | | +| timestamp [ (p) ] with time zone | timestamp | stored in UTC format in database | +| xml | string | | +| json | string | | +| super | string | | +| geometry | bytes | | +| hllsketch | string | | diff --git a/amazon-redshift-plugin/docs/Redshift-connector.md b/amazon-redshift-plugin/docs/Redshift-connector.md new file mode 100644 index 000000000..368d9e09f --- /dev/null +++ b/amazon-redshift-plugin/docs/Redshift-connector.md @@ -0,0 +1,26 @@ +# Amazon Redshift Connection + +Description +----------- +Use this connection to access data in an Amazon Redshift database using JDBC. + +Properties +---------- +**Name:** Name of the connection. Connection names must be unique in a namespace. + +**Description:** Description of the connection. + +**JDBC Driver name:** Name of the JDBC driver to use. + +**Host:** Host of the current master instance of Redshift cluster. + +**Port:** Port that Redshift master instance is listening to. + +**Database:** Redshift database name. + +**Username:** User identity for connecting to the specified database. + +**Password:** Password to use to connect to the specified database. + +**Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments +will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/amazon-redshift-plugin/icons/Redshift-batchsource.png b/amazon-redshift-plugin/icons/Redshift-batchsource.png new file mode 100644 index 0000000000000000000000000000000000000000..11c334799daa890a320e5d4aa934d44a71bc988f GIT binary patch literal 6145 zcmZu#1yqzx+a^||Q;_adIwY4aC6*4Qdx2$PS&1bT0YOSB5m-P{K$=Aw=@M2-atS3C zqy(hl$M=1|@B8aHXP%k4=epy%=FEA{GcgAGT4cnG#5g!OWDm8~jc)poKaG&!=A9Vs z$#c`;`Wk7e;#7<2-`bB9h`*& zAl^4<92|uJxtk%x8E(%W0P*zll?woJ{)Ldc8UHbhaI*gefqMWsE%Xf7)u1qEb}3;= zVNp&+Vs>_R1(=hIoRPZbKkhd(Ag3D~?ky)G;_vS->@OhDAdiwoT# zgnR?N;PwGRUcOv^2l-bVb!T5k7}y&QhI+C8iEHlweF_J1a{fv5@8_R7A>RK^_$QiVSudEWxWws zk@zOlzcxydSU~s-5C?~L^r5W!WrW127Uq3WOr7C^6(52)IPjce(E0uiBta9&8DqCFWCwAOt zVy|_7-r0B-*v%1JTRD@l({?(tjYRg193K8W96r3aP&m^#90cvn@-(`yf90YLk9&&-Wm|%K8QXz1(z{; z>fEL99bj9DIec3ulx8C}OpWjHwr+s8)3J#3tZ5n7udB!qJ@wY|ZEGjDk_*vgq{KP& zS4WOq%C}#=*-Ob9vj=J>pnzCmVZw6)7ZV4mmT5~ z3Y|0z;lfl4UFs~uZ$%59szrESZZHl5rfCE?>Hzm?*D92sX+=6C`$LXVL*3GmmkL89 z;n1|zm$UwoDx=7Wl=raXq;FLpgA%-YjV{^WpQ&+XFlfEJHLY<(s>risp&+3!NS!h1 z6+i*{GC2O4znqEDEl%-g8K>Y;?d?@Z@xx>1% z$H1z?VjN4kp+h6NI+GSmZ=^^#%E~8_=JFM9d~li2vk>><5k+EE!4@{J*I(DJ5xMhx zhd;Y(o*xXLcr2bzl-G6OD#Ae-3~Cs@TMATiC$a@7L$h)%-~A@M(_|j$6MpXtUZ+g$ z>$h78Q)O3vOmJ@4R@Chwq>WVdTncqw76~wwN|&&H&6hVJ(@Oqv>zWqN@++JTS{9=R z`b3Ob>l`XXjPLmsGC50cRSEo%UTc`kmHu@|Jhw_Gm`qJv0UnTCe@@*N$Q@ksVEO*> z&Ub%Dp5F-^?`-c?MFx~8s&uK97nRFSJ1aJmA?|}j4wi$B*Yvk!(#{`s-rloSMuiO@ zT+E6AFC)22KU69A?DW8fVPE+hC6x!rd=BK|Q*4 zky(#(W#M3#nm4o7J|FTU8@eJ}6U?I|6%=|z0XH}K5L4-fmCAJT!KiE6bV7dpoWd;l zn$`n3I{Mzc=wa#4EIggz#E5z+uUylABTUkvq8~ zpwZ{Cp(;lKhIb2<)rm%k$gV&9cDKKJCxG|}6sAtOahZD60&}7=lhT#hWjW;fm41QJ z$FNr*gGN}bz;{!i5>SIl72-2Zw=u#E*ZyokTVUO*gTEdJ9d2g7pL?(rEx3QVm_b>l zgnS+7H&ynAs&ao)6~F(mXh)&oi)Z_@oaQFFCSwi0zL;Ml(=`}V9w_4_(Zz#O#AjtK zU}pT!q}Zy@)2PUz-dDXC$upv~`s zi76W(Uq!`&b8|G=V4_*Czj=O>anN-A?@|MHU$0feFOmL1-+IrHt%~{k+jO;Fj|Jbg zr`p`HJ?Uz{!rtd8+nZ+?sWb>H-MrlFB&5J=Q`$(SF8=lNV-PtsO?A#AJvN92``HKB z8jH;^Z4iU(&zV&lpmk=?{nH4oXut`A>OtQ zRs)gi&-Fa7jjzrspwt|n9CHr}+MQf6tuKs3w7Uoo>*dIK59OEw?HrF4>kVnn^kgo4 z!cTZ)I?YnHsi!~RCgxYLH@>%Gh1}Q=x zxgDFMcv+{9=f={SInK9n)=3POT`G#R;_oqEzc*GvHNNaC?bIIO%e9bEd^8lx_Q|k1 z7xl4zqC@8H3dkqUW0#HjYQV?qU>&41pyXLpD(7==s(SUxwH;sc3U3O9RQ^o5DS~=4 za2PNkIfEL7I_5J;zbTGLXEYil#}RHZU*LPC{+S_4=QYlJ2jQN!)#=HYh7WcnRc`>$ z8Y$^m+jE(^gk5_5n=XQK9fT*d>HIhO><7>VV*rL#9rbF>AnPPA@AWq0K4<6UChR>jFpIrzeC zndogOJnp9@@_rs$DtDrvG+zL%Q+XLxtFQ#1!I}Zh&a`(;CI9OF;vNyyC7s z2&ozVdZyS<%mmA*HxN|-GoPS<`)o<~S1`L#QW-v3fH9!Re2G`0xq(F_k8R@SAZpNr zXJVzenD*$7g_>@Y_LsVEl#FcJU$PCjIDJsg-+R5u($c1bUd&ZiSMJJWA^{W#(t6K- z)Ry*$7vWz{ouU}jnnpq7=-*7d?sy|z_C-jG8U!4oj&q|z(q+XG4ol{b-GNU>8R+y= zk9ujAemw7HCnA&B&>cg$#4EdMmyu=}m7}|u0;O1H3V2Ld=plmMIlyw458z?@J3E@G zUl3u24<&`DuSky>4*A&`!=n+a1oUI+!Wh4Dr7p1z74*J;7W!zyg=@c>MUkoW?} zhMI5p9l)!Vw4XrYw=Jy>ow@E*TKkqU(2)woE1kIn*sMJzx4s^?dfKwmbK>@%kXPQ2a!u zMT=N8Oa{^A{uA5!wO0C7*wRtc(|W21jOmdu4{OkwQMUR)LUhWv4T&6Kwxgc`m7Z5W zdyqSoIp~}y8ce3aKC$qXXJ795R$t&IcG?5LFnO+6U3_e_uHsUj*sszH*!dpV)b10t zgteySs%o*`rZ=U^bl4j8Cd`%2ZY-1O?#{yv^UN&e2cR>@64=yD@NZNMClY%bsXgfW zY9TC*{*XOas~$AIhdF%qhMNFb8cwiWhROSGj$(h@+t@9RF6F!itRWm0L(i44s5TLg!7JTIm?~iLJk|xHbofX@l zwJvWlYrPdXS9SHi`-Y3FGw?`ZA!nN&9(HM!401$AbZiQ}w&Ch8>Hi(0LV*|G;D?B3 z&qplwJsH0|jk~evnXt4VD+BZ>p1X7?UM0{Zy&V6@T8tuS*!&sTbS+q2++gIyopb(t zc1+z79YpG`qmH-#-RJ|~+J#nG)yuQn^*3gZqw1P4*=qq69R>&VlT!Y9^0sG<;~!eo zNB*dwPs>ucZ^OAhYC_u79;zC)**+U3Y8`&<#!mnQ0FUF<^Z7BC2*hG{Jl~Eoes!^b zZc7`*N89^&S4yxONs~P8Xhy_&p!w8|Fdqb`K6WW`~r7jQra#r2D4OPB~dqSZ`YOj~QPcGIHtu`o?edGQP zV|52XU5ULqi`f`Q)Rj$}EHA7Fk|I)T5EG=b zErUCoO}Gn>8PRchMH13XOY`ejaiMdpxAdd@&MmD-I0UiQJMp&p zraOna74A_wzGesGRd-LCDO6>=NZI?k2Yyr{FiPIfomyy-s_UHxRt32u#Cz)0i61Cm z4;JPZrJNn->S@0dBj(Z0<9U(BOlidzpR#2BfB{#Q&KfNi8E&SDPU6tGt^ZD!oV>bo z1#fwBuk_Y}Xv%S3aUGAm=ST>OV7plz#%`ZZ=WU*bL&m90d!n<>hl5qC>o1cM1~}=F zRM^X@CysTNOTIc_gUtnp*spzaY^n&$AdpE|d}KFS4GPn?h#vQ2NFr>lf+MxvDpQGT z+b709ELbS_0S@t^?~b9~zMx8?4M)Xf3>qf!T9>X8<4-I;cjP-9E_DK5g{^j82mbWKM6OK?h#4|H4C#JD2Vq?em3J)QM<`Az z?~7&1ivnMFbO@8(7S5BDHPv!V)U&ZxW#kv%H|7}zYBU{#}?_K zdjP6{z#*h-n>+0~1$%I?HMfLZ?V6iSjd+0fb1)A5ftqA^%cm-fd%{*uJ(2#R*e%`u zhEHi%r`QmAMc$Q3bMcIhU)90RX5*WWGbhT0-Qk^OPH95mMklt5z1KP??HwXMdAq4S`V9|KEQV7*YnyqMgq11WAZSv;KrnIEV3e!svBM{JhwqwuaT$n~SV_o1xDOO>OF zM|$N$qTxL`9L+)2dq092FOzE}%g@B^WY+!d1czz{NOT!r`Ud*(0ZdOLYJZtiymV-$ zS+7WscHpWw;HynxTkd3o36F_bC*S49UUtoMUB}y2HnTlA?H(_qbEy+~?B~-^FVhgQ zzjb|j_&M9vjnMkX%%gS!>4Q!E1Es6n^gh`o`Qb+7HF#&~RgU;?hen-K`5@{cyov!5 z%aTpXlbl{@^Y?3m*&B1-N)riZs~xz)UFwTGTBk|&CWPyDx10d*R5IKVzpMUGScx?D zfC%U-(pNirZ?xhov{5Z$*Dg2xwpozkv+K<|jl6dY&%$A8Grf|@t4|9$!wp$O$bxCk zXoH)$4L(wpJWNVrh(5W~toGx;l_AsNL47fO$AcYD?95Rx>NX+}&JgUi*K(c^1MLgK z%D1Ub;fzT}q-h*64kx*ORbt9dWxY&fR9&_uJG zXS@l?h$d;j8U1 zWh7h*i7a)DGv|9;jJeIv#}#3$PhOdL=RTpKP>@d;ytElu<&t^flOKc*w~AC%+TNGH z@MnUHw0sMYV_?_dAZx+(iO(SMW$WZle5|YF=W?w__?`4UJtt_cm0vo2_MwH~)rc?08laQF?douUlsQ((k z{k?KQ%X}{HbpV+1yzRLXvq>H3IGEg(TDfa$2v{&BW|qsOSvBHXFhN2GqQAmv6ia5U zx;qn(bMp*;=#^EMa%#U2OpW{mTbRW~g*p*xx|9LPTei40kVR}$2@ + + + + database-plugins-parent + io.cdap.plugin + 1.12.0-SNAPSHOT + + + Amazon Redshift plugin + amazon-redshift-plugin + 4.0.0 + + + 2.1.0.18 + + + + + redshift + http://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release + + + + + + io.cdap.cdap + cdap-etl-api + + + io.cdap.plugin + database-commons + ${project.version} + + + io.cdap.plugin + hydrator-common + + + com.google.guava + guava + + + + + com.amazon.redshift + redshift-jdbc42 + ${redshift-jdbc.version} + test + + + io.cdap.plugin + database-commons + ${project.version} + test-jar + test + + + io.cdap.cdap + hydrator-test + + + io.cdap.cdap + cdap-data-pipeline3_2.12 + + + junit + junit + + + org.mockito + mockito-core + test + + + io.cdap.cdap + cdap-api + provided + + + org.jetbrains + annotations + RELEASE + compile + + + + + + io.cdap + cdap-maven-plugin + + + org.apache.felix + maven-bundle-plugin + 5.1.2 + true + + + <_exportcontents> + io.cdap.plugin.amazon.redshift.*; + io.cdap.plugin.db.source.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* + + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + + + + diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java new file mode 100644 index 000000000..fb8cac4a7 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java @@ -0,0 +1,117 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Category; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.connector.Connector; +import io.cdap.cdap.etl.api.connector.ConnectorSpec; +import io.cdap.cdap.etl.api.connector.ConnectorSpecRequest; +import io.cdap.cdap.etl.api.connector.PluginSpec; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.common.ReferenceNames; +import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBPath; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Amazon Redshift Database Connector that connects to Amazon Redshift database via JDBC. + */ +@Plugin(type = Connector.PLUGIN_TYPE) +@Name(RedshiftConnector.NAME) +@Description("Connection to access data in Amazon Redshift using JDBC.") +@Category("Database") +public class RedshiftConnector extends AbstractDBSpecificConnector { + public static final String NAME = RedshiftConstants.PLUGIN_NAME; + private final RedshiftConnectorConfig config; + + public RedshiftConnector(RedshiftConnectorConfig config) { + super(config); + this.config = config; + } + + @Override + protected DBConnectorPath getDBConnectorPath(String path) throws IOException { + return new DBPath(path, true); + } + + @Override + public boolean supportSchema() { + return true; + } + + @Override + protected Class getDBRecordType() { + return RedshiftDBRecord.class; + } + + @Override + public StructuredRecord transform(LongWritable longWritable, RedshiftDBRecord redshiftDBRecord) { + return redshiftDBRecord.getRecord(); + } + + @Override + protected SchemaReader getSchemaReader(String sessionID) { + return new RedshiftSchemaReader(sessionID); + } + + @Override + protected String getTableName(String database, String schema, String table) { + return String.format("\"%s\".\"%s\"", schema, table); + } + + @Override + protected String getRandomQuery(String tableName, int limit) { + return String.format("SELECT * FROM %s\n" + + "TABLESAMPLE BERNOULLI (100.0 * %d / (SELECT COUNT(*) FROM %s))", + tableName, limit, tableName); + } + + @Override + protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath path, + ConnectorSpec.Builder builder) { + Map sourceProperties = new HashMap<>(); + setConnectionProperties(sourceProperties, request); + builder + .addRelatedPlugin(new PluginSpec(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, sourceProperties)); + + String schema = path.getSchema(); + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.NUM_SPLITS, "1"); + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.FETCH_SIZE, + RedshiftSource.RedshiftSourceConfig.DEFAULT_FETCH_SIZE); + String table = path.getTable(); + if (table == null) { + return; + } + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.IMPORT_QUERY, + getTableQuery(path.getDatabase(), schema, table)); + sourceProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java new file mode 100644 index 000000000..f05f26d10 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java @@ -0,0 +1,87 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; + +import javax.annotation.Nullable; + +/** + * Configuration for Redshift connector + */ +public class RedshiftConnectorConfig extends AbstractDBConnectorConfig { + + @Name(ConnectionConfig.HOST) + @Description( + "The endpoint of the Amazon Redshift cluster.") + @Macro + private String host; + + @Name(ConnectionConfig.PORT) + @Description("Database port number") + @Macro + @Nullable + private Integer port; + + @Name(ConnectionConfig.DATABASE) + @Description("Database name to connect to") + @Macro + private String database; + + public RedshiftConnectorConfig(String username, String password, String jdbcPluginName, + String connectionArguments, String host, + String database, @Nullable Integer port) { + this.user = username; + this.password = password; + this.jdbcPluginName = jdbcPluginName; + this.connectionArguments = connectionArguments; + this.host = host; + this.database = database; + this.port = port; + } + + public String getDatabase() { + return database; + } + + public String getHost() { + return host; + } + + public int getPort() { + return port == null ? 5439 : port; + } + + @Override + public String getConnectionString() { + return String.format( + RedshiftConstants.REDSHIFT_CONNECTION_STRING_FORMAT, + host, + getPort(), + database); + } + + @Override + public boolean canConnect() { + return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && + !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); + } +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java new file mode 100644 index 000000000..081052fb1 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +/** Amazon Redshift constants. */ +public final class RedshiftConstants { + + private RedshiftConstants() { + } + + public static final String PLUGIN_NAME = "Redshift"; + public static final String REDSHIFT_CONNECTION_STRING_FORMAT = "jdbc:redshift://%s:%s/%s"; +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java new file mode 100644 index 000000000..38e9140d8 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java @@ -0,0 +1,129 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.util.DBUtils; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; + +/** + * Writable class for Redshift Source + */ +public class RedshiftDBRecord extends DBRecord { + + /** + * Used in map-reduce. Do not remove. + */ + @SuppressWarnings("unused") + public RedshiftDBRecord() { + } + + @Override + protected void handleField(ResultSet resultSet, StructuredRecord.Builder recordBuilder, Schema.Field field, + int columnIndex, int sqlType, int sqlPrecision, int sqlScale) throws SQLException { + ResultSetMetaData metadata = resultSet.getMetaData(); + String columnTypeName = metadata.getColumnTypeName(columnIndex); + if (isUseSchema(metadata, columnIndex)) { + setFieldAccordingToSchema(resultSet, recordBuilder, field, columnIndex); + return; + } + + // HandleTimestamp + if (sqlType == Types.TIMESTAMP && columnTypeName.equalsIgnoreCase("timestamp")) { + Timestamp timestamp = resultSet.getTimestamp(columnIndex, DBUtils.PURE_GREGORIAN_CALENDAR); + if (timestamp != null) { + ZonedDateTime zonedDateTime = OffsetDateTime.of(timestamp.toLocalDateTime(), OffsetDateTime.now().getOffset()) + .atZoneSameInstant(ZoneId.of("UTC")); + Schema nonNullableSchema = field.getSchema().isNullable() ? + field.getSchema().getNonNullable() : field.getSchema(); + setZonedDateTimeBasedOnOutputSchema(recordBuilder, nonNullableSchema.getLogicalType(), + field.getName(), zonedDateTime); + } else { + recordBuilder.set(field.getName(), null); + } + return; + } + + // HandleTimestampTZ + if (sqlType == Types.TIMESTAMP && columnTypeName.equalsIgnoreCase("timestamptz")) { + OffsetDateTime timestamp = resultSet.getObject(columnIndex, OffsetDateTime.class); + if (timestamp != null) { + recordBuilder.setTimestamp(field.getName(), timestamp.atZoneSameInstant(ZoneId.of("UTC"))); + } else { + recordBuilder.set(field.getName(), null); + } + return; + } + + // HandleNumeric + int columnType = metadata.getColumnType(columnIndex); + if (columnType == Types.NUMERIC) { + Schema nonNullableSchema = field.getSchema().isNullable() ? + field.getSchema().getNonNullable() : field.getSchema(); + int precision = metadata.getPrecision(columnIndex); + if (precision == 0 && Schema.Type.STRING.equals(nonNullableSchema.getType())) { + // When output schema is set to String for precision less numbers + recordBuilder.set(field.getName(), resultSet.getString(columnIndex)); + } else if (Schema.LogicalType.DECIMAL.equals(nonNullableSchema.getLogicalType())) { + BigDecimal originalDecimalValue = resultSet.getBigDecimal(columnIndex); + if (originalDecimalValue != null) { + BigDecimal newDecimalValue = new BigDecimal(originalDecimalValue.toPlainString()) + .setScale(nonNullableSchema.getScale(), RoundingMode.HALF_EVEN); + recordBuilder.setDecimal(field.getName(), newDecimalValue); + } + } + return; + } + setField(resultSet, recordBuilder, field, columnIndex, sqlType, sqlPrecision, sqlScale); + } + + private void setZonedDateTimeBasedOnOutputSchema(StructuredRecord.Builder recordBuilder, + Schema.LogicalType logicalType, + String fieldName, + ZonedDateTime zonedDateTime) { + if (Schema.LogicalType.DATETIME.equals(logicalType)) { + recordBuilder.setDateTime(fieldName, zonedDateTime.toLocalDateTime()); + } else if (Schema.LogicalType.TIMESTAMP_MICROS.equals(logicalType)) { + recordBuilder.setTimestamp(fieldName, zonedDateTime); + } + } + + private static boolean isUseSchema(ResultSetMetaData metadata, int columnIndex) throws SQLException { + String columnTypeName = metadata.getColumnTypeName(columnIndex); + // If the column Type Name is present in the String mapped Redshift types then return true. + return RedshiftSchemaReader.STRING_MAPPED_REDSHIFT_TYPES_NAMES.contains(columnTypeName); + } + + @Override + protected SchemaReader getSchemaReader() { + return new RedshiftSchemaReader(); + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java new file mode 100644 index 000000000..df9938a45 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java @@ -0,0 +1,117 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.db.CommonSchemaReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; +import java.util.Set; + +/** + * Redshift Schema Reader class + */ +public class RedshiftSchemaReader extends CommonSchemaReader { + + private static final Logger LOG = LoggerFactory.getLogger(RedshiftSchemaReader.class); + + public static final Set STRING_MAPPED_REDSHIFT_TYPES_NAMES = ImmutableSet.of( + "timetz", "money" + ); + + private final String sessionID; + + public RedshiftSchemaReader() { + this(null); + } + + public RedshiftSchemaReader(String sessionID) { + super(); + this.sessionID = sessionID; + } + + @Override + public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLException { + String typeName = metadata.getColumnTypeName(index); + int columnType = metadata.getColumnType(index); + + if (STRING_MAPPED_REDSHIFT_TYPES_NAMES.contains(typeName)) { + return Schema.of(Schema.Type.STRING); + } + if (typeName.equalsIgnoreCase("INT")) { + return Schema.of(Schema.Type.INT); + } + if (typeName.equalsIgnoreCase("BIGINT")) { + return Schema.of(Schema.Type.LONG); + } + + // If it is a numeric type without precision then use the Schema of String to avoid any precision loss + if (Types.NUMERIC == columnType) { + int precision = metadata.getPrecision(index); + if (precision == 0) { + LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " + + "converting into STRING type to avoid any precision loss.", + metadata.getColumnName(index), + metadata.getColumnTypeName(index))); + return Schema.of(Schema.Type.STRING); + } + } + + if (typeName.equalsIgnoreCase("timestamp")) { + return Schema.of(Schema.LogicalType.DATETIME); + } + + return super.getSchema(metadata, index); + } + + @Override + public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { + if (sessionID == null) { + return false; + } + return metadata.getColumnName(index).equals("c_" + sessionID) || + metadata.getColumnName(index).equals("sqn_" + sessionID); + } + + @Override + public List getSchemaFields(ResultSet resultSet) throws SQLException { + List schemaFields = Lists.newArrayList(); + ResultSetMetaData metadata = resultSet.getMetaData(); + // ResultSetMetadata columns are numbered starting with 1 + for (int i = 1; i <= metadata.getColumnCount(); i++) { + if (shouldIgnoreColumn(metadata, i)) { + continue; + } + String columnName = metadata.getColumnName(i); + Schema columnSchema = getSchema(metadata, i); + // Setting up schema as nullable as cdata driver doesn't provide proper information about isNullable. + columnSchema = Schema.nullableOf(columnSchema); + Schema.Field field = Schema.Field.of(columnName, columnSchema); + schemaFields.add(field); + } + return schemaFields; + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java new file mode 100644 index 000000000..1b5894de9 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java @@ -0,0 +1,128 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Metadata; +import io.cdap.cdap.api.annotation.MetadataProperty; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.cdap.etl.api.connector.Connector; +import io.cdap.plugin.common.Asset; +import io.cdap.plugin.common.ConfigUtil; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; +import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.DBUtils; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; + +import java.util.Collections; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * Batch source to read from an Amazon Redshift database. + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(RedshiftConstants.PLUGIN_NAME) +@Description( + "Reads from a Amazon Redshift database table(s) using a configurable SQL query." + + " Outputs one record for each row returned by the query.") +@Metadata(properties = {@MetadataProperty(key = Connector.PLUGIN_TYPE, value = RedshiftConnector.NAME)}) +public class RedshiftSource + extends AbstractDBSource { + + private final RedshiftSourceConfig redshiftSourceConfig; + + public RedshiftSource(RedshiftSourceConfig redshiftSourceConfig) { + super(redshiftSourceConfig); + this.redshiftSourceConfig = redshiftSourceConfig; + } + + @Override + protected SchemaReader getSchemaReader() { + return new RedshiftSchemaReader(); + } + + @Override + protected Class getDBRecordType() { + return RedshiftDBRecord.class; + } + + @Override + protected String createConnectionString() { + return String.format( + RedshiftConstants.REDSHIFT_CONNECTION_STRING_FORMAT, + redshiftSourceConfig.connection.getHost(), + redshiftSourceConfig.connection.getPort(), + redshiftSourceConfig.connection.getDatabase()); + } + + @Override + protected LineageRecorder getLineageRecorder(BatchSourceContext context) { + String fqn = DBUtils.constructFQN("redshift", redshiftSourceConfig.getConnection().getHost(), + redshiftSourceConfig.getConnection().getPort(), + redshiftSourceConfig.getConnection().getDatabase(), + redshiftSourceConfig.getReferenceName()); + Asset.Builder assetBuilder = Asset.builder(redshiftSourceConfig.getReferenceName()).setFqn(fqn); + return new LineageRecorder(context, assetBuilder.build()); + } + + /** + * Redshift source config. + */ + public static class RedshiftSourceConfig extends AbstractDBSpecificSourceConfig { + + @Name(ConfigUtil.NAME_USE_CONNECTION) + @Nullable + @Description("Whether to use an existing connection.") + private Boolean useConnection; + + @Name(ConfigUtil.NAME_CONNECTION) + @Macro + @Nullable + @Description("The existing connection to use.") + private RedshiftConnectorConfig connection; + + @Override + public Map getDBSpecificArguments() { + return Collections.emptyMap(); + } + + @Override + public Integer getFetchSize() { + Integer fetchSize = super.getFetchSize(); + return fetchSize == null ? Integer.parseInt(DEFAULT_FETCH_SIZE) : fetchSize; + } + + @Override + protected RedshiftConnectorConfig getConnection() { + return connection; + } + + @Override + public void validate(FailureCollector collector) { + ConfigUtil.validateConnection(this, useConnection, connection, collector); + super.validate(collector); + } + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java new file mode 100644 index 000000000..a43eb4302 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.plugin.db.connector.DBSpecificConnectorBaseTest; +import org.junit.Test; + +import java.io.IOException; + +/** + * Unit tests for {@link RedshiftConnector} + */ +public class RedshiftConnectorTest extends DBSpecificConnectorBaseTest { + + private static final String JDBC_DRIVER_CLASS_NAME = "com.amazon.redshift.Driver"; + + @Test + public void test() throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { + test(new RedshiftConnector( + new RedshiftConnectorConfig(username, password, JDBC_PLUGIN_NAME, connectionArguments, host, database, + port)), + JDBC_DRIVER_CLASS_NAME, RedshiftConstants.PLUGIN_NAME); + } +} + diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java new file mode 100644 index 000000000..39579cb60 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Unit tests for {@link RedshiftConnector} + */ +public class RedshiftConnectorUnitTest { + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + private static final RedshiftConnector CONNECTOR = new RedshiftConnector(null); + + /** + * Unit test for getTableName() + */ + @Test + public void getTableNameTest() { + Assert.assertEquals("\"schema\".\"table\"", + CONNECTOR.getTableName("db", "schema", "table")); + } + + /** + * Unit tests for getTableQuery() + */ + @Test + public void getTableQueryTest() { + String tableName = CONNECTOR.getTableName("db", "schema", "table"); + + // random query + Assert.assertEquals(String.format("SELECT * FROM %s\n" + + "TABLESAMPLE BERNOULLI (100.0 * %d / (SELECT COUNT(*) FROM %s))", + tableName, 100, tableName), + CONNECTOR.getRandomQuery(tableName, 100)); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java new file mode 100644 index 000000000..4d11004e4 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java @@ -0,0 +1,155 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.util.DBUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +/** + * Unit Test class for the PostgresDBRecord + */ +@RunWith(MockitoJUnitRunner.class) +public class RedshiftDBRecordUnitTest { + + private static final int DEFAULT_PRECISION = 38; + + /** + * Validate the precision less Numbers handling against following use cases. + * 1. Ensure that the numeric type with [p,s] set as [38,4] detect as BigDecimal(38,4) in cdap. + * 2. Ensure that the numeric type without [p,s] detect as String type in cdap. + * + * @throws Exception + */ + @Test + public void validatePrecisionLessDecimalParsing() throws Exception { + Schema.Field field1 = Schema.Field.of("ID1", Schema.decimalOf(DEFAULT_PRECISION, 4)); + Schema.Field field2 = Schema.Field.of("ID2", Schema.of(Schema.Type.STRING)); + + Schema schema = Schema.recordOf( + "dbRecord", + field1, + field2 + ); + + ResultSetMetaData resultSetMetaData = Mockito.mock(ResultSetMetaData.class); + Mockito.when(resultSetMetaData.getColumnType(Mockito.eq(1))).thenReturn(Types.NUMERIC); + Mockito.when(resultSetMetaData.getPrecision(Mockito.eq(1))).thenReturn(DEFAULT_PRECISION); + Mockito.when(resultSetMetaData.getColumnType(eq(2))).thenReturn(Types.NUMERIC); + when(resultSetMetaData.getPrecision(eq(2))).thenReturn(0); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + + when(resultSet.getMetaData()).thenReturn(resultSetMetaData); + when(resultSet.getBigDecimal(eq(1))).thenReturn(BigDecimal.valueOf(123.4568)); + when(resultSet.getString(eq(2))).thenReturn("123.4568"); + + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + RedshiftDBRecord dbRecord = new RedshiftDBRecord(); + dbRecord.handleField(resultSet, builder, field1, 1, Types.NUMERIC, DEFAULT_PRECISION, 4); + dbRecord.handleField(resultSet, builder, field2, 2, Types.NUMERIC, 0, -127); + + StructuredRecord record = builder.build(); + Assert.assertTrue(record.getDecimal("ID1") instanceof BigDecimal); + Assert.assertEquals(record.getDecimal("ID1"), BigDecimal.valueOf(123.4568)); + Assert.assertTrue(record.get("ID2") instanceof String); + Assert.assertEquals(record.get("ID2"), "123.4568"); + } + + @Test + public void validateTimestampType() throws SQLException { + OffsetDateTime offsetDateTime = OffsetDateTime.of(2023, 1, 1, 1, 0, 0, 0, ZoneOffset.UTC); + ResultSetMetaData metaData = Mockito.mock(ResultSetMetaData.class); + when(metaData.getColumnTypeName(eq(0))).thenReturn("timestamp"); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + when(resultSet.getMetaData()).thenReturn(metaData); + when(resultSet.getTimestamp(eq(0), eq(DBUtils.PURE_GREGORIAN_CALENDAR))) + .thenReturn(Timestamp.from(offsetDateTime.toInstant())); + + Schema.Field field1 = Schema.Field.of("field1", Schema.of(Schema.LogicalType.DATETIME)); + Schema schema = Schema.recordOf( + "dbRecord", + field1 + ); + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + + RedshiftDBRecord dbRecord = new RedshiftDBRecord(); + dbRecord.handleField(resultSet, builder, field1, 0, Types.TIMESTAMP, 0, 0); + StructuredRecord record = builder.build(); + Assert.assertNotNull(record); + Assert.assertNotNull(record.getDateTime("field1")); + Assert.assertEquals(record.getDateTime("field1").toInstant(ZoneOffset.UTC), offsetDateTime.toInstant()); + + // Validate backward compatibility + + field1 = Schema.Field.of("field1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS)); + schema = Schema.recordOf( + "dbRecord", + field1 + ); + builder = StructuredRecord.builder(schema); + dbRecord.handleField(resultSet, builder, field1, 0, Types.TIMESTAMP, 0, 0); + record = builder.build(); + Assert.assertNotNull(record); + Assert.assertNotNull(record.getTimestamp("field1")); + Assert.assertEquals(record.getTimestamp("field1").toInstant(), offsetDateTime.toInstant()); + } + + @Test + public void validateTimestampTZType() throws SQLException { + OffsetDateTime offsetDateTime = OffsetDateTime.of(2023, 1, 1, 1, 0, 0, 0, ZoneOffset.UTC); + ResultSetMetaData metaData = Mockito.mock(ResultSetMetaData.class); + when(metaData.getColumnTypeName(eq(0))).thenReturn("timestamptz"); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + when(resultSet.getMetaData()).thenReturn(metaData); + when(resultSet.getObject(eq(0), eq(OffsetDateTime.class))).thenReturn(offsetDateTime); + + Schema.Field field1 = Schema.Field.of("field1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS)); + Schema schema = Schema.recordOf( + "dbRecord", + field1 + ); + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + + RedshiftDBRecord dbRecord = new RedshiftDBRecord(); + dbRecord.handleField(resultSet, builder, field1, 0, Types.TIMESTAMP, 0, 0); + StructuredRecord record = builder.build(); + Assert.assertNotNull(record); + Assert.assertNotNull(record.getTimestamp("field1", ZoneId.of("UTC"))); + Assert.assertEquals(record.getTimestamp("field1", ZoneId.of("UTC")).toInstant(), offsetDateTime.toInstant()); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java new file mode 100644 index 000000000..2d21c4478 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.plugin.db.connector.DBSpecificFailedConnectionTest; +import org.junit.Test; + +import java.io.IOException; + +public class RedshiftFailedConnectionTest extends DBSpecificFailedConnectionTest { + private static final String JDBC_DRIVER_CLASS_NAME = "com.amazon.redshift.Driver"; + + @Test + public void test() throws ClassNotFoundException, IOException { + + RedshiftConnector connector = new RedshiftConnector( + new RedshiftConnectorConfig("username", "password", "jdbc", "", "localhost", "db", 5432)); + + super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + + "jdbc:redshift://localhost:5432/db and arguments: " + + "{user=username}. Error: ConnectException: Connection refused " + + "(Connection refused)."); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java new file mode 100644 index 000000000..5df4fb300 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java @@ -0,0 +1,218 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.base.Charsets; +import com.google.common.base.Throwables; +import com.google.common.collect.Sets; +import io.cdap.cdap.api.artifact.ArtifactSummary; +import io.cdap.cdap.api.plugin.PluginClass; +import io.cdap.cdap.datapipeline.DataPipelineApp; +import io.cdap.cdap.proto.id.ArtifactId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.batch.DatabasePluginTestBase; +import io.cdap.plugin.db.sink.ETLDBOutputFormat; +import io.cdap.plugin.db.source.DataDrivenETLDBInputFormat; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.Date; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +/** + * Base test class for Redshift plugins. + */ +public abstract class RedshiftPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftPluginTestBase.class); + protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); + protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); + protected static final long CURRENT_TS = System.currentTimeMillis(); + + protected static final String JDBC_DRIVER_NAME = "redshift"; + protected static final Map BASE_PROPS = new HashMap<>(); + + protected static String connectionUrl; + protected static int year; + protected static final int PRECISION = 10; + protected static final int SCALE = 6; + private static int startCount; + + @BeforeClass + public static void setupTest() throws Exception { + if (startCount++ > 0) { + return; + } + + getProperties(); + + Calendar calendar = Calendar.getInstance(); + calendar.setTime(new Date(CURRENT_TS)); + year = calendar.get(Calendar.YEAR); + + setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); + + addPluginArtifact(NamespaceId.DEFAULT.artifact(JDBC_DRIVER_NAME, "1.0.0"), + DATAPIPELINE_ARTIFACT_ID, + RedshiftSource.class, DBRecord.class, + ETLDBOutputFormat.class, DataDrivenETLDBInputFormat.class, DBRecord.class); + + // add mysql 3rd party plugin + PluginClass mysqlDriver = new PluginClass(ConnectionConfig.JDBC_PLUGIN_TYPE, JDBC_DRIVER_NAME, + "redshift driver class", Driver.class.getName(), + null, Collections.emptyMap()); + addPluginArtifact(NamespaceId.DEFAULT.artifact("redshift-jdbc-connector", "1.0.0"), + DATAPIPELINE_ARTIFACT_ID, + Sets.newHashSet(mysqlDriver), Driver.class); + + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + + connectionUrl = "jdbc:redshift://" + BASE_PROPS.get(ConnectionConfig.HOST) + ":" + + BASE_PROPS.get(ConnectionConfig.PORT) + "/" + BASE_PROPS.get(ConnectionConfig.DATABASE); + Connection conn = createConnection(); + createTestTables(conn); + prepareTestData(conn); + } + + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("redshift.clusterEndpoint")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("redshift.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("redshift.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("redshift.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("redshift.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + + protected static void createTestTables(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + // create a table that the action will truncate at the end of the run + stmt.execute("CREATE TABLE \"dbActionTest\" (x int, day varchar(10))"); + // create a table that the action will truncate at the end of the run + stmt.execute("CREATE TABLE \"postActionTest\" (x int, day varchar(10))"); + + stmt.execute("CREATE TABLE my_table" + + "(" + + "\"ID\" INT NOT NULL," + + "\"NAME\" VARCHAR(40) NOT NULL," + + "\"SCORE\" REAL," + + "\"GRADUATED\" BOOLEAN," + + "\"NOT_IMPORTED\" VARCHAR(30)," + + "\"SMALLINT_COL\" SMALLINT," + + "\"BIG\" BIGINT," + + "\"NUMERIC_COL\" NUMERIC(" + PRECISION + "," + SCALE + ")," + + "\"DECIMAL_COL\" DECIMAL(" + PRECISION + "," + SCALE + ")," + + "\"DOUBLE_PREC_COL\" DOUBLE PRECISION," + + "\"DATE_COL\" DATE," + + "\"TIME_COL\" TIME," + + "\"TIMESTAMP_COL\" TIMESTAMP(3)," + + "\"TEXT_COL\" TEXT," + + "\"CHAR_COL\" CHAR(100)," + + "\"BYTEA_COL\" BYTEA" + + ")"); + stmt.execute("CREATE TABLE \"MY_DEST_TABLE\" AS " + + "SELECT * FROM my_table"); + stmt.execute("CREATE TABLE your_table AS " + + "SELECT * FROM my_table"); + } + } + + protected static void prepareTestData(Connection conn) throws SQLException { + try ( + Statement stmt = conn.createStatement(); + PreparedStatement pStmt1 = + conn.prepareStatement("INSERT INTO my_table " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?," + + " ?, ?, ?, ?, ?, ?)"); + PreparedStatement pStmt2 = + conn.prepareStatement("INSERT INTO your_table " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?," + + " ?, ?, ?, ?, ?, ?)")) { + + stmt.execute("insert into \"dbActionTest\" values (1, '1970-01-01')"); + stmt.execute("insert into \"postActionTest\" values (1, '1970-01-01')"); + + populateData(pStmt1, pStmt2); + } + } + + private static void populateData(PreparedStatement... stmts) throws SQLException { + // insert the same data into both tables: my_table and your_table + for (PreparedStatement pStmt : stmts) { + for (int i = 1; i <= 5; i++) { + String name = "user" + i; + pStmt.setInt(1, i); + pStmt.setString(2, name); + pStmt.setDouble(3, 123.45 + i); + pStmt.setBoolean(4, (i % 2 == 0)); + pStmt.setString(5, "random" + i); + pStmt.setShort(6, (short) i); + pStmt.setLong(7, (long) i); + pStmt.setBigDecimal(8, new BigDecimal("123.45").add(new BigDecimal(i))); + pStmt.setBigDecimal(9, new BigDecimal("123.45").add(new BigDecimal(i))); + pStmt.setDouble(10, 123.45 + i); + pStmt.setDate(11, new Date(CURRENT_TS)); + pStmt.setTime(12, new Time(CURRENT_TS)); + pStmt.setTimestamp(13, new Timestamp(CURRENT_TS)); + pStmt.setString(14, name); + pStmt.setString(15, "char" + i); + pStmt.setBytes(16, name.getBytes(Charsets.UTF_8)); + pStmt.executeUpdate(); + } + } + } + + public static Connection createConnection() { + try { + Class.forName(Driver.class.getCanonicalName()); + return DriverManager.getConnection(connectionUrl, BASE_PROPS.get(ConnectionConfig.USER), + BASE_PROPS.get(ConnectionConfig.PASSWORD)); + } catch (Exception e) { + throw Throwables.propagate(e); + } + } + + @AfterClass + public static void tearDownDB() { + try (Connection conn = createConnection(); + Statement stmt = conn.createStatement()) { + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); + } + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java new file mode 100644 index 000000000..95ad0938b --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.common.test.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * This is a test suite that runs all the tests for Redshift plugins. + */ +@RunWith(TestSuite.class) +@Suite.SuiteClasses({ + RedshiftSourceTestRun.class, +}) +public class RedshiftPluginTestSuite extends RedshiftPluginTestBase { +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java new file mode 100644 index 000000000..1ac41bcd0 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java @@ -0,0 +1,332 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.common.Bytes; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.dataset.table.Table; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.mock.batch.MockSink; +import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; +import io.cdap.cdap.etl.proto.v2.ETLPlugin; +import io.cdap.cdap.etl.proto.v2.ETLStage; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.test.ApplicationManager; +import io.cdap.cdap.test.DataSetManager; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.DBConfig; +import io.cdap.plugin.db.source.AbstractDBSource; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Time; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Test for Redshift source plugin. + */ +public class RedshiftSourceTestRun extends RedshiftPluginTestBase { + + @Test + @SuppressWarnings("ConstantConditions") + public void testDBMacroSupport() throws Exception { + String importQuery = "SELECT * FROM my_table WHERE \"DATE_COL\" <= '${logicalStartTime(yyyy-MM-dd,1d)}' " + + "AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(ID),MAX(ID) from my_table"; + String splitBy = "ID"; + + ImmutableMap sourceProps = ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBTestSource").build(); + + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + sourceProps + ); + + ETLPlugin sinkConfig = MockSink.getPlugin("macroOutputTable"); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBMacro"); + runETLOnce(appManager, ImmutableMap.of("logical.start.time", String.valueOf(CURRENT_TS))); + + DataSetManager outputManager = getDataset("macroOutputTable"); + Assert.assertTrue(MockSink.readOutput(outputManager).isEmpty()); + } + + @Test + @SuppressWarnings("ConstantConditions") + public void testDBSource() throws Exception { + String importQuery = "SELECT \"ID\", \"NAME\", \"SCORE\", \"GRADUATED\", \"SMALLINT_COL\", \"BIG\", " + + "\"NUMERIC_COL\", \"CHAR_COL\", \"DECIMAL_COL\", \"BYTEA_COL\", \"DATE_COL\", \"TIME_COL\", \"TIMESTAMP_COL\", " + + "\"TEXT_COL\", \"DOUBLE_PREC_COL\" FROM my_table " + + "WHERE \"ID\" < 3 AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") from my_table"; + String splitBy = "ID"; + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBSourceTest") + .build(), + null + ); + + String outputDatasetName = "output-dbsourcetest"; + ETLPlugin sinkConfig = MockSink.getPlugin(outputDatasetName); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBSource"); + runETLOnce(appManager); + + DataSetManager
outputManager = getDataset(outputDatasetName); + List outputRecords = MockSink.readOutput(outputManager); + + Assert.assertEquals(2, outputRecords.size()); + String userid = outputRecords.get(0).get("NAME"); + StructuredRecord row1 = "user1".equals(userid) ? outputRecords.get(0) : outputRecords.get(1); + StructuredRecord row2 = "user1".equals(userid) ? outputRecords.get(1) : outputRecords.get(0); + + // Verify data + Assert.assertEquals("user1", row1.get("NAME")); + Assert.assertEquals("user2", row2.get("NAME")); + Assert.assertEquals("user1", row1.get("TEXT_COL")); + Assert.assertEquals("user2", row2.get("TEXT_COL")); + Assert.assertEquals("char1", ((String) row1.get("CHAR_COL")).trim()); + Assert.assertEquals("char2", ((String) row2.get("CHAR_COL")).trim()); + Assert.assertEquals(124.45f, ((Float) row1.get("SCORE")).doubleValue(), 0.000001); + Assert.assertEquals(125.45f, ((Float) row2.get("SCORE")).doubleValue(), 0.000001); + Assert.assertEquals(false, row1.get("GRADUATED")); + Assert.assertEquals(true, row2.get("GRADUATED")); + Assert.assertNull(row1.get("NOT_IMPORTED")); + Assert.assertNull(row2.get("NOT_IMPORTED")); + + Assert.assertEquals(1, (int) row1.get("SMALLINT_COL")); + Assert.assertEquals(2, (int) row2.get("SMALLINT_COL")); + Assert.assertEquals(1, (long) row1.get("BIG")); + Assert.assertEquals(2, (long) row2.get("BIG")); + + Assert.assertEquals(new BigDecimal("124.45", new MathContext(PRECISION)).setScale(SCALE), + row1.getDecimal("NUMERIC_COL")); + Assert.assertEquals(new BigDecimal("125.45", new MathContext(PRECISION)).setScale(SCALE), + row2.getDecimal("NUMERIC_COL")); + Assert.assertEquals(new BigDecimal("124.45", new MathContext(PRECISION)).setScale(SCALE), + row1.getDecimal("DECIMAL_COL")); + + Assert.assertEquals(124.45, (double) row1.get("DOUBLE_PREC_COL"), 0.000001); + Assert.assertEquals(125.45, (double) row2.get("DOUBLE_PREC_COL"), 0.000001); + // Verify time columns + java.util.Date date = new java.util.Date(CURRENT_TS); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + LocalDate expectedDate = Date.valueOf(sdf.format(date)).toLocalDate(); + sdf = new SimpleDateFormat("H:mm:ss"); + LocalTime expectedTime = Time.valueOf(sdf.format(date)).toLocalTime(); + ZonedDateTime expectedTs = date.toInstant().atZone(ZoneId.ofOffset("UTC", ZoneOffset.UTC)); + Assert.assertEquals(expectedDate, row1.getDate("DATE_COL")); + Assert.assertEquals(expectedTime, row1.getTime("TIME_COL")); + Assert.assertEquals(expectedTs, row1.getTimestamp("TIMESTAMP_COL", ZoneId.ofOffset("UTC", ZoneOffset.UTC))); + + // verify binary columns + Assert.assertEquals("user1", Bytes.toString(((ByteBuffer) row1.get("BYTEA_COL")).array(), 0, 5)); + Assert.assertEquals("user2", Bytes.toString(((ByteBuffer) row2.get("BYTEA_COL")).array(), 0, 5)); + } + + @Test + public void testDbSourceMultipleTables() throws Exception { + String importQuery = "SELECT \"my_table\".\"ID\", \"your_table\".\"NAME\" FROM \"my_table\", \"your_table\"" + + "WHERE \"my_table\".\"ID\" < 3 and \"my_table\".\"ID\" = \"your_table\".\"ID\" and $CONDITIONS"; + String boundingQuery = "SELECT MIN(MIN(\"my_table\".\"ID\"), MIN(\"your_table\".\"ID\")), " + + "MAX(MAX(\"my_table\".\"ID\"), MAX(\"your_table\".\"ID\"))"; + String splitBy = "\"my_table\".\"ID\""; + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBMultipleTest") + .build(), + null + ); + + String outputDatasetName = "output-multitabletest"; + ETLPlugin sinkConfig = MockSink.getPlugin(outputDatasetName); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBSourceWithMultipleTables"); + runETLOnce(appManager); + + // records should be written + DataSetManager
outputManager = getDataset(outputDatasetName); + List outputRecords = MockSink.readOutput(outputManager); + Assert.assertEquals(2, outputRecords.size()); + String userid = outputRecords.get(0).get("NAME"); + StructuredRecord row1 = "user1".equals(userid) ? outputRecords.get(0) : outputRecords.get(1); + StructuredRecord row2 = "user1".equals(userid) ? outputRecords.get(1) : outputRecords.get(0); + // Verify data + Assert.assertEquals("user1", row1.get("NAME")); + Assert.assertEquals("user2", row2.get("NAME")); + Assert.assertEquals(1, row1.get("ID").intValue()); + Assert.assertEquals(2, row2.get("ID").intValue()); + } + + @Test + public void testUserNamePasswordCombinations() throws Exception { + String importQuery = "SELECT * FROM \"my_table\" WHERE $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") from \"my_table\""; + String splitBy = "\"ID\""; + + ETLPlugin sinkConfig = MockSink.getPlugin("outputTable"); + + Map baseSourceProps = ImmutableMap.builder() + .put(ConnectionConfig.HOST, BASE_PROPS.get(ConnectionConfig.HOST)) + .put(ConnectionConfig.PORT, BASE_PROPS.get(ConnectionConfig.PORT)) + .put(ConnectionConfig.DATABASE, BASE_PROPS.get(ConnectionConfig.DATABASE)) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "UserPassDBTest") + .build(); + + ApplicationId appId = NamespaceId.DEFAULT.app("dbTest"); + + // null user name, null password. Should succeed. + // as source + ETLPlugin dbConfig = new ETLPlugin(RedshiftConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, + baseSourceProps, null); + ETLStage table = new ETLStage("uniqueTableSink", sinkConfig); + ETLStage database = new ETLStage("databaseSource", dbConfig); + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + AppRequest appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); + deployApplication(appId, appRequest); + + // null user name, non-null password. Should fail. + // as source + Map noUser = new HashMap<>(baseSourceProps); + noUser.put(DBConfig.PASSWORD, "password"); + database = new ETLStage("databaseSource", new ETLPlugin(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, noUser, null)); + etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + assertDeploymentFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT, + "Deploying DB Source with null username but non-null password should have failed."); + + // non-null username, non-null, but empty password. Should succeed. + // as source + Map emptyPassword = new HashMap<>(baseSourceProps); + emptyPassword.put(DBConfig.USER, "root"); + emptyPassword.put(DBConfig.PASSWORD, ""); + database = new ETLStage("databaseSource", new ETLPlugin(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, emptyPassword, null)); + etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); + deployApplication(appId, appRequest); + } + + @Test + public void testNonExistentDBTable() throws Exception { + // source + String importQuery = "SELECT \"ID\", \"NAME\" FROM \"dummy\" WHERE ID < 3 AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") FROM \"dummy\""; + String splitBy = "\"ID\""; + ETLPlugin sinkConfig = MockSink.getPlugin("table"); + ETLPlugin sourceBadNameConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBNonExistentTest") + .build(), + null); + ETLStage sink = new ETLStage("sink", sinkConfig); + ETLStage sourceBadName = new ETLStage("sourceBadName", sourceBadNameConfig); + + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(sourceBadName) + .addStage(sink) + .addConnection(sourceBadName.getName(), sink.getName()) + .build(); + ApplicationId appId = NamespaceId.DEFAULT.app("dbSourceNonExistingTest"); + assertDeployAppFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT); + + // Bad connection + ETLPlugin sourceBadConnConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.HOST, BASE_PROPS.get(ConnectionConfig.HOST)) + .put(ConnectionConfig.PORT, BASE_PROPS.get(ConnectionConfig.PORT)) + .put(ConnectionConfig.DATABASE, "dumDB") + .put(ConnectionConfig.USER, BASE_PROPS.get(ConnectionConfig.USER)) + .put(ConnectionConfig.PASSWORD, BASE_PROPS.get(ConnectionConfig.PASSWORD)) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "RedshiftTest") + .build(), + null); + ETLStage sourceBadConn = new ETLStage("sourceBadConn", sourceBadConnConfig); + etlConfig = ETLBatchConfig.builder() + .addStage(sourceBadConn) + .addStage(sink) + .addConnection(sourceBadConn.getName(), sink.getName()) + .build(); + assertDeployAppFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT); + } +} diff --git a/amazon-redshift-plugin/widgets/Redshift-batchsource.json b/amazon-redshift-plugin/widgets/Redshift-batchsource.json new file mode 100644 index 000000000..91e860ee9 --- /dev/null +++ b/amazon-redshift-plugin/widgets/Redshift-batchsource.json @@ -0,0 +1,240 @@ +{ + "metadata": { + "spec-version": "1.5" + }, + "display-name": "Redshift", + "configuration-groups": [ + { + "label": "Connection", + "properties": [ + { + "widget-type": "toggle", + "label": "Use connection", + "name": "useConnection", + "widget-attributes": { + "on": { + "value": "true", + "label": "YES" + }, + "off": { + "value": "false", + "label": "NO" + }, + "default": "false" + } + }, + { + "widget-type": "connection-select", + "label": "Connection", + "name": "connection", + "widget-attributes": { + "connectionType": "Redshift" + } + }, + { + "widget-type": "plugin-list", + "label": "JDBC Driver name", + "name": "jdbcPluginName", + "widget-attributes": { + "plugin-type": "jdbc" + } + }, + { + "widget-type": "textbox", + "label": "Host", + "name": "host", + "widget-attributes": { + "placeholder": "Redshift endpoint host name." + } + }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5439" + } + }, + { + "widget-type": "textbox", + "label": "Username", + "name": "user" + }, + { + "widget-type": "password", + "label": "Password", + "name": "password" + }, + { + "widget-type": "keyvalue", + "label": "Connection Arguments", + "name": "connectionArguments", + "widget-attributes": { + "showDelimiter": "false", + "key-placeholder": "Key", + "value-placeholder": "Value", + "kv-delimiter" : "=", + "delimiter" : ";" + } + } + ] + }, + { + "label": "Basic", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName", + "widget-attributes": { + "placeholder": "Name used to identify this source for lineage. Typically, the name of the table/view." + } + }, + { + "widget-type": "textbox", + "label": "Database", + "name": "database" + }, + { + "widget-type": "connection-browser", + "widget-category": "plugin", + "widget-attributes": { + "connectionType": "Redshift", + "label": "Browse Database" + } + } + ] + }, + { + "label": "SQL Query", + "properties": [ + { + "widget-type": "textarea", + "label": "Import Query", + "name": "importQuery", + "widget-attributes": { + "rows": "4" + } + }, + { + "widget-type": "get-schema", + "widget-category": "plugin" + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "textarea", + "label": "Bounding Query", + "name": "boundingQuery", + "widget-attributes": { + "rows": "4" + } + }, + { + "widget-type": "textbox", + "label": "Split Column", + "name": "splitBy" + }, + { + "widget-type": "textbox", + "label": "Number of Splits", + "name": "numSplits", + "widget-attributes": { + "default": "1" + } + }, + { + "widget-type": "number", + "label": "Fetch Size", + "name": "fetchSize", + "widget-attributes": { + "default": "1000", + "minimum": "0" + } + } + ] + } + ], + "outputs": [ + { + "name": "schema", + "widget-type": "schema", + "widget-attributes": { + "schema-types": [ + "boolean", + "int", + "long", + "float", + "double", + "bytes", + "string" + ], + "schema-default-type": "string" + } + } + ], + "filters": [ + { + "name": "showConnectionProperties ", + "condition": { + "expression": "useConnection == false" + }, + "show": [ + { + "type": "property", + "name": "jdbcPluginName" + }, + { + "type": "property", + "name": "instanceType" + }, + { + "type": "property", + "name": "host" + }, + { + "type": "property", + "name": "port" + }, + { + "type": "property", + "name": "user" + }, + { + "type": "property", + "name": "password" + }, + { + "type": "property", + "name": "database" + }, + { + "type": "property", + "name": "connectionArguments" + } + ] + }, + { + "name": "showConnectionId", + "condition": { + "expression": "useConnection == true" + }, + "show": [ + { + "type": "property", + "name": "connection" + } + ] + }, + ], + "jump-config": { + "datasets": [ + { + "ref-property-name": "referenceName" + } + ] + } +} diff --git a/amazon-redshift-plugin/widgets/Redshift-connector.json b/amazon-redshift-plugin/widgets/Redshift-connector.json new file mode 100644 index 000000000..3a2af8e01 --- /dev/null +++ b/amazon-redshift-plugin/widgets/Redshift-connector.json @@ -0,0 +1,75 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Redshift", + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "widget-type": "plugin-list", + "label": "JDBC Driver name", + "name": "jdbcPluginName", + "widget-attributes": { + "plugin-type": "jdbc" + } + }, + { + "widget-type": "textbox", + "label": "Host", + "name": "host", + "widget-attributes": { + "default": "localhost" + } + }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5439" + } + }, + { + "widget-type": "textbox", + "label": "Database", + "name": "database" + } + ] + }, + { + "label": "Credentials", + "properties": [ + { + "widget-type": "textbox", + "label": "Username", + "name": "user" + }, + { + "widget-type": "password", + "label": "Password", + "name": "password" + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "keyvalue", + "label": "Connection Arguments", + "name": "connectionArguments", + "widget-attributes": { + "showDelimiter": "false", + "key-placeholder": "Key", + "value-placeholder": "Value", + "kv-delimiter": "=", + "delimiter": ";" + } + } + ] + } + ], + "outputs": [] +} diff --git a/pom.xml b/pom.xml index 74932ee90..847e157e9 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,7 @@ cloudsql-postgresql-plugin teradata-plugin generic-db-argument-setter + amazon-redshift-plugin From 4d8bc975215979ca887fc4be88acfa52de79ae9c Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Tue, 5 Dec 2023 17:00:10 +0530 Subject: [PATCH 04/62] Fix for connection arguments not getting updated. --- .../mysql/CloudSQLMySQLConnectorConfig.java | 6 ++--- .../plugin/mysql/MysqlConnectorConfig.java | 6 ++--- .../mysql/MysqlFailedConnectionTest.java | 26 +++++++++++++++---- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java index 1e89d5a95..e763f6235 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java @@ -105,14 +105,14 @@ public String getConnectionString() { @Override public Properties getConnectionArgumentsProperties() { Properties properties = super.getConnectionArgumentsProperties(); - properties.put(JDBC_PROPERTY_CONNECT_TIMEOUT_MILLIS, "20000"); - properties.put(JDBC_PROPERTY_SOCKET_TIMEOUT_MILLIS, "20000"); + properties.putIfAbsent(JDBC_PROPERTY_CONNECT_TIMEOUT_MILLIS, "20000"); + properties.putIfAbsent(JDBC_PROPERTY_SOCKET_TIMEOUT_MILLIS, "20000"); return properties; } @Override public boolean canConnect() { return super.canConnect() && !containsMacro(CloudSQLUtil.CONNECTION_NAME) && - !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); + !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); } } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java index 9b481e4fe..8c20798d3 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java @@ -57,9 +57,9 @@ public int getDefaultPort() { public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); // the unit below is milli-second - prop.put(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); - prop.put(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); - prop.put(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); + prop.putIfAbsent(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); + prop.putIfAbsent(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); + prop.putIfAbsent(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); // MySQL property to ensure that TINYINT(1) type data is not converted to MySQL Bit/Boolean type in the ResultSet. prop.putIfAbsent(MYSQL_TINYINT1_IS_BIT, "false"); return prop; diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java index a1be6a754..5c4f35828 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java @@ -31,10 +31,26 @@ public void test() throws ClassNotFoundException, IOException { new MysqlConnectorConfig("localhost", 3306, "username", "password", "jdbc", "")); super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + - "jdbc:mysql://localhost:3306 and arguments: {user=username, " + - "rewriteBatchedStatements=true, " + - "connectTimeout=20000, tinyInt1isBit=false, " + - "socketTimeout=20000}. Error: " + - "ConnectException: Connection refused (Connection refused)."); + "jdbc:mysql://localhost:3306 and arguments: {user=username, " + + "rewriteBatchedStatements=true, " + + "connectTimeout=20000, tinyInt1isBit=false, " + + "socketTimeout=20000}. Error: " + + "ConnectException: Connection refused (Connection refused)."); } + + @Test + public void testWithUpdatedConnectionArguments() throws ClassNotFoundException, IOException { + + MysqlConnector connector = new MysqlConnector( + new MysqlConnectorConfig("localhost", 3306, "username", "password", "jdbc", + "connectTimeout=30000;socketTimeout=30000")); + + super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + + "jdbc:mysql://localhost:3306 and arguments: {user=username, " + + "rewriteBatchedStatements=true, " + + "connectTimeout=30000, tinyInt1isBit=false, " + + "socketTimeout=30000}. Error: " + + "ConnectException: Connection refused (Connection refused)."); + } + } From e0756904fc0d4c7f6effde309080fdde85414509 Mon Sep 17 00:00:00 2001 From: Shubhangi-cs Date: Wed, 6 Dec 2023 12:24:34 +0530 Subject: [PATCH 05/62] Redshift Junits Redshift Junits --- .../redshift/RedshiftConnectorConfig.java | 12 +- .../amazon/redshift/RedshiftSource.java | 8 ++ .../redshift/RedshiftConnectorUnitTest.java | 13 ++ .../redshift/RedshiftSchemaReaderTest.java | 131 ++++++++++++++++++ .../amazon/redshift/RedshiftSourceTest.java | 98 +++++++++++++ .../widgets/Redshift-batchsource.json | 2 +- 6 files changed, 257 insertions(+), 7 deletions(-) create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReaderTest.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTest.java diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java index f05f26d10..bae0013b3 100644 --- a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java @@ -72,16 +72,16 @@ public int getPort() { @Override public String getConnectionString() { - return String.format( - RedshiftConstants.REDSHIFT_CONNECTION_STRING_FORMAT, - host, - getPort(), - database); + return String.format( + RedshiftConstants.REDSHIFT_CONNECTION_STRING_FORMAT, + host, + getPort(), + database); } @Override public boolean canConnect() { return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && - !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); + !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); } } diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java index 1b5894de9..6a0df3a2d 100644 --- a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java @@ -16,6 +16,7 @@ package io.cdap.plugin.amazon.redshift; +import com.google.common.annotations.VisibleForTesting; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Metadata; @@ -108,6 +109,13 @@ public Map getDBSpecificArguments() { return Collections.emptyMap(); } + @VisibleForTesting + public RedshiftSourceConfig(@Nullable Boolean useConnection, + @Nullable RedshiftConnectorConfig connection) { + this.useConnection = useConnection; + this.connection = connection; + } + @Override public Integer getFetchSize() { Integer fetchSize = super.getFetchSize(); diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java index 39579cb60..47e8b0a52 100644 --- a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java @@ -39,6 +39,19 @@ public void getTableNameTest() { CONNECTOR.getTableName("db", "schema", "table")); } + @Test + public void getRandomQuery() { + Assert.assertEquals("SELECT * FROM TestData\n" + + "TABLESAMPLE BERNOULLI (100.0 * 10 / (SELECT COUNT(*) FROM TestData))", + CONNECTOR.getRandomQuery("TestData", 10)); + } + + @Test + public void getDBRecordType() { + Assert.assertEquals("class io.cdap.plugin.amazon.redshift.RedshiftDBRecord", + CONNECTOR.getDBRecordType().toString()); + } + /** * Unit tests for getTableQuery() */ diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReaderTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReaderTest.java new file mode 100644 index 000000000..206b4ae9f --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReaderTest.java @@ -0,0 +1,131 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.collect.Lists; +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class RedshiftSchemaReaderTest { + + @Test + public void testGetSchema() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader(); + + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(metadata.getColumnTypeName(1)).thenReturn("timetz"); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.TIMESTAMP); + + Schema schema = schemaReader.getSchema(metadata, 1); + + Assert.assertEquals(Schema.of(Schema.Type.STRING), schema); + } + + @Test + public void testGetSchemaWithIntType() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader(); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(metadata.getColumnTypeName(1)).thenReturn("INT"); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.NUMERIC); + Schema schema = schemaReader.getSchema(metadata, 1); + + Assert.assertEquals(Schema.of(Schema.Type.INT), schema); + } + + @Test + public void testGetSchemaWithNumericTypeWithPrecision() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader(); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(metadata.getColumnTypeName(1)).thenReturn("STRING"); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.NUMERIC); + Mockito.when(metadata.getPrecision(1)).thenReturn(0); + + Schema schema = schemaReader.getSchema(metadata, 1); + + Assert.assertEquals(Schema.of(Schema.Type.STRING), schema); + } + + @Test + public void testGetSchemaWithOtherTypes() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader(); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(metadata.getColumnTypeName(1)).thenReturn("BIGINT"); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.BIGINT); + Schema schema = schemaReader.getSchema(metadata, 1); + + Assert.assertEquals(Schema.of(Schema.Type.LONG), schema); + + Mockito.when(metadata.getColumnTypeName(2)).thenReturn("timestamp"); + Mockito.when(metadata.getColumnType(2)).thenReturn(Types.TIMESTAMP); + + schema = schemaReader.getSchema(metadata, 2); + + Assert.assertEquals(Schema.of(Schema.LogicalType.DATETIME), schema); + } + + @Test + public void testShouldIgnoreColumn() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader("sessionID"); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(metadata.getColumnName(1)).thenReturn("c_sessionID"); + Assert.assertTrue(schemaReader.shouldIgnoreColumn(metadata, 1)); + Mockito.when(metadata.getColumnName(2)).thenReturn("sqn_sessionID"); + Assert.assertTrue(schemaReader.shouldIgnoreColumn(metadata, 2)); + Mockito.when(metadata.getColumnName(3)).thenReturn("columnName"); + Assert.assertFalse(schemaReader.shouldIgnoreColumn(metadata, 3)); + } + + @Test + public void testGetSchemaFields() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader(); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + // Mock two columns with different types + Mockito.when(metadata.getColumnCount()).thenReturn(2); + Mockito.when(metadata.getColumnTypeName(1)).thenReturn("INT"); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.NUMERIC); + Mockito.when(metadata.getColumnName(1)).thenReturn("column1"); + + Mockito.when(metadata.getColumnTypeName(2)).thenReturn("BIGINT"); + Mockito.when(metadata.getColumnType(2)).thenReturn(Types.BIGINT); + Mockito.when(metadata.getColumnName(2)).thenReturn("column2"); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("column1", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + expectedSchemaFields.add(Schema.Field.of("column2", Schema.nullableOf(Schema.of(Schema.Type.LONG)))); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTest.java new file mode 100644 index 000000000..d09de8f0d --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTest.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class RedshiftSourceTest { + + @Test + public void testGetDBSpecificArguments() { + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "host", "database", 1101); + RedshiftSource.RedshiftSourceConfig config = new RedshiftSource.RedshiftSourceConfig(false, connectorConfig); + Map dbSpecificArguments = config.getDBSpecificArguments(); + Assert.assertEquals(0, dbSpecificArguments.size()); + } + + @Test + public void testGetFetchSize() { + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "host", "database", 1101); + RedshiftSource.RedshiftSourceConfig config = new RedshiftSource.RedshiftSourceConfig(false, connectorConfig); + Integer fetchSize = config.getFetchSize(); + Assert.assertEquals(1000, fetchSize.intValue()); + } + + @Test + public void testGetSchemaReader() { + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "host", "database", 1101); + RedshiftSource source = new RedshiftSource(new RedshiftSource.RedshiftSourceConfig(false, connectorConfig)); + SchemaReader schemaReader = source.getSchemaReader(); + Assert.assertTrue(schemaReader instanceof RedshiftSchemaReader); + } + + @Test + public void testGetDBRecordType() { + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "host", "database", 1101); + RedshiftSource source = new RedshiftSource(new RedshiftSource.RedshiftSourceConfig(false, connectorConfig)); + Class dbRecordType = source.getDBRecordType(); + Assert.assertEquals(RedshiftDBRecord.class, dbRecordType); + } + + @Test + public void testCreateConnectionString() { + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "localhost", "test", 5439); + RedshiftSource.RedshiftSourceConfig config = new RedshiftSource.RedshiftSourceConfig(false, connectorConfig); + + RedshiftSource source = new RedshiftSource(config); + String connectionString = source.createConnectionString(); + Assert.assertEquals("jdbc:redshift://localhost:5439/test", connectionString); + } + + @Test + public void testGetLineageRecorder() { + BatchSourceContext context = Mockito.mock(BatchSourceContext.class); + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "host", "database", 1101); + RedshiftSource.RedshiftSourceConfig config = new RedshiftSource.RedshiftSourceConfig(false, connectorConfig); + RedshiftSource source = new RedshiftSource(config); + + LineageRecorder lineageRecorder = source.getLineageRecorder(context); + Assert.assertNotNull(lineageRecorder); + } +} diff --git a/amazon-redshift-plugin/widgets/Redshift-batchsource.json b/amazon-redshift-plugin/widgets/Redshift-batchsource.json index 91e860ee9..943e2d24e 100644 --- a/amazon-redshift-plugin/widgets/Redshift-batchsource.json +++ b/amazon-redshift-plugin/widgets/Redshift-batchsource.json @@ -135,7 +135,7 @@ }, { "widget-type": "textbox", - "label": "Split Column", + "label": "Split-By Field Name", "name": "splitBy" }, { From a0ad9bfcf62b7a026d1daa59a6779e5fc5db394a Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Wed, 3 Jan 2024 08:58:04 +0000 Subject: [PATCH 06/62] ConnectionString calculation refactoring (#476) --- .../io/cdap/plugin/oracle/OracleAction.java | 9 +------ .../cdap/plugin/oracle/OracleConnector.java | 11 ++------ .../plugin/oracle/OracleConnectorConfig.java | 8 +----- .../cdap/plugin/oracle/OracleConstants.java | 26 +++++++++++++++++++ .../cdap/plugin/oracle/OraclePostAction.java | 8 +----- .../io/cdap/plugin/oracle/OracleSource.java | 11 ++------ 6 files changed, 33 insertions(+), 40 deletions(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java index 81604858a..9b5331d11 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java @@ -57,16 +57,9 @@ public static class OracleActionConfig extends DBSpecificQueryConfig { @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(this.connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(this.connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, host, port, database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, host, port, database); - } + return OracleConstants.getConnectionString(this.connectionType, host, port, database); } - @Override protected Map getDBSpecificArguments() { return ImmutableMap.of(OracleConstants.DEFAULT_BATCH_VALUE, String.valueOf(defaultBatchValue)); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index fde72c8ad..bc7907b26 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -125,15 +125,8 @@ protected String getConnectionString(@Nullable String database) { if (database == null) { return config.getConnectionString(); } - if (OracleConstants.TNS_CONNECTION_TYPE.equals(config.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(config.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, config.getHost(), - config.getPort(), database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, - config.getHost(), config.getPort(), database); - } + return OracleConstants.getConnectionString(config.getConnectionType(), + config.getHost(), config.getPort(), database); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 73b005243..716cbd310 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -63,13 +63,7 @@ public OracleConnectorConfig(String host, int port, String user, String password @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, host, getPort(), database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, host, getPort(), database); - } + return OracleConstants.getConnectionString(database, host, getPort(), database); } @Name(OracleConstants.CONNECTION_TYPE) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index 040780a89..ec44d7b94 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -16,6 +16,8 @@ package io.cdap.plugin.oracle; +import javax.annotation.Nullable; + /** * Oracle Constants. */ @@ -36,4 +38,28 @@ private OracleConstants() { public static final String NAME_DATABASE = "database"; public static final String TNS_CONNECTION_TYPE = "TNS"; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; + + /** + * Returns the Connection String for the given ConnectionType. + * + * @param connectionType TNS/Service/SID + * @param host Host name of the oracle server + * @param port Port of the oracle server + * @param database Database to connect to + * @return Connection String based on the given ConnectionType + */ + public static String getConnectionString(String connectionType, + @Nullable String host, + @Nullable int port, + String database) { + if (OracleConstants.TNS_CONNECTION_TYPE.equalsIgnoreCase(connectionType)) { + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); + } + if (OracleConstants.SERVICE_CONNECTION_TYPE.equalsIgnoreCase(connectionType)) { + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, + host, port, database); + } + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, + host, port, database); + } } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java index f8ebd9ac2..4862aebfa 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java @@ -57,13 +57,7 @@ public static class OracleQueryActionConfig extends DBSpecificQueryActionConfig @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(this.connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(this.connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, host, port, database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, host, port, database); - } + return OracleConstants.getConnectionString(this.connectionType, host, port, database); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 9a554a4a4..eca7e2532 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -131,15 +131,8 @@ public OracleSourceConfig(String host, int port, String user, String password, S @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(connection.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, connection.getDatabase()); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(connection.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, connection.getHost(), - connection.getPort(), connection.getDatabase()); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, - connection.getHost(), connection.getPort(), connection.getDatabase()); - } + return OracleConstants.getConnectionString(connection.getConnectionType(), connection.getHost(), + connection.getPort(), connection.getDatabase()); } @Override From e2b5b4d2d2c58794fe4a5d0d5b73292fee10305b Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Thu, 4 Jan 2024 04:38:05 +0000 Subject: [PATCH 07/62] Fixed Typo in connection function call. (#477) --- .../main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 716cbd310..a60476bd5 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -63,7 +63,7 @@ public OracleConnectorConfig(String host, int port, String user, String password @Override public String getConnectionString() { - return OracleConstants.getConnectionString(database, host, getPort(), database); + return OracleConstants.getConnectionString(connectionType, host, getPort(), database); } @Name(OracleConstants.CONNECTION_TYPE) From b6b35aa815b8f8448d4ef29b134b64a315f3fe48 Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Thu, 4 Jan 2024 11:57:43 +0000 Subject: [PATCH 08/62] Add cucumber report url to the workflow --- .github/workflows/e2e.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1ced579cb..a49e68043 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -156,13 +156,6 @@ jobs: CLOUDSQL_MYSQL_PASSWORD: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_PASSWORD }} CLOUDSQL_MYSQL_CONNECTION_NAME: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_CONNECTION_NAME }} - - name: Upload report - uses: actions/upload-artifact@v3 - if: always() - with: - name: Cucumber report - ${{ matrix.module }} - path: ./**/target/cucumber-reports - - name: Upload debug files uses: actions/upload-artifact@v3 if: always() @@ -171,9 +164,13 @@ jobs: path: ./**/target/e2e-debug - name: Upload files to GCS - uses: google-github-actions/upload-cloud-storage@v0 + uses: google-github-actions/upload-cloud-storage@v2 if: always() with: path: ./plugin destination: e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }} glob: '**/target/cucumber-reports/**' + + - name: Cucumber Report URL + if: always() + run: echo "https://storage.googleapis.com/e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }}/plugin/${{ matrix.module }}/target/cucumber-reports/advanced-reports/cucumber-html-reports/overview-features.html" From 84b1f9370248a554780a4b7c200479aa095fa1cd Mon Sep 17 00:00:00 2001 From: "danish.amjad" Date: Fri, 15 Mar 2024 23:09:25 +0100 Subject: [PATCH 09/62] Fix non-nullable datetime when zeroDateTimeBehavior is CONVERT_TO_NULL. --- .../mysql/CloudSQLMySQLConnector.java | 3 +- .../cloudsql/mysql/CloudSQLMySQLSource.java | 7 +++ .../io/cdap/plugin/mysql/MysqlConnector.java | 3 +- .../io/cdap/plugin/mysql/MysqlConstants.java | 1 + .../cdap/plugin/mysql/MysqlSchemaReader.java | 34 ++++++++++ .../io/cdap/plugin/mysql/MysqlSource.java | 2 +- .../java/io/cdap/plugin/mysql/MysqlUtil.java | 17 +++++ .../mysql/MysqlSchemaReaderUnitTest.java | 33 ++++++++++ .../cdap/plugin/mysql/MysqlUtilUnitTest.java | 62 +++++++++++++++++++ 9 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java index a5ee68787..b4b87c81b 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java @@ -16,6 +16,7 @@ package io.cdap.plugin.cloudsql.mysql; +import com.google.common.collect.Maps; import io.cdap.cdap.api.annotation.Category; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; @@ -75,7 +76,7 @@ public StructuredRecord transform(LongWritable longWritable, MysqlDBRecord mysql @Override protected SchemaReader getSchemaReader(String sessionID) { - return new MysqlSchemaReader(sessionID); + return new MysqlSchemaReader(sessionID, Maps.fromProperties(config.getConnectionArgumentsProperties())); } @Override diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index b8b6fbf27..b0bea9e7a 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -31,9 +31,11 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.mysql.MysqlDBRecord; +import io.cdap.plugin.mysql.MysqlSchemaReader; import io.cdap.plugin.util.CloudSQLUtil; import io.cdap.plugin.util.DBUtils; import org.apache.hadoop.mapreduce.lib.db.DBWritable; @@ -120,6 +122,11 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { return new LineageRecorder(context, assetBuilder.build()); } + @Override + protected SchemaReader getSchemaReader() { + return new MysqlSchemaReader(null, cloudsqlMysqlSourceConfig.getConnectionArguments()); + } + /** CloudSQL MySQL source config. */ public static class CloudSQLMySQLSourceConfig extends AbstractDBSpecificSourceConfig { diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java index 3dede5d49..e7e935135 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java @@ -16,6 +16,7 @@ package io.cdap.plugin.mysql; +import com.google.common.collect.Maps; import io.cdap.cdap.api.annotation.Category; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; @@ -62,7 +63,7 @@ public boolean supportSchema() { @Override protected SchemaReader getSchemaReader(String sessionID) { - return new MysqlSchemaReader(sessionID); + return new MysqlSchemaReader(sessionID, Maps.fromProperties(config.getConnectionArgumentsProperties())); } @Override diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java index 39c0b8d08..54593f580 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java @@ -39,6 +39,7 @@ private MysqlConstants() { public static final String TRUST_CERT_KEYSTORE_PASSWORD = "trustCertificateKeyStorePassword"; public static final String MYSQL_CONNECTION_STRING_FORMAT = "jdbc:mysql://%s:%s/%s"; public static final String USE_CURSOR_FETCH = "useCursorFetch"; + public static final String ZERO_DATE_TIME_BEHAVIOR = "zeroDateTimeBehavior"; /** * Query to set SQL_MODE system variable. diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java index a842ba568..50907c063 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java @@ -16,12 +16,16 @@ package io.cdap.plugin.mysql; +import com.google.common.collect.Lists; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.CommonSchemaReader; +import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; +import java.util.List; +import java.util.Map; /** * Schema reader for mapping Mysql DB type @@ -31,12 +35,42 @@ public class MysqlSchemaReader extends CommonSchemaReader { public static final String YEAR_TYPE_NAME = "YEAR"; public static final String MEDIUMINT_UNSIGNED_TYPE_NAME = "MEDIUMINT UNSIGNED"; private final String sessionID; + private boolean zeroDateTimeToNull; public MysqlSchemaReader(String sessionID) { super(); this.sessionID = sessionID; } + public MysqlSchemaReader(String sessionID, Map connectionArguments) { + super(); + this.sessionID = sessionID; + this.zeroDateTimeToNull = MysqlUtil.isZeroDateTimeToNull(connectionArguments); + } + + @Override + public List getSchemaFields(ResultSet resultSet) throws SQLException { + List schemaFields = Lists.newArrayList(); + ResultSetMetaData metadata = resultSet.getMetaData(); + // ResultSetMetadata columns are numbered starting with 1 + for (int i = 1; i <= metadata.getColumnCount(); i++) { + if (shouldIgnoreColumn(metadata, i)) { + continue; + } + + String columnName = metadata.getColumnName(i); + Schema columnSchema = getSchema(metadata, i); + + if (ResultSetMetaData.columnNullable == metadata.isNullable(i) + || (zeroDateTimeToNull && MysqlUtil.isDateTimeLikeType(metadata.getColumnType(i)))) { + columnSchema = Schema.nullableOf(columnSchema); + } + Schema.Field field = Schema.Field.of(columnName, columnSchema); + schemaFields.add(field); + } + return schemaFields; + } + @Override public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { return metadata.getColumnName(index).equals("c_" + sessionID) || diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index 71f113436..a91139196 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -81,7 +81,7 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { @Override protected SchemaReader getSchemaReader() { - return new MysqlSchemaReader(null); + return new MysqlSchemaReader(null, mysqlSourceConfig.getConnectionArguments()); } /** diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java index c1c770c06..abb4aa27b 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap; +import java.sql.Types; import java.util.Map; /** @@ -91,4 +92,20 @@ public static Map composeDbSpecificArgumentsMap(Boolean autoReco public static String getConnectionString(String host, Integer port, String database) { return String.format(MysqlConstants.MYSQL_CONNECTION_STRING_FORMAT, host, port, database); } + + public static boolean isDateTimeLikeType(int columnType) { + int[] dateTimeLikeTypes = new int[]{Types.TIMESTAMP, Types.TIMESTAMP_WITH_TIMEZONE, Types.DATE}; + + for (int dttType : dateTimeLikeTypes) { + if (dttType == columnType) { + return true; + } + } + return false; + } + + public static boolean isZeroDateTimeToNull(Map connectionArguments) { + String argValue = connectionArguments.getOrDefault(MysqlConstants.ZERO_DATE_TIME_BEHAVIOR, ""); + return argValue.equals("CONVERT_TO_NULL") || argValue.equals("convertToNull"); + } } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java index 28582bc3b..fa7029c8f 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java @@ -21,9 +21,13 @@ import org.junit.Test; import org.mockito.Mockito; +import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class MysqlSchemaReaderUnitTest { @@ -37,4 +41,33 @@ public void validateYearTypeToStringTypeConversion() throws SQLException { Schema schema = schemaReader.getSchema(metadata, 1); Assert.assertTrue(Schema.of(Schema.Type.INT).equals(schema)); } + + @Test + public void validateZeroDateTimeBehavior() throws SQLException { + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(1); + Mockito.when(metadata.getColumnName(Mockito.eq(1))).thenReturn("some_date"); + + Mockito.when(metadata.getColumnType(Mockito.eq(1))).thenReturn(Types.DATE); + Mockito.when(metadata.getColumnTypeName(Mockito.eq(1))).thenReturn(MysqlSchemaReader.YEAR_TYPE_NAME); + + // non-nullable column + Mockito.when(metadata.isNullable(Mockito.eq(1))).thenReturn(0); + + // test that non-nullable date remains non-nullable when no conn arg is present + MysqlSchemaReader schemaReader = new MysqlSchemaReader(null); + List schemaFields = schemaReader.getSchemaFields(resultSet); + Assert.assertFalse(schemaFields.get(0).getSchema().isNullable()); + + // test that it converts non-nullable date column to nullable when zeroDateTimeBehavior is convert to null + Map connectionArguments = new HashMap<>(); + connectionArguments.put("zeroDateTimeBehavior", "CONVERT_TO_NULL"); + + schemaReader = new MysqlSchemaReader(null, connectionArguments); + schemaFields = schemaReader.getSchemaFields(resultSet); + Assert.assertTrue(schemaFields.get(0).getSchema().isNullable()); + } } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java new file mode 100644 index 000000000..9481068f1 --- /dev/null +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java @@ -0,0 +1,62 @@ + +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql; + +import org.junit.Test; + +import java.sql.Types; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MysqlUtilUnitTest { + + @Test + public void testIsZeroDateTimeToNull() { + Map connArgsMap = new HashMap<>(1); + + connArgsMap.put("zeroDateTimeBehavior", ""); + assertFalse(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + + connArgsMap.put("zeroDateTimeBehavior", "ROUND"); + assertFalse(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + + connArgsMap.put("zeroDateTimeBehavior", "CONVERT_TO_NULL"); + assertTrue(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + + connArgsMap.put("zeroDateTimeBehavior", "convertToNull"); + assertTrue(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + } + + @Test + public void testIsDateTimeLikeType() { + int dateType = Types.DATE; + int timestampType = Types.TIMESTAMP; + int timestampWithTimezoneType = Types.TIMESTAMP_WITH_TIMEZONE; + int timeType = Types.TIME; + int stringType = Types.VARCHAR; + + assertTrue(MysqlUtil.isDateTimeLikeType(dateType)); + assertTrue(MysqlUtil.isDateTimeLikeType(timestampType)); + assertTrue(MysqlUtil.isDateTimeLikeType(timestampWithTimezoneType)); + assertFalse(MysqlUtil.isDateTimeLikeType(timeType)); + assertFalse(MysqlUtil.isDateTimeLikeType(stringType)); + } +} From 648826b0014e1a5a44254fdee937efea35c30344 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Fri, 14 Jun 2024 17:25:48 +0530 Subject: [PATCH 10/62] oracle issue commit --- .../io/cdap/plugin/oracle/OracleSinkDBRecord.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java index 7bbd25f22..01b9a8247 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java @@ -17,9 +17,12 @@ package io.cdap.plugin.oracle; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.ColumnType; import io.cdap.plugin.db.SchemaReader; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.List; /** @@ -37,4 +40,14 @@ public OracleSinkDBRecord(StructuredRecord record, List columnTypes) protected SchemaReader getSchemaReader() { return new OracleSinkSchemaReader(); } + + @Override + protected void insertOperation(PreparedStatement stmt) throws SQLException { + for (int fieldIndex = 0; fieldIndex < columnTypes.size(); fieldIndex++) { + ColumnType columnType = columnTypes.get(fieldIndex); + // Get the field from the schema using the column name with ignoring case. + Schema.Field field = record.getSchema().getField(columnType.getName(), true); + writeToDB(stmt, field, fieldIndex); + } + } } From 4e339524261bba58d1f92b3f93e323cf7f9ca071 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Tue, 18 Jun 2024 16:57:28 +0530 Subject: [PATCH 11/62] Add e2e test case for small case schema. Add e2e test case for small case schema. --- .../features/sink/OracleRunTime.feature | 50 +++++++++++++++++++ .../java/io.cdap.plugin/BQValidation.java | 13 +++-- .../common.stepsdesign/TestSetupHooks.java | 14 ++++++ .../oracle/stepsdesign/Oracle.java | 21 +++++++- .../resources/pluginParameters.properties | 4 ++ .../BigQueryCreateTableQuerySmallCase.txt | 1 + .../BigQueryInsertDataQuerySmallCase.txt | 6 +++ 7 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 oracle-plugin/src/e2e-test/resources/testdata/BigQuery/BigQueryCreateTableQuerySmallCase.txt create mode 100644 oracle-plugin/src/e2e-test/resources/testdata/BigQuery/BigQueryInsertDataQuerySmallCase.txt diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature index c2b56e8b7..67293700b 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature @@ -117,3 +117,53 @@ Feature: Oracle - Verify data transfer from BigQuery source to Oracle sink Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @BQ_SOURCE_TEST_SMALL_CASE @ORACLE_TEST_TABLE + Scenario: To verify data is getting transferred from BigQuery source to Oracle sink successfully when schema is coming in small case + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Oracle" to establish connection + Then Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "bqOutputDatatypesSchemaSmallCase" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Oracle sink + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table with record counts of BigQuery table + Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table with case diff --git a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java index 6edfcc8fd..b7d93c80a 100644 --- a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java +++ b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java @@ -68,11 +68,12 @@ public static boolean validateDBToBQRecordValues(String schema, String sourceTab ResultSet.HOLD_CURSORS_OVER_COMMIT); ResultSet rsSource = statement1.executeQuery(getSourceQuery); - return compareResultSetAndJsonData(rsSource, jsonResponse); + return compareResultSetAndJsonData(rsSource, jsonResponse, false); } } - public static boolean validateBQToDBRecordValues(String schema, String sourceTable, String targetTable) + public static boolean validateBQToDBRecordValues(String schema, String sourceTable, String targetTable, + boolean isSchemaSmallCase) throws SQLException, ClassNotFoundException, ParseException, IOException, InterruptedException { List jsonResponse = new ArrayList<>(); List bigQueryRows = new ArrayList<>(); @@ -88,7 +89,7 @@ public static boolean validateBQToDBRecordValues(String schema, String sourceTab ResultSet.HOLD_CURSORS_OVER_COMMIT); ResultSet rsTarget = statement1.executeQuery(getTargetQuery); - return compareResultSetAndJsonData(rsTarget, jsonResponse); + return compareResultSetAndJsonData(rsTarget, jsonResponse, isSchemaSmallCase); } } @@ -119,7 +120,8 @@ private static void getBigQueryTableData(String table, List bigQueryRows * @throws ParseException If an error occurs while parsing the data. */ - public static boolean compareResultSetAndJsonData(ResultSet rsSource, List bigQueryData) + public static boolean compareResultSetAndJsonData(ResultSet rsSource, List bigQueryData, + boolean isSchemaSmallCase) throws SQLException, ParseException { ResultSetMetaData mdSource = rsSource.getMetaData(); boolean result = false; @@ -146,7 +148,8 @@ public static boolean compareResultSetAndJsonData(ResultSet rsSource, List Date: Thu, 20 Jun 2024 17:48:31 +0530 Subject: [PATCH 12/62] e2e workflow added for release branches --- .github/workflows/e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a49e68043..f0a29342f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,9 +16,9 @@ name: Build e2e tests on: push: - branches: [ develop ] + branches: [ develop, release/** ] pull_request: - branches: [ develop ] + branches: [ develop, release/** ] types: [ opened, synchronize, reopened, labeled ] workflow_dispatch: From eecb77b7d70a01ad5aa6c40091140544c3ea4de2 Mon Sep 17 00:00:00 2001 From: Aakash Nayak Date: Mon, 2 Sep 2024 15:11:43 +0530 Subject: [PATCH 13/62] Run build with unit tests without elevated permissions --- .github/workflows/build-report.yml | 51 ++++++++++++++++++++++++++++++ .github/workflows/build.yml | 41 +++++++++++------------- .github/workflows/trigger.yml | 46 --------------------------- 3 files changed, 70 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/build-report.yml delete mode 100644 .github/workflows/trigger.yml diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml new file mode 100644 index 000000000..691236ceb --- /dev/null +++ b/.github/workflows/build-report.yml @@ -0,0 +1,51 @@ +# Copyright © 2024 Cask Data, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven +# Note: Any changes to this workflow would be used only after merging into develop +name: Build Unit Tests Report + +on: + workflow_run: + workflows: + - Build with unit tests + types: + - completed + +jobs: + build: + runs-on: ubuntu-latest + + if: ${{ github.event.workflow_run.conclusion != 'skipped' }} + + steps: + # Pinned 1.0.0 version + - uses: marocchino/action-workflow_run-status@54b6e87d6cb552fc5f36dbe9a722a6048725917a + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + path: artifacts/ + + - name: Surefire Report + # Pinned 3.5.2 version + uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32 + if: always() + with: + report_paths: '**/target/surefire-reports/TEST-*.xml' + github_token: ${{ secrets.GITHUB_TOKEN }} + detailed_summary: true + commit: ${{ github.event.workflow_run.head_sha }} + check_name: Build Test Report + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ce0eb526..1fb6ac78a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,22 +15,28 @@ name: Build with unit tests on: - workflow_run: - workflows: - - Trigger build - types: - - completed + push: + branches: [ develop, release/** ] + pull_request: + branches: [ develop, release/** ] + types: [opened, synchronize, reopened, labeled] jobs: build: runs-on: k8s-runner-build - if: ${{ github.event.workflow_run.conclusion != 'skipped' }} - + # We allow builds: + # 1) When it's a merge into a branch + # 2) For PRs that are labeled as build and + # - It's a code change + # - A build label was just added + # A bit complex, but prevents builds when other labels are manipulated + if: > + github.event_name == 'push' + || (contains(github.event.pull_request.labels.*.name, 'build') + && (github.event.action != 'labeled' || github.event.label.name == 'build') + ) steps: - # Pinned 1.0.0 version - - uses: haya14busa/action-workflow_run-status@967ed83efa565c257675ed70cfe5231f062ddd94 - - uses: actions/checkout@v3 with: ref: ${{ github.event.workflow_run.head_sha }} @@ -47,21 +53,12 @@ jobs: run: mvn clean test -fae -T 2 -B -V -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 - name: Archive build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: Build debug files + name: reports-${{ github.run_id }} path: | **/target/rat.txt **/target/surefire-reports/* - - name: Surefire Report - # Pinned 3.5.2 version - uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32 - if: always() - with: - report_paths: '**/target/surefire-reports/TEST-*.xml' - github_token: ${{ secrets.GITHUB_TOKEN }} - detailed_summary: true - commit: ${{ github.event.workflow_run.head_sha }} - check_name: Test Report \ No newline at end of file + diff --git a/.github/workflows/trigger.yml b/.github/workflows/trigger.yml deleted file mode 100644 index 11db8ac25..000000000 --- a/.github/workflows/trigger.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright © 2022 Cask Data, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -# This workflow will trigger build.yml only when needed. -# This way we don't flood main workflow run list -# Note that build.yml from develop will be used even for PR builds -# Also it will have access to the proper GITHUB_SECRET - -name: Trigger build - -on: - push: - branches: [ develop, release/** ] - pull_request: - branches: [ develop, release/** ] - types: [opened, synchronize, reopened, labeled] - workflow_dispatch: - -jobs: - trigger: - runs-on: ubuntu-latest - - # We allow builds: - # 1) When triggered manually - # 2) When it's a merge into a branch - # 3) For PRs that are labeled as build and - # - It's a code change - # - A build label was just added - # A bit complex, but prevents builds when other labels are manipulated - if: > - github.event_name == 'workflow_dispatch' - || github.event_name == 'push' - || (contains(github.event.pull_request.labels.*.name, 'build') - && (github.event.action != 'labeled' || github.event.label.name == 'build') - ) - steps: - - name: Trigger build - run: echo Maven build will be triggered now \ No newline at end of file From e9dd995453cc614607156f869afa5333de9214ed Mon Sep 17 00:00:00 2001 From: vipinbhatt Date: Mon, 7 Oct 2024 11:10:38 +0530 Subject: [PATCH 14/62] Enable TLS Support (DTS - Specific) --- .../io/cdap/plugin/oracle/OracleAction.java | 2 +- .../cdap/plugin/oracle/OracleConnector.java | 2 +- .../plugin/oracle/OracleConnectorConfig.java | 17 +++- .../cdap/plugin/oracle/OracleConstants.java | 86 +++++++++++++++++-- .../cdap/plugin/oracle/OraclePostAction.java | 2 +- .../io/cdap/plugin/oracle/OracleSource.java | 6 +- .../oracle/OracleFailedConnectionTest.java | 3 +- oracle-plugin/widgets/Oracle-batchsink.json | 20 +++++ oracle-plugin/widgets/Oracle-batchsource.json | 20 +++++ oracle-plugin/widgets/Oracle-connector.json | 20 +++++ 10 files changed, 159 insertions(+), 19 deletions(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java index 9b5331d11..d698e17da 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java @@ -57,7 +57,7 @@ public static class OracleActionConfig extends DBSpecificQueryConfig { @Override public String getConnectionString() { - return OracleConstants.getConnectionString(this.connectionType, host, port, database); + return OracleConstants.getConnectionString(this.connectionType, host, port, database, null); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index bc7907b26..3d2f7399a 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -126,7 +126,7 @@ protected String getConnectionString(@Nullable String database) { return config.getConnectionString(); } return OracleConstants.getConnectionString(config.getConnectionType(), - config.getHost(), config.getPort(), database); + config.getHost(), config.getPort(), database, config.getSSlMode()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index a60476bd5..10022364a 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -43,12 +43,12 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { - this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null); + this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, - String role) { + String role, Boolean useSSL) { this.host = host; this.port = port; @@ -59,11 +59,12 @@ public OracleConnectorConfig(String host, int port, String user, String password this.connectionType = connectionType; this.database = database; this.role = role; + this.useSSL = useSSL; } @Override public String getConnectionString() { - return OracleConstants.getConnectionString(connectionType, host, getPort(), database); + return OracleConstants.getConnectionString(connectionType, host, getPort(), database, useSSL); } @Name(OracleConstants.CONNECTION_TYPE) @@ -86,6 +87,11 @@ public String getConnectionString() { @Nullable private String transactionIsolationLevel; + @Name(OracleConstants.USE_SSL) + @Description("Turns on SSL encryption. Connection will fail if SSL is not available") + @Nullable + public Boolean useSSL; + @Override protected int getDefaultPort() { return 1521; @@ -103,6 +109,11 @@ public String getDatabase() { return database; } + public Boolean getSSlMode() { + // return false if useSSL is null, otherwise return its value + return useSSL != null && useSSL; + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index ec44d7b94..dc38f80ac 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -29,6 +29,10 @@ private OracleConstants() { public static final String PLUGIN_NAME = "Oracle"; public static final String ORACLE_CONNECTION_STRING_SID_FORMAT = "jdbc:oracle:thin:@%s:%s:%s"; public static final String ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT = "jdbc:oracle:thin:@//%s:%s/%s"; + // Connection formats to accept protocol (e.g., jdbc:oracle:thin:@://:/) + public static final String ORACLE_CONNECTION_STRING_SID_FORMAT_WITH_PROTOCOL = "jdbc:oracle:thin:@%s:%s:%s/%s"; + public static final String ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT_WITH_PROTOCOL = + "jdbc:oracle:thin:@%s://%s:%s/%s"; public static final String ORACLE_CONNECTION_STRING_TNS_FORMAT = "jdbc:oracle:thin:@%s"; public static final String DEFAULT_BATCH_VALUE = "defaultBatchValue"; public static final String DEFAULT_ROW_PREFETCH = "defaultRowPrefetch"; @@ -36,28 +40,92 @@ private OracleConstants() { public static final String CONNECTION_TYPE = "connectionType"; public static final String ROLE = "role"; public static final String NAME_DATABASE = "database"; - public static final String TNS_CONNECTION_TYPE = "TNS"; + public static final String TNS_CONNECTION_TYPE = "tns"; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; + public static final String USE_SSL = "useSSL"; /** - * Returns the Connection String for the given ConnectionType. + * Constructs the Oracle connection string based on the provided connection type, host, port, and database. + * If SSL is enabled, the connection protocol will be "tcps" instead of "tcp". * * @param connectionType TNS/Service/SID * @param host Host name of the oracle server * @param port Port of the oracle server * @param database Database to connect to - * @return Connection String based on the given ConnectionType + * @param useSSL Whether SSL/TLS is required(YES/NO) + * @return Connection String based on the given parameters and connection type. */ public static String getConnectionString(String connectionType, @Nullable String host, @Nullable int port, - String database) { - if (OracleConstants.TNS_CONNECTION_TYPE.equalsIgnoreCase(connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); + String database, + @Nullable Boolean useSSL) { + // Use protocol as "tcps" when SSL is requested or else use "tcp". + String connectionProtocol; + boolean isSSLEnabled = false; + if (useSSL != null && useSSL) { + connectionProtocol = "tcps"; + isSSLEnabled = true; + } else { + connectionProtocol = "tcp"; } - if (OracleConstants.SERVICE_CONNECTION_TYPE.equalsIgnoreCase(connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, - host, port, database); + + switch (connectionType.toLowerCase()) { + case OracleConstants.TNS_CONNECTION_TYPE: + // TNS connection doesn't require protocol + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); + case OracleConstants.SERVICE_CONNECTION_TYPE: + // Create connection string for SERVICE type. + return getConnectionStringWithService(host, port, database, connectionProtocol, isSSLEnabled); + default: + // Default to SID format if no matching case is found. + return getConnectionStringWithSID(host, port, database, connectionProtocol, isSSLEnabled); + } + } + + /** + * Constructs the connection string for a SERVICE connection type. + * + * @param host Host name of the Oracle server. + * @param port Port of the Oracle server. + * @param database Database name to connect to. + * @param connectionProtocol Protocol to use for the connection ("tcp" or "tcps"). + * @param isSSLEnabled Indicates if SSL is enabled. + * @return Formatted connection string for a SERVICE connection. + */ + private static String getConnectionStringWithService(@Nullable String host, + @Nullable int port, + String database, + String connectionProtocol, + boolean isSSLEnabled) { + // Choose the appropriate format based on whether SSL is enabled. + if (isSSLEnabled) { + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT_WITH_PROTOCOL, + connectionProtocol, host, port, database); + } + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, + host, port, database); + } + + /** + * Constructs the connection string for a SID connection type. + * + * @param host Host name of the Oracle server. + * @param port Port of the Oracle server. + * @param database Database name to connect to. + * @param connectionProtocol Protocol to use for the connection ("tcp" or "tcps"). + * @param isSSLEnabled Indicates if SSL is enabled. + * @return Formatted connection string for a SID connection. + */ + private static String getConnectionStringWithSID(@Nullable String host, + @Nullable int port, + String database, + String connectionProtocol, + boolean isSSLEnabled) { + // Choose the appropriate format based on whether SSL is enabled. + if (isSSLEnabled) { + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT_WITH_PROTOCOL, + connectionProtocol, host, port, database); } return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, host, port, database); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java index 4862aebfa..e11e455c1 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java @@ -57,7 +57,7 @@ public static class OracleQueryActionConfig extends DBSpecificQueryActionConfig @Override public String getConnectionString() { - return OracleConstants.getConnectionString(this.connectionType, host, port, database); + return OracleConstants.getConnectionString(this.connectionType, host, port, database, null); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index eca7e2532..6df62e63e 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -117,9 +117,9 @@ public OracleSourceConfig(String host, int port, String user, String password, S String connectionArguments, String connectionType, String database, String role, int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, - String boundingQuery, String splitBy) { + String boundingQuery, String splitBy, Boolean useSSL) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, - connectionType, database, role); + connectionType, database, role, useSSL); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; @@ -132,7 +132,7 @@ public OracleSourceConfig(String host, int port, String user, String password, S @Override public String getConnectionString() { return OracleConstants.getConnectionString(connection.getConnectionType(), connection.getHost(), - connection.getPort(), connection.getDatabase()); + connection.getPort(), connection.getDatabase(), connection.getSSlMode()); } @Override diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java index a2c9bcd5e..7ec6f3844 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java @@ -28,7 +28,8 @@ public class OracleFailedConnectionTest extends DBSpecificFailedConnectionTest { public void test() throws ClassNotFoundException, IOException { OracleConnector connector = new OracleConnector( - new OracleConnectorConfig("localhost", 1521, "username", "password", "jdbc", "", "database")); + new OracleConnectorConfig("localhost", 1521, "username", "password", "jdbc", "", + "SID", "database")); super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string:" + " jdbc:oracle:thin:@localhost:1521:database and arguments: " + diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index 30d5b345f..8d6168780 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -100,6 +100,26 @@ "default": "TRANSACTION_SERIALIZABLE" } }, + { + "widget-type": "hidden", + "label": "TLS Encryption", + "name": "useSSL", + "description": "Enable TLS encryption (true/false)", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 0fc0a5285..5eca20cc4 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -100,6 +100,26 @@ "default": "TRANSACTION_SERIALIZABLE" } }, + { + "widget-type": "hidden", + "label": "TLS Encryption", + "name": "useSSL", + "description": "Enable TLS encryption (true/false)", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 46f006c9c..628027caf 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -109,6 +109,26 @@ ], "default": "TRANSACTION_SERIALIZABLE" } + }, + { + "widget-type": "hidden", + "label": "TLS Encryption", + "name": "useSSL", + "description": "Enable TLS encryption (true/false)", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] }, From ca328a2fea43e0be57ca183546d51010e11e6214 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Tue, 15 Oct 2024 23:22:21 +0530 Subject: [PATCH 15/62] Added fix for date datatype Added fix for date datatype --- .../features/sink/OracleRunTime.feature | 51 +++++++++++++++++++ .../java/io.cdap.plugin/BQValidation.java | 48 ++++++++++++++--- .../java/io.cdap.plugin/OracleClient.java | 10 ++++ .../common.stepsdesign/TestSetupHooks.java | 30 +++++++++++ .../resources/pluginParameters.properties | 6 +++ .../BigQuery/CreateBQTableQueryFileDate.txt | 1 + .../BigQuery/InsertBQDataQueryFileDate.txt | 1 + .../plugin/oracle/OracleSourceDBRecord.java | 4 +- 8 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 oracle-plugin/src/e2e-test/resources/testdata/BigQuery/CreateBQTableQueryFileDate.txt create mode 100644 oracle-plugin/src/e2e-test/resources/testdata/BigQuery/InsertBQDataQueryFileDate.txt diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature index 67293700b..70b1bdba6 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature @@ -167,3 +167,54 @@ Feature: Oracle - Verify data transfer from BigQuery source to Oracle sink Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table with case + + + @BQ_SOURCE_TEST_DATE @ORACLE_DATE_TABLE + Scenario: To verify data is getting transferred from BigQuery source to Oracle sink successfully when schema is having date and timestamp fields + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Oracle" to establish connection + Then Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesDateTimeSchema" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Oracle sink + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table with record counts of BigQuery table + Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table diff --git a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java index b7d93c80a..b5a82e420 100644 --- a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java +++ b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java @@ -33,7 +33,12 @@ import java.sql.Types; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; import java.util.Date; import java.util.List; @@ -44,6 +49,13 @@ public class BQValidation { + private static final List TIMESTAMP_DATE_FORMATS = Arrays.asList( + new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss"), + new SimpleDateFormat("yyyy-MM-dd")); + private static final List TIMESTAMP_TZ_DATE_FORMATS = Arrays.asList( + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssXXX"), + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX")); + /** * Extracts entire data from source and target tables. * @@ -173,21 +185,43 @@ public static boolean compareResultSetAndJsonData(ResultSet rsSource, List Date: Wed, 23 Oct 2024 14:59:17 +0530 Subject: [PATCH 16/62] Escape column names --- .../cloudsql/mysql/CloudSQLMySQLSink.java | 25 +++++++++++++ .../cloudsql/mysql/CloudSQLMySQLSinkTest.java | 35 +++++++++++++++++++ .../io/cdap/plugin/mysql/MysqlDBRecord.java | 9 +++++ .../java/io/cdap/plugin/mysql/MysqlSink.java | 24 +++++++++++++ .../io/cdap/plugin/mysql/MysqlSinkTest.java | 35 +++++++++++++++++++ 5 files changed, 128 insertions(+) create mode 100644 cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java create mode 100644 mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index 271012f7e..6149c114b 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -16,6 +16,7 @@ package io.cdap.plugin.cloudsql.mysql; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import io.cdap.cdap.api.annotation.Description; @@ -25,6 +26,7 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.batch.BatchSink; @@ -40,7 +42,11 @@ import io.cdap.plugin.util.CloudSQLUtil; import io.cdap.plugin.util.DBUtils; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.StringJoiner; import javax.annotation.Nullable; /** Sink support for a CloudSQL MySQL database. */ @@ -52,6 +58,7 @@ public class CloudSQLMySQLSink extends AbstractDBSink { private final CloudSQLMySQLSinkConfig cloudsqlMysqlSinkConfig; + private static final Character ESCAPE_CHAR = '`'; public CloudSQLMySQLSink(CloudSQLMySQLSinkConfig cloudsqlMysqlSinkConfig) { super(cloudsqlMysqlSinkConfig); @@ -78,6 +85,24 @@ protected DBRecord getDBRecord(StructuredRecord output) { return new MysqlDBRecord(output, columnTypes); } + @Override + protected void setColumnsInfo(List fields) { + List columnsList = new ArrayList<>(); + StringJoiner columnsJoiner = new StringJoiner(","); + for (Schema.Field field : fields) { + columnsList.add(field.getName()); + columnsJoiner.add(ESCAPE_CHAR + field.getName() + ESCAPE_CHAR); + } + + super.columns = Collections.unmodifiableList(columnsList); + super.dbColumns = columnsJoiner.toString(); + } + + @VisibleForTesting + String getDbColumns() { + return dbColumns; + } + @Override protected LineageRecorder getLineageRecorder(BatchSinkContext context) { String host; diff --git a/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java new file mode 100644 index 000000000..65a14502e --- /dev/null +++ b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.cloudsql.mysql; + +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; + +public class CloudSQLMySQLSinkTest { + @Test + public void testSetColumnsInfo() { + Schema outputSchema = Schema.recordOf("output", + Schema.Field.of("id", Schema.of(Schema.Type.INT)), + Schema.Field.of("name", Schema.of(Schema.Type.STRING)), + Schema.Field.of("insert", Schema.of(Schema.Type.STRING))); + CloudSQLMySQLSink cloudSQLMySQLSink = new CloudSQLMySQLSink(new CloudSQLMySQLSink.CloudSQLMySQLSinkConfig()); + Assert.assertNotNull(outputSchema.getFields()); + cloudSQLMySQLSink.setColumnsInfo(outputSchema.getFields()); + Assert.assertEquals("`id`,`name`,`insert`", cloudSQLMySQLSink.getDbColumns()); + } +} diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java index 0560b10c3..94b711786 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java @@ -93,4 +93,13 @@ protected void writeNonNullToDB(PreparedStatement stmt, Schema fieldSchema, super.writeNonNullToDB(stmt, fieldSchema, fieldName, fieldIndex); } + + @Override + protected void insertOperation(PreparedStatement stmt) throws SQLException { + for (int fieldIndex = 0; fieldIndex < columnTypes.size(); fieldIndex++) { + ColumnType columnType = columnTypes.get(fieldIndex); + Schema.Field field = record.getSchema().getField(columnType.getName(), true); + writeToDB(stmt, field, fieldIndex); + } + } } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index c839cb12b..f71371026 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -16,6 +16,7 @@ package io.cdap.plugin.mysql; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; @@ -24,6 +25,7 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSink; import io.cdap.cdap.etl.api.batch.BatchSinkContext; @@ -39,9 +41,12 @@ import io.cdap.plugin.db.sink.FieldsValidator; import io.cdap.plugin.util.DBUtils; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.StringJoiner; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -54,6 +59,7 @@ public class MysqlSink extends AbstractDBSink { private final MysqlSinkConfig mysqlSinkConfig; + private static final Character ESCAPE_CHAR = '`'; public MysqlSink(MysqlSinkConfig mysqlSinkConfig) { super(mysqlSinkConfig); @@ -85,6 +91,24 @@ protected SchemaReader getSchemaReader() { return new MysqlSchemaReader(null); } + @Override + protected void setColumnsInfo(List fields) { + List columnsList = new ArrayList<>(); + StringJoiner columnsJoiner = new StringJoiner(","); + for (Schema.Field field : fields) { + columnsList.add(field.getName()); + columnsJoiner.add(ESCAPE_CHAR + field.getName() + ESCAPE_CHAR); + } + + super.columns = Collections.unmodifiableList(columnsList); + super.dbColumns = columnsJoiner.toString(); + } + + @VisibleForTesting + String getDbColumns() { + return dbColumns; + } + /** * MySQL action configuration. */ diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java new file mode 100644 index 000000000..1dd4e809e --- /dev/null +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql; + +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; + +public class MysqlSinkTest { + @Test + public void testSetColumnsInfo() { + Schema outputSchema = Schema.recordOf("output", + Schema.Field.of("id", Schema.of(Schema.Type.INT)), + Schema.Field.of("name", Schema.of(Schema.Type.STRING)), + Schema.Field.of("insert", Schema.of(Schema.Type.STRING))); + MysqlSink mySQLSink = new MysqlSink(new MysqlSink.MysqlSinkConfig()); + Assert.assertNotNull(outputSchema.getFields()); + mySQLSink.setColumnsInfo(outputSchema.getFields()); + Assert.assertEquals("`id`,`name`,`insert`", mySQLSink.getDbColumns()); + } +} From 9425f0cddbc672e34ce672c3d1f5769ce599af7f Mon Sep 17 00:00:00 2001 From: psainics Date: Thu, 21 Nov 2024 11:13:30 +0530 Subject: [PATCH 17/62] Bump CDAP version to 6.11.0-SNAPSHOT --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 847e157e9..83bf5996c 100644 --- a/pom.xml +++ b/pom.xml @@ -61,10 +61,10 @@ true UTF-8 - 6.9.1 + 6.11.0-SNAPSHOT 2.11.1 13.0.1 - 2.3.0 + 3.3.6 2.2.4 4.13 2.0.1 From 2d497ae3a060bd67b7ec513d102c038586a71413 Mon Sep 17 00:00:00 2001 From: AnkitCLI Date: Mon, 2 Dec 2024 12:00:18 +0530 Subject: [PATCH 18/62] upgrade github actions v3 to v4 --- .github/workflows/build.yml | 4 ++-- .github/workflows/e2e.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1fb6ac78a..55cd4617e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,12 +37,12 @@ jobs: && (github.event.action != 'labeled' || github.event.label.name == 'build') ) steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.workflow_run.head_sha }} - name: Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f0a29342f..c710ef929 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -45,7 +45,7 @@ jobs: steps: # Pinned 1.0.0 version - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: path: plugin submodules: 'recursive' @@ -61,13 +61,13 @@ jobs: - '${{ matrix.module }}/**/e2e-test/**' - name: Checkout e2e test repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: cdapio/cdap-e2e-tests path: e2e - name: Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} @@ -157,7 +157,7 @@ jobs: CLOUDSQL_MYSQL_CONNECTION_NAME: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_CONNECTION_NAME }} - name: Upload debug files - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: Debug files - ${{ matrix.module }} From 26dc98e0dc1b6cb84491294f4a825a62071be7fe Mon Sep 17 00:00:00 2001 From: psainics Date: Thu, 5 Dec 2024 18:30:12 +0530 Subject: [PATCH 19/62] Add MysqlErrorDetailsProvider --- .../plugin/db/DBErrorDetailsProvider.java | 124 ++++++++++++++++++ .../main/java/io/cdap/plugin/db/DBRecord.java | 12 +- .../cdap/plugin/db/sink/AbstractDBSink.java | 15 ++- .../plugin/db/source/AbstractDBSource.java | 28 +++- .../mysqlsink/RunTimeWithMacros.feature | 3 + .../features/mysqlsource/RunTime.feature | 1 + .../mysqlsource/RunTimeWithMacros.feature | 4 + .../io/cdap/plugin/mysql/MysqlConstants.java | 8 +- .../mysql/MysqlErrorDetailsProvider.java | 45 +++++++ .../java/io/cdap/plugin/mysql/MysqlSink.java | 5 + .../io/cdap/plugin/mysql/MysqlSource.java | 5 + .../java/io/cdap/plugin/mysql/MysqlUtil.java | 7 +- .../resources/errorMessage.properties | 2 +- 13 files changed, 250 insertions(+), 9 deletions(-) create mode 100644 database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java create mode 100644 mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java b/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java new file mode 100644 index 000000000..bafdba3e3 --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java @@ -0,0 +1,124 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db; + +import com.google.common.base.Strings; +import com.google.common.base.Throwables; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; +import io.cdap.cdap.api.exception.ProgramFailureException; +import io.cdap.cdap.etl.api.exception.ErrorContext; +import io.cdap.cdap.etl.api.exception.ErrorDetailsProvider; + +import java.sql.SQLException; +import java.util.List; + +/** + * A custom ErrorDetailsProvider for Database plugins. + */ +public class DBErrorDetailsProvider implements ErrorDetailsProvider { + + public ProgramFailureException getExceptionDetails(Exception e, ErrorContext errorContext) { + List causalChain = Throwables.getCausalChain(e); + for (Throwable t : causalChain) { + if (t instanceof ProgramFailureException) { + // if causal chain already has program failure exception, return null to avoid double wrap. + return null; + } + if (t instanceof SQLException) { + return getProgramFailureException((SQLException) t, errorContext); + } + if (t instanceof IllegalArgumentException) { + return getProgramFailureException((IllegalArgumentException) t, errorContext); + } + if (t instanceof IllegalStateException) { + return getProgramFailureException((IllegalStateException) t, errorContext); + } + } + return null; + } + + /** + * Get a ProgramFailureException with the given error + * information from {@link SQLException}. + * + * @param e The SQLException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(SQLException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String sqlState = e.getSQLState(); + int errorCode = e.getErrorCode(); + String errorMessageWithDetails = String.format( + "Error occurred in the phase: '%s'. Error message: '%s'. Error code: '%s'. sqlState: '%s'", + errorContext.getPhase(), errorMessage, errorCode, sqlState); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessageWithDetails.endsWith(".")) { + errorMessageWithDetails = errorMessageWithDetails + "."; + } + errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, + externalDocumentationLink); + } + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessageWithDetails, getErrorTypeFromErrorCode(errorCode), false, e); + } + + /** + * Get a ProgramFailureException with the given error + * information from {@link IllegalArgumentException}. + * + * @param e The IllegalArgumentException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(IllegalArgumentException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.USER, false, e); + } + + /** + * Get a ProgramFailureException with the given error + * information from {@link IllegalStateException}. + * + * @param e The IllegalStateException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(IllegalStateException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } + + /** + * Get the external documentation link for the client errors if available. + * + * @return The external documentation link as a {@link String}. + */ + protected String getExternalDocumentationLink() { + return null; + } + + protected ErrorType getErrorTypeFromErrorCode(int errorCode) { + return ErrorType.UNKNOWN; + } +} diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java index 31bb78938..a0b1ec14e 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java @@ -21,6 +21,9 @@ import io.cdap.cdap.api.common.Bytes; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.Lazy; import org.apache.hadoop.conf.Configurable; @@ -305,7 +308,10 @@ protected void updateOperation(PreparedStatement stmt) throws SQLException { * @throws SQLException */ protected void upsertOperation(PreparedStatement stmt) throws SQLException { - throw new UnsupportedOperationException(); + String errorMessage = "Upsert operation is not supported for this plugin."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessage, ErrorType.SYSTEM, false, new UnsupportedOperationException(errorMessage)); + } private boolean fillUpdateParams(List updatedKeyList, ColumnType columnType) { @@ -366,7 +372,9 @@ private void writeToDataOut(DataOutput out, Schema.Field field) throws IOExcepti out.write((byte[]) fieldValue); break; default: - throw new IOException(String.format("Unsupported datatype: %s with value: %s.", fieldType, fieldValue)); + String errorMessage = String.format("Unsupported datatype: %s with value: %s.", fieldType, fieldValue); + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessage, ErrorType.USER, false, new IOException(errorMessage)); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 2deac8ce4..26a95405b 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -32,6 +32,7 @@ import io.cdap.cdap.etl.api.StageConfigurer; import io.cdap.cdap.etl.api.batch.BatchRuntimeContext; import io.cdap.cdap.etl.api.batch.BatchSinkContext; +import io.cdap.cdap.etl.api.exception.ErrorDetailsProviderSpec; import io.cdap.cdap.etl.api.validation.InvalidStageException; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.ReferenceBatchSink; @@ -42,6 +43,7 @@ import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.DBConfig; +import io.cdap.plugin.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.Operation; import io.cdap.plugin.db.SchemaReader; @@ -163,6 +165,16 @@ public void validateOperations(FailureCollector collector, T dbSinkConfig, @Null } } + /** + * Returns the ErrorDetailsProvider class name. + * Override this method to provide a custom ErrorDetailsProvider class name. + * + * @return ErrorDetailsProvider class name + */ + protected String getErrorDetailsProviderClassName() { + return DBErrorDetailsProvider.class.getName(); + } + @Override public void prepareRun(BatchSinkContext context) { String connectionString = dbSinkConfig.getConnectionString(); @@ -227,7 +239,8 @@ public void prepareRun(BatchSinkContext context) { configuration.set(ETLDBOutputFormat.COMMIT_BATCH_SIZE, context.getArguments().get(ETLDBOutputFormat.COMMIT_BATCH_SIZE)); } - + // set error details provider + context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); addOutputContext(context); } protected void addOutputContext(BatchSinkContext context) { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 987b5cc17..5d7447fc8 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -25,6 +25,9 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; @@ -32,6 +35,7 @@ import io.cdap.cdap.etl.api.StageConfigurer; import io.cdap.cdap.etl.api.batch.BatchRuntimeContext; import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.cdap.etl.api.exception.ErrorDetailsProviderSpec; import io.cdap.cdap.internal.io.SchemaTypeAdapter; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.ReferenceBatchSource; @@ -41,6 +45,7 @@ import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.DBConfig; +import io.cdap.plugin.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.TransactionIsolationLevel; @@ -119,8 +124,9 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { collector.addFailure("Unable to instantiate JDBC driver: " + e.getMessage(), null) .withStacktrace(e.getStackTrace()); } catch (SQLException e) { - collector.addFailure("SQL error while getting query schema: " + e.getMessage(), null) - .withStacktrace(e.getStackTrace()); + String details = String.format("SQL error while getting query schema: Error: %s, SQLState: %s, ErrorCode: %s", + e.getMessage(), e.getSQLState(), e.getErrorCode()); + collector.addFailure(details, null).withStacktrace(e.getStackTrace()); } catch (Exception e) { collector.addFailure(e.getMessage(), null).withStacktrace(e.getStackTrace()); } @@ -194,7 +200,11 @@ private Schema loadSchemaFromDB(Class driverClass) } catch (SQLException e) { // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc driver in classpath - throw new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode()); + String errorMessageWithDetails = String.format("Error occurred while trying to get schema from database." + + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + e.getMessage(), errorMessageWithDetails, ErrorType.USER, false, new SQLException(e.getMessage(), + e.getSQLState(), e.getErrorCode())); } finally { driverCleanup.destroy(); } @@ -212,6 +222,16 @@ protected SchemaReader getSchemaReader() { return new CommonSchemaReader(); } + /** + * Returns the ErrorDetailsProvider class name. + * Override this method to provide a custom ErrorDetailsProvider class name. + * + * @return ErrorDetailsProvider class name + */ + protected String getErrorDetailsProviderClassName() { + return DBErrorDetailsProvider.class.getName(); + } + private DriverCleanup loadPluginClassAndGetDriver(Class driverClass) throws IllegalAccessException, InstantiationException, SQLException { @@ -268,6 +288,8 @@ public void prepareRun(BatchSourceContext context) throws Exception { lineageRecorder.recordRead("Read", "Read from database plugin", schema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList())); } + // set error details provider + context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); context.setInput(Input.of(sourceConfig.getReferenceName(), new SourceInputFormatProvider( DataDrivenETLDBInputFormat.class, connectionConfigAccessor.getConfiguration()))); } diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature index 0c4a5995e..9f2c5d301 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature @@ -100,6 +100,7 @@ Feature: MySQL Sink - Run time scenarios (macro) Then Wait till pipeline is in running state Then Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs @BQ_SOURCE_TEST @MYSQL_TARGET_TABLE @Mysql_Required Scenario: Verify that the pipeline fails when user provides invalid Credentials for connection with Macros @@ -135,4 +136,6 @@ Feature: MySQL Sink - Run time scenarios (macro) And Enter runtime argument value "invalid.password" for key "password" And Run the Pipeline in Runtime with runtime arguments And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature index 096de1f62..d62fec005 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature @@ -143,6 +143,7 @@ Feature: MySQL Source - Run time scenarios Then Save the pipeline Then Preview and run the pipeline Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs Then Verify the preview run status of pipeline in the logs is "failed" @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature index 2ae314de8..940b97caf 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature @@ -201,7 +201,9 @@ Feature: MySQL Source - Run time scenarios (macro) And Enter runtime argument value "invalid.query" for key "importQuery" And Run the Pipeline in Runtime with runtime arguments And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required Scenario: Verify that pipeline fails when user provides invalid Credentials for connection with Macros @@ -241,4 +243,6 @@ Feature: MySQL Source - Run time scenarios (macro) And Enter runtime argument value "invalid.password" for key "Password" And Run the Pipeline in Runtime with runtime arguments And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java index 54593f580..a73e14e9f 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java @@ -16,12 +16,18 @@ package io.cdap.plugin.mysql; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; + /** * MySQL Constants. */ public final class MysqlConstants { private MysqlConstants() { - throw new AssertionError("Should not instantiate static utility class."); + String errorMessage = "Should not instantiate static utility class."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessage, ErrorType.SYSTEM, false, new AssertionError(errorMessage)); } public static final String PLUGIN_NAME = "Mysql"; diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java new file mode 100644 index 000000000..2f0a8f739 --- /dev/null +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java @@ -0,0 +1,45 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql; + +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.plugin.db.DBErrorDetailsProvider; + +/** + * A custom ErrorDetailsProvider for MySQL plugins. + */ +public class MysqlErrorDetailsProvider extends DBErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return "https://dev.mysql.com/doc/mysql-errors/9.0/en/"; + } + + @Override + protected ErrorType getErrorTypeFromErrorCode(int errorCode) { + // https://dev.mysql.com/doc/refman/9.0/en/error-message-elements.html#error-code-ranges + if (errorCode >= 1000 && errorCode <= 5999) { + return ErrorType.USER; + } else if (errorCode >= 10000 && errorCode <= 51999) { + // SYSTEM errors: Enterprise and user-defined custom error messages + return ErrorType.SYSTEM; + } else { + // UNKNOWN errors: Anything outside defined range + return ErrorType.UNKNOWN; + } + } +} diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index f71371026..42488b31e 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -109,6 +109,11 @@ String getDbColumns() { return dbColumns; } + @Override + protected String getErrorDetailsProviderClassName() { + return MysqlErrorDetailsProvider.class.getName(); + } + /** * MySQL action configuration. */ diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index a91139196..173c6d07f 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -84,6 +84,11 @@ protected SchemaReader getSchemaReader() { return new MysqlSchemaReader(null, mysqlSourceConfig.getConnectionArguments()); } + @Override + protected String getErrorDetailsProviderClassName() { + return MysqlErrorDetailsProvider.class.getName(); + } + /** * MySQL source config. */ diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java index abb4aa27b..3c3cbedcf 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java @@ -17,6 +17,9 @@ package io.cdap.plugin.mysql; import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import java.sql.Types; import java.util.Map; @@ -26,7 +29,9 @@ */ public final class MysqlUtil { private MysqlUtil() { - throw new AssertionError("Should not instantiate static utility class."); + String errorMessage = "Should not instantiate static utility class."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessage, ErrorType.SYSTEM, false, new AssertionError(errorMessage)); } /** diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index 12588ac61..b981faf81 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -7,7 +7,7 @@ errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must b errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource Oracle 'numSplits' is invalid: Value of field\ \ class io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig.numSplits is expected to be a number. errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. -errorMessageInvalidSourceDatabase=SQL error while getting query schema: Listener refused the connection with the following \ +errorMessageInvalidSourceDatabase=SQL error while getting query schema: Error: Listener refused the connection with the following \ error: ORA-12514, TNS:listener does not currently know of service requested in connect descriptor errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query From 5a01e86edd73f32d2e3df1faaf94590b8432616c Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 13 Dec 2024 15:49:35 +0530 Subject: [PATCH 20/62] Bump cdap-data lower bound --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 83bf5996c..fa4724a47 100644 --- a/pom.xml +++ b/pom.xml @@ -369,8 +369,8 @@ 1.1.0 - system:cdap-data-pipeline[6.9.0-SNAPSHOT,7.0.0-SNAPSHOT) - system:cdap-data-streams[6.9.0-SNAPSHOT,7.0.0-SNAPSHOT) + system:cdap-data-pipeline[6.11.0-SNAPSHOT,7.0.0-SNAPSHOT) + system:cdap-data-streams[6.11.0-SNAPSHOT,7.0.0-SNAPSHOT) From 24860ed53fcc1b921a9b6bef58be1151215c340c Mon Sep 17 00:00:00 2001 From: psainics Date: Tue, 17 Dec 2024 14:44:49 +0530 Subject: [PATCH 21/62] Add CloudSQLMySQLErrorDetailsProvider --- .../sink/CloudMySqlRunTimeMacro.feature | 4 +++ .../features/source/CloudMySqlRunTime.feature | 2 ++ .../CloudSQLMySQLErrorDetailsProvider.java | 31 +++++++++++++++++++ .../cloudsql/mysql/CloudSQLMySQLSink.java | 5 +++ .../cloudsql/mysql/CloudSQLMySQLSource.java | 5 +++ 5 files changed, 47 insertions(+) create mode 100644 cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java diff --git a/cloudsql-mysql-plugin/src/e2e-test/features/sink/CloudMySqlRunTimeMacro.feature b/cloudsql-mysql-plugin/src/e2e-test/features/sink/CloudMySqlRunTimeMacro.feature index 717f9dcf5..5ab8b4727 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/features/sink/CloudMySqlRunTimeMacro.feature +++ b/cloudsql-mysql-plugin/src/e2e-test/features/sink/CloudMySqlRunTimeMacro.feature @@ -142,7 +142,9 @@ Feature: CloudMySql Sink - Run time scenarios (macro) Then Enter runtime argument value "invalidTablename" for key "invalidTablename" Then Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidTableName | @@ -181,7 +183,9 @@ Feature: CloudMySql Sink - Run time scenarios (macro) Then Enter runtime argument value "invalidPassword" for key "Password" Then Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidCredentials | diff --git a/cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTime.feature b/cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTime.feature index b884fa7fa..242e53d5d 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTime.feature +++ b/cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTime.feature @@ -224,7 +224,9 @@ Feature: CloudMySql Source - Run time scenarios Then Deploy the pipeline Then Run the Pipeline in Runtime Then Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidBoundingQuery | diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java new file mode 100644 index 000000000..87f5ed2ec --- /dev/null +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.cloudsql.mysql; + + +import io.cdap.plugin.mysql.MysqlErrorDetailsProvider; + +/** + * A custom ErrorDetailsProvider for CloudSQL MySQL plugins. + */ +public class CloudSQLMySQLErrorDetailsProvider extends MysqlErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return "https://cloud.google.com/sql/docs/mysql/error-messages"; + } +} diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index 6149c114b..86a8e6f52 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -103,6 +103,11 @@ String getDbColumns() { return dbColumns; } + @Override + protected String getErrorDetailsProviderClassName() { + return CloudSQLMySQLErrorDetailsProvider.class.getName(); + } + @Override protected LineageRecorder getLineageRecorder(BatchSinkContext context) { String host; diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index b0bea9e7a..aad074ba4 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -127,6 +127,11 @@ protected SchemaReader getSchemaReader() { return new MysqlSchemaReader(null, cloudsqlMysqlSourceConfig.getConnectionArguments()); } + @Override + protected String getErrorDetailsProviderClassName() { + return CloudSQLMySQLErrorDetailsProvider.class.getName(); + } + /** CloudSQL MySQL source config. */ public static class CloudSQLMySQLSourceConfig extends AbstractDBSpecificSourceConfig { From 593719286230f7df5272d6e9d9fa30fd8984cca9 Mon Sep 17 00:00:00 2001 From: psainics Date: Tue, 17 Dec 2024 10:30:57 +0530 Subject: [PATCH 22/62] Add errorCodeType, errorCode & supportedDocURL in exception --- .../mysql/CloudSQLMySQLErrorDetailsProvider.java | 3 ++- .../cloudsql/mysql/CloudSQLMySQLSource.java | 5 +++++ .../cdap/plugin/db/DBErrorDetailsProvider.java | 5 ++++- .../main/java/io/cdap/plugin/db/DBRecord.java | 1 - .../cdap/plugin/db/source/AbstractDBSource.java | 16 +++++++++++++++- .../main/java/io/cdap/plugin/util/DBUtils.java | 2 ++ .../plugin/mysql/MysqlErrorDetailsProvider.java | 3 ++- .../java/io/cdap/plugin/mysql/MysqlSource.java | 5 +++++ 8 files changed, 35 insertions(+), 5 deletions(-) diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java index 87f5ed2ec..fe276a27a 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java @@ -18,6 +18,7 @@ import io.cdap.plugin.mysql.MysqlErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; /** * A custom ErrorDetailsProvider for CloudSQL MySQL plugins. @@ -26,6 +27,6 @@ public class CloudSQLMySQLErrorDetailsProvider extends MysqlErrorDetailsProvider @Override protected String getExternalDocumentationLink() { - return "https://cloud.google.com/sql/docs/mysql/error-messages"; + return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL; } } diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index aad074ba4..8273169c0 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -81,6 +81,11 @@ protected Class getDBRecordType() { return MysqlDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL; + } + @Override protected String createConnectionString() { if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase( diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java b/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java index bafdba3e3..22a979d58 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java @@ -19,11 +19,13 @@ import com.google.common.base.Strings; import com.google.common.base.Throwables; import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCodeType; import io.cdap.cdap.api.exception.ErrorType; import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.etl.api.exception.ErrorContext; import io.cdap.cdap.etl.api.exception.ErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; import java.sql.SQLException; import java.util.List; @@ -76,7 +78,8 @@ private ProgramFailureException getProgramFailureException(SQLException e, Error externalDocumentationLink); } return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, errorMessageWithDetails, getErrorTypeFromErrorCode(errorCode), false, e); + errorMessage, errorMessageWithDetails, getErrorTypeFromErrorCode(errorCode), false, ErrorCodeType.SQLSTATE, + sqlState, externalDocumentationLink, e); } /** diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java index a0b1ec14e..a5a9fcf5f 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java @@ -17,7 +17,6 @@ package io.cdap.plugin.db; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import io.cdap.cdap.api.common.Bytes; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 5d7447fc8..559985758 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -26,6 +26,7 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.lib.KeyValue; import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCodeType; import io.cdap.cdap.api.exception.ErrorType; import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.cdap.api.plugin.PluginConfig; @@ -202,8 +203,17 @@ private Schema loadSchemaFromDB(Class driverClass) // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc driver in classpath String errorMessageWithDetails = String.format("Error occurred while trying to get schema from database." + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessageWithDetails.endsWith(".")) { + errorMessageWithDetails = errorMessageWithDetails + "."; + } + errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, + externalDocumentationLink); + } throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - e.getMessage(), errorMessageWithDetails, ErrorType.USER, false, new SQLException(e.getMessage(), + e.getMessage(), errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); } finally { driverCleanup.destroy(); @@ -363,6 +373,10 @@ protected Class getDBRecordType() { return DBRecord.class; } + protected String getExternalDocumentationLink() { + return null; + } + @Override public void initialize(BatchRuntimeContext context) throws Exception { super.initialize(context); diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index 584c7bb3f..d526d5e36 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -60,6 +60,8 @@ public final class DBUtils { private static final Logger LOG = LoggerFactory.getLogger(DBUtils.class); public static final Calendar PURE_GREGORIAN_CALENDAR = createPureGregorianCalender(); + public static final String MYSQL_SUPPORTED_DOC_URL = "https://dev.mysql.com/doc/mysql-errors/9.0/en/"; + public static final String CLOUDSQLMYSQL_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/mysql/error-messages"; // Java by default uses October 15, 1582 as a Gregorian cut over date. // Any timestamp created with time less than this cut over date is treated as Julian date. diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java index 2f0a8f739..00f6d9810 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java @@ -18,6 +18,7 @@ import io.cdap.cdap.api.exception.ErrorType; import io.cdap.plugin.db.DBErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; /** * A custom ErrorDetailsProvider for MySQL plugins. @@ -26,7 +27,7 @@ public class MysqlErrorDetailsProvider extends DBErrorDetailsProvider { @Override protected String getExternalDocumentationLink() { - return "https://dev.mysql.com/doc/mysql-errors/9.0/en/"; + return DBUtils.MYSQL_SUPPORTED_DOC_URL; } @Override diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index 173c6d07f..971b76809 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -69,6 +69,11 @@ protected Class getDBRecordType() { return MysqlDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MYSQL_SUPPORTED_DOC_URL; + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("mysql", From 9b9d97964b1e61c4191ded985563b725caaee697 Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 20 Dec 2024 05:09:56 +0530 Subject: [PATCH 23/62] Add PostgresErrorDetailsProvider --- .../plugin/db/DBErrorDetailsProvider.java | 18 ++-- .../java/io/cdap/plugin/util/DBUtils.java | 2 + .../mysql/MysqlErrorDetailsProvider.java | 2 +- .../source/PostgresqlRunTime.feature | 2 + .../plugin/postgres/PostgresConstants.java | 8 +- .../PostgresErrorDetailsProvider.java | 96 +++++++++++++++++++ .../io/cdap/plugin/postgres/PostgresSink.java | 5 + .../cdap/plugin/postgres/PostgresSource.java | 10 ++ 8 files changed, 134 insertions(+), 9 deletions(-) create mode 100644 postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java b/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java index 22a979d58..cc731d6ac 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java @@ -25,7 +25,6 @@ import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.etl.api.exception.ErrorContext; import io.cdap.cdap.etl.api.exception.ErrorDetailsProvider; -import io.cdap.plugin.util.DBUtils; import java.sql.SQLException; import java.util.List; @@ -67,8 +66,8 @@ private ProgramFailureException getProgramFailureException(SQLException e, Error String sqlState = e.getSQLState(); int errorCode = e.getErrorCode(); String errorMessageWithDetails = String.format( - "Error occurred in the phase: '%s'. Error message: '%s'. Error code: '%s'. sqlState: '%s'", - errorContext.getPhase(), errorMessage, errorCode, sqlState); + "Error occurred in the phase: '%s' with sqlState: '%s', errorCode: '%s', errorMessage: %s", + errorContext.getPhase(), sqlState, errorCode, errorMessage); String externalDocumentationLink = getExternalDocumentationLink(); if (!Strings.isNullOrEmpty(externalDocumentationLink)) { if (!errorMessageWithDetails.endsWith(".")) { @@ -77,9 +76,10 @@ private ProgramFailureException getProgramFailureException(SQLException e, Error errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, externalDocumentationLink); } - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, errorMessageWithDetails, getErrorTypeFromErrorCode(errorCode), false, ErrorCodeType.SQLSTATE, - sqlState, externalDocumentationLink, e); + return ErrorUtils.getProgramFailureException(Strings.isNullOrEmpty(sqlState) ? + new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN) : getErrorCategoryFromSqlState(sqlState), + errorMessage, errorMessageWithDetails, getErrorTypeFromErrorCode(errorCode, sqlState), true, + ErrorCodeType.SQLSTATE, sqlState, externalDocumentationLink, e); } /** @@ -121,7 +121,11 @@ protected String getExternalDocumentationLink() { return null; } - protected ErrorType getErrorTypeFromErrorCode(int errorCode) { + protected ErrorType getErrorTypeFromErrorCode(int errorCode, String sqlState) { return ErrorType.UNKNOWN; } + + protected ErrorCategory getErrorCategoryFromSqlState(String sqlState) { + return new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN); + } } diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index d526d5e36..a20471e2b 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -62,6 +62,8 @@ public final class DBUtils { public static final Calendar PURE_GREGORIAN_CALENDAR = createPureGregorianCalender(); public static final String MYSQL_SUPPORTED_DOC_URL = "https://dev.mysql.com/doc/mysql-errors/9.0/en/"; public static final String CLOUDSQLMYSQL_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/mysql/error-messages"; + public static final String POSTGRES_SUPPORTED_DOC_URL = + "https://www.postgresql.org/docs/current/errcodes-appendix.html"; // Java by default uses October 15, 1582 as a Gregorian cut over date. // Any timestamp created with time less than this cut over date is treated as Julian date. diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java index 00f6d9810..251f0fc74 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java @@ -31,7 +31,7 @@ protected String getExternalDocumentationLink() { } @Override - protected ErrorType getErrorTypeFromErrorCode(int errorCode) { + protected ErrorType getErrorTypeFromErrorCode(int errorCode, String sqlState) { // https://dev.mysql.com/doc/refman/9.0/en/error-message-elements.html#error-code-ranges if (errorCode >= 1000 && errorCode <= 5999) { return ErrorType.USER; diff --git a/postgresql-plugin/src/e2e-test/features/postgresql/source/PostgresqlRunTime.feature b/postgresql-plugin/src/e2e-test/features/postgresql/source/PostgresqlRunTime.feature index ad83a1607..99a6aca3c 100644 --- a/postgresql-plugin/src/e2e-test/features/postgresql/source/PostgresqlRunTime.feature +++ b/postgresql-plugin/src/e2e-test/features/postgresql/source/PostgresqlRunTime.feature @@ -147,7 +147,9 @@ Feature: PostgreSQL - Verify data transfer from PostgreSQL source to BigQuery si And Save and Deploy Pipeline And Run the Pipeline in Runtime And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidBoundingQuery | diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConstants.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConstants.java index bed7a3ec3..4ed7cf804 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConstants.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConstants.java @@ -16,13 +16,19 @@ package io.cdap.plugin.postgres; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; + /** * Postgres constants. */ public final class PostgresConstants { private PostgresConstants() { - throw new AssertionError("Should not instantiate static utility class."); + String errorMessage = "Should not instantiate static utility class."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessage, ErrorType.SYSTEM, false, new AssertionError(errorMessage)); } public static final String PLUGIN_NAME = "Postgres"; diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java new file mode 100644 index 000000000..3202a3e28 --- /dev/null +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java @@ -0,0 +1,96 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.postgres; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.plugin.db.DBErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +import java.util.HashMap; +import java.util.Map; + +/** + * A custom ErrorDetailsProvider for Postgres plugins. + */ +public class PostgresErrorDetailsProvider extends DBErrorDetailsProvider { + // https://www.postgresql.org/docs/current/errcodes-appendix.html + private static final Map ERROR_CODE_TO_ERROR_TYPE; + private static final Map ERROR_CODE_TO_ERROR_CATEGORY; + static { + ERROR_CODE_TO_ERROR_TYPE = new HashMap<>(); + ERROR_CODE_TO_ERROR_TYPE.put("01", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("02", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("08", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("0A", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("22", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("23", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("28", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("40", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("42", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("53", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("54", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("55", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("57", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("58", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("P0", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("XX", ErrorType.SYSTEM); + + ErrorCategory.ErrorCategoryEnum plugin = ErrorCategory.ErrorCategoryEnum.PLUGIN; + ERROR_CODE_TO_ERROR_CATEGORY = new HashMap<>(); + ERROR_CODE_TO_ERROR_CATEGORY.put("01", new ErrorCategory(plugin, "Warning")); + ERROR_CODE_TO_ERROR_CATEGORY.put("02", new ErrorCategory(plugin, "No Data")); + ERROR_CODE_TO_ERROR_CATEGORY.put("08", new ErrorCategory(plugin, "Postgres Server Connection Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0A", new ErrorCategory(plugin, "Postgres Server Feature Not Supported")); + ERROR_CODE_TO_ERROR_CATEGORY.put("22", new ErrorCategory(plugin, "Postgres Server Data Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("23", new ErrorCategory(plugin, "Postgres Integrity Constraint Violation")); + ERROR_CODE_TO_ERROR_CATEGORY.put("28", new ErrorCategory(plugin, "Postgres Invalid Authorization Specification")); + ERROR_CODE_TO_ERROR_CATEGORY.put("40", new ErrorCategory(plugin, "Transaction Rollback")); + ERROR_CODE_TO_ERROR_CATEGORY.put("42", new ErrorCategory(plugin, "Syntax Error or Access Rule Violation")); + ERROR_CODE_TO_ERROR_CATEGORY.put("53", new ErrorCategory(plugin, "Postgres Server Insufficient Resources")); + ERROR_CODE_TO_ERROR_CATEGORY.put("54", new ErrorCategory(plugin, "Postgres Program Limit Exceeded")); + ERROR_CODE_TO_ERROR_CATEGORY.put("55", new ErrorCategory(plugin, "Object Not in Prerequisite State")); + ERROR_CODE_TO_ERROR_CATEGORY.put("57", new ErrorCategory(plugin, "Operator Intervention")); + ERROR_CODE_TO_ERROR_CATEGORY.put("58", new ErrorCategory(plugin, "Postgres Server System Error")); + ERROR_CODE_TO_ERROR_CATEGORY.put("P0", new ErrorCategory(plugin, "PL/pgSQL Error")); + ERROR_CODE_TO_ERROR_CATEGORY.put("XX", new ErrorCategory(plugin, "Postgres Server Internal Error")); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.POSTGRES_SUPPORTED_DOC_URL; + } + + @Override + protected ErrorType getErrorTypeFromErrorCode(int errorCode, String sqlState) { + if (!Strings.isNullOrEmpty(sqlState) && sqlState.length() >= 2 && + ERROR_CODE_TO_ERROR_TYPE.containsKey(sqlState.substring(0, 2))) { + return ERROR_CODE_TO_ERROR_TYPE.get(sqlState.substring(0, 2)); + } + return ErrorType.UNKNOWN; + } + + @Override + protected ErrorCategory getErrorCategoryFromSqlState(String sqlState) { + if (!Strings.isNullOrEmpty(sqlState) && sqlState.length() >= 2 && + ERROR_CODE_TO_ERROR_CATEGORY.containsKey(sqlState.substring(0, 2))) { + return ERROR_CODE_TO_ERROR_CATEGORY.get(sqlState.substring(0, 2)); + } + return new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN); + } +} diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index 8fd91cc63..3becf5f27 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -116,6 +116,11 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getErrorDetailsProviderClassName() { + return PostgresErrorDetailsProvider.class.getName(); + } + /** * PostgreSQL action configuration. */ diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index d6677884f..8e3c091f9 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -67,11 +67,21 @@ protected SchemaReader getSchemaReader() { return new PostgresSchemaReader(); } + @Override + protected String getErrorDetailsProviderClassName() { + return PostgresErrorDetailsProvider.class.getName(); + } + @Override protected Class getDBRecordType() { return PostgresDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.POSTGRES_SUPPORTED_DOC_URL; + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("postgres", From 206d5c9fb87276a12b7f1321a6d5c6575a0ca02c Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 20 Dec 2024 08:20:12 +0530 Subject: [PATCH 24/62] Add CloudSQLPostgreSQLErrorDetailsProvider --- ...loudSQLPostgreSQLErrorDetailsProvider.java | 30 +++++++++++++++++++ .../postgres/CloudSQLPostgreSQLSink.java | 5 ++++ .../postgres/CloudSQLPostgreSQLSource.java | 10 +++++++ .../java/io/cdap/plugin/util/DBUtils.java | 2 ++ 4 files changed, 47 insertions(+) create mode 100644 cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLErrorDetailsProvider.java diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLErrorDetailsProvider.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLErrorDetailsProvider.java new file mode 100644 index 000000000..cfb402468 --- /dev/null +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLErrorDetailsProvider.java @@ -0,0 +1,30 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.cloudsql.postgres; + +import io.cdap.plugin.postgres.PostgresErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for CloudSQL PostgreSQL plugin. + */ +public class CloudSQLPostgreSQLErrorDetailsProvider extends PostgresErrorDetailsProvider { + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL; + } +} diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index f2c04e051..c8ca0d6dc 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -147,6 +147,11 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { return new LineageRecorder(context, assetBuilder.build()); } + @Override + protected String getErrorDetailsProviderClassName() { + return CloudSQLPostgreSQLErrorDetailsProvider.class.getName(); + } + /** CloudSQL PostgreSQL sink config. */ public static class CloudSQLPostgreSQLSinkConfig extends AbstractDBSpecificSinkConfig { diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java index 6d6ba29f8..e32651a9a 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java @@ -86,6 +86,16 @@ protected Class getDBRecordType() { return PostgresDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL; + } + + @Override + protected String getErrorDetailsProviderClassName() { + return CloudSQLPostgreSQLErrorDetailsProvider.class.getName(); + } + @Override protected String createConnectionString() { if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase( diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index a20471e2b..ffcfdc375 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -64,6 +64,8 @@ public final class DBUtils { public static final String CLOUDSQLMYSQL_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/mysql/error-messages"; public static final String POSTGRES_SUPPORTED_DOC_URL = "https://www.postgresql.org/docs/current/errcodes-appendix.html"; + public static final String CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL = + "https://cloud.google.com/sql/docs/postgres/error-messages"; // Java by default uses October 15, 1582 as a Gregorian cut over date. // Any timestamp created with time less than this cut over date is treated as Julian date. From 6c2677301d5612d2fba09742e87f3bffe64526ef Mon Sep 17 00:00:00 2001 From: suryakumari Date: Wed, 18 Dec 2024 13:49:44 +0530 Subject: [PATCH 25/62] e2e mysql source additional tests --- .../mysqlsource/DesignTimeValidation.feature | 74 +++++++++ .../features/mysqlsource/RunTime.feature | 148 +++++++++++++++++- .../mysqlsource/RunTimeWithMacros.feature | 55 +++++++ .../java/io/cdap/plugin/MysqlClient.java | 16 +- .../common/stepsdesign/TestSetupHooks.java | 21 ++- .../resources/errorMessage.properties | 3 + .../resources/pluginParameters.properties | 10 +- 7 files changed, 315 insertions(+), 12 deletions(-) diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature index ee317eac6..49cf4390e 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature @@ -141,3 +141,77 @@ Feature: MySQL Source - Design time validation scenarios | referenceName | | database | | importQuery | + + @Mysql_Required + Scenario: To verify MySQL source plugin validation error message with invalid reference test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "invalidRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Validate button + Then Verify that the Plugin Property: "referenceName" is displaying an in-line error message: "invalidreferenceName.error.message" + + @Mysql_Required + Scenario: To verify MySQL source plugin validation error message when fetch size is changed to zero + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Replace input plugin property: "fetchSize" with value: "zeroValue" + Then Click on the Validate button + Then Verify that the Plugin Property: "fetchSize" is displaying an in-line error message: "errorMessageInvalidFetchSize" + + @Mysql_Required + Scenario: To verify MySQL source plugin validation error message when number of Split value is changed to zero + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Replace input plugin property: "numSplits" with value: "zeroValue" + Then Click on the Validate button + Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "errorMessageInvalidNumberOfSplits" + + @Mysql_Required + Scenario: To verify MySQL source plugin validation error message with blank bounding query + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "invalidImportQuery" + Then Replace input plugin property: "splitBy" with value: "splitBy" + Then Replace input plugin property: "numSplits" with value: "numberOfSplits" + Then Click on the Validate button + Then Verify that the Plugin Property: "boundingQuery" is displaying an in-line error message: "boundingQuery.error.message" + Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "boundingQuery.error.message" + Then Verify that the Plugin Property: "importQuery" is displaying an in-line error message: "errorMessageInvalidImportQuery" diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature index d62fec005..1ad2f8cc1 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature @@ -188,4 +188,150 @@ Feature: MySQL Source - Run time scenarios Then Wait till pipeline is in running state Then Open and capture logs Then Verify the pipeline status is "Succeeded" - Then Validate the values of records transferred to target table is equal to the values from source table \ No newline at end of file + Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required + Scenario: To verify data is getting transferred from Mysql to Mysql successfully when connection arguments are set to read boolean datatype + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter key value pairs for plugin property: "connectionArguments" with values from json: "connectionArgumentsForBooleanDataType" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSourceSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Enter key value pairs for plugin property: "connectionArguments" with values from json: "connectionArgumentsForBooleanDataType" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for MySQL sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST + Scenario: To verify data is getting transferred from Mysql to Mysql successfully with bounding query + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Enter textarea plugin property: "boundingQuery" with value: "boundingQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for MySQL sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @CONNECTION + Scenario: To verify data is getting transferred from Mysql to Mysql successfully with use connection + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + And Select Mysql Connection + And Enter input plugin property: "name" with value: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + And Use new connection + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Use new connection + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for MySQL sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature index 940b97caf..4c6a77576 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature @@ -246,3 +246,58 @@ Feature: MySQL Source - Run time scenarios (macro) And Open and capture logs And Verify the pipeline status is "Failed" And Close the pipeline logs + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST + Scenario: To verify data is getting transferred from Mysql to Mysql successfully when connection arguments,import,bounding query are macro enabled + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSource" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Click on the Macro button of Property: "importQuery" and set the value in textarea: "mysqlImportQuery" + Then Replace input plugin property: "database" with value: "databaseName" + Then Click on the Macro button of Property: "boundingQuery" and set the value in textarea: "mysqlBoundingQuery" + Then Click on the Get Schema button + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSource" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "selectQuery" for key "mysqlImportQuery" + Then Enter runtime argument value "boundingQuery" for key "mysqlBoundingQuery" + And Run the preview of pipeline with runtime arguments + Then Verify the preview of pipeline is "success" + And Close the preview + And Deploy the pipeline + And Run the Pipeline in Runtime + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSource" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "selectQuery" for key "mysqlImportQuery" + Then Enter runtime argument value "boundingQuery" for key "mysqlBoundingQuery" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java index d80e4dfc4..9bcea8e02 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java @@ -138,7 +138,7 @@ public static void createSourceTable(String sourceTable) throws SQLException, Cl try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) { String createSourceTableQuery = "CREATE TABLE IF NOT EXISTS " + sourceTable + - "(id int, lastName varchar(255), PRIMARY KEY (id))"; + "(id int, lastName varchar(255), PRIMARY KEY (id), is_active BOOLEAN NOT NULL)"; statement.executeUpdate(createSourceTableQuery); // Truncate table to clean the data of last failure run. @@ -146,12 +146,12 @@ public static void createSourceTable(String sourceTable) throws SQLException, Cl statement.executeUpdate(truncateSourceTableQuery); // Insert dummy data. - statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" + - "VALUES (1, 'Simpson')"); - statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" + - "VALUES (2, 'McBeal')"); - statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" + - "VALUES (3, 'Flinstone')"); + statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName, is_active)" + + "VALUES (1, 'Simpson', true)"); + statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName, is_active)" + + "VALUES (2, 'McBeal', true)"); + statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName, is_active)" + + "VALUES (3, 'Flinstone', false)"); } } @@ -159,7 +159,7 @@ public static void createTargetTable(String targetTable) throws SQLException, Cl try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) { String createTargetTableQuery = "CREATE TABLE IF NOT EXISTS " + targetTable + - "(id int, lastName varchar(255), PRIMARY KEY (id))"; + "(id int, lastName varchar(255), PRIMARY KEY (id), is_active BOOLEAN NOT NULL)"; statement.executeUpdate(createTargetTableQuery); // Truncate table to clean the data of last failure run. String truncateTargetTableQuery = "TRUNCATE TABLE " + targetTable; diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index 2a86269b5..01475c75e 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -17,6 +17,8 @@ package io.cdap.plugin.common.stepsdesign; import com.google.cloud.bigquery.BigQueryException; +import io.cdap.e2e.pages.actions.CdfConnectionActions; +import io.cdap.e2e.pages.actions.CdfPluginPropertiesActions; import io.cdap.e2e.utils.BigQueryClient; import io.cdap.e2e.utils.PluginPropertyUtils; import io.cdap.plugin.MysqlClient; @@ -47,7 +49,10 @@ private static void setTableName() { String targetTableName = String.format("TargetTable_%s", randomString); PluginPropertyUtils.addPluginProp("sourceTable", sourceTableName); PluginPropertyUtils.addPluginProp("targetTable", targetTableName); - PluginPropertyUtils.addPluginProp("selectQuery", String.format("select * from %s", sourceTableName)); + PluginPropertyUtils.addPluginProp("selectQuery", String.format("select * from %s " + + "WHERE $CONDITIONS", sourceTableName)); + PluginPropertyUtils.addPluginProp("boundingQuery", String.format("select MIN(id),MAX(id)" + + " from %s", sourceTableName)); } @Before(order = 1) @@ -212,5 +217,19 @@ public static void setNewConnectionName() { PluginPropertyUtils.addPluginProp("connection.name", connectionName); BeforeActions.scenario.write("New Connection name: " + connectionName); } + + private static void deleteConnection(String connectionType, String connectionName) throws IOException { + CdfConnectionActions.openWranglerConnectionsPage(); + CdfConnectionActions.expandConnections(connectionType); + CdfConnectionActions.openConnectionActionMenu(connectionType, connectionName); + CdfConnectionActions.selectConnectionAction(connectionType, connectionName, "Delete"); + CdfPluginPropertiesActions.clickPluginPropertyButton("Delete"); + } + + @After(order = 1, value = "@CONNECTION") + public static void deleteBQConnection() throws IOException { + deleteConnection("MySQL", "connection.name"); + PluginPropertyUtils.removePluginProp("connection.name"); + } } diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties index 0c1ac0400..e3985cd61 100644 --- a/mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -11,3 +11,6 @@ boundingQuery.error.message=Bounding Query must be specified if Number of Splits splitfield.error.message=Split-By Field Name must be specified if Number of Splits is not set to 1 invalid.sink.database.message=Exception while trying to validate schema blank.username.message=Username is required when password is given. +errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. +errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. +errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. diff --git a/mysql-plugin/src/e2e-test/resources/pluginParameters.properties b/mysql-plugin/src/e2e-test/resources/pluginParameters.properties index 215e4b830..5cf1095c0 100644 --- a/mysql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/mysql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -44,10 +44,11 @@ invalid.tableName=table@123 sourceRef=source targetRef=target -outputSchema=[{"key":"id","value":"int"},{"key":"lastName","value":"string"}] +outputSchema=[{"key":"id","value":"int"},{"key":"lastName","value":"string"},{"key":"is_active","value":"int"}] invalid.query= select * invalid.password=mysqlroot1 invalid.database=test123 +invalidRef=%$%^_test #bq queries file path @@ -98,4 +99,9 @@ datatypesSchema=[{"key":"ID","value":"string"},{"key":"COL1","value":"long"},{"k {"key":"COL33","value":"string"}] bqOutputDatatypesSchema=[{"key":"id","value":"long"},{"key":"lastName","value":"string"}] - +connectionArguments=queryTimeout=50 +zeroValue=0 +invalidImportQuery=select +numberOfSplits=2 +connectionArgumentsForBooleanDataType=[{"key":"tinyInt1isBit","value":"true"}] +outputSourceSchema=[{"key":"id","value":"int"},{"key":"lastName","value":"string"},{"key":"is_active","value":"boolean"}] From 954447bb17d3cce33ec17fce4db0fc96d9ac8fde Mon Sep 17 00:00:00 2001 From: suryakumari Date: Fri, 20 Dec 2024 15:54:36 +0530 Subject: [PATCH 26/62] e2e mysql sink additional tests --- .../mysqlsink/DesignTimeValidation.feature | 88 +++++++++++++++++++ .../features/mysqlsink/RunTime.feature | 54 ++++++++++++ .../mysqlsink/RunTimeWithMacros.feature | 52 +++++++++++ .../resources/errorMessage.properties | 2 + 4 files changed, 196 insertions(+) diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature index 839752ae9..2c7050d08 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature @@ -167,3 +167,91 @@ Feature: MySQL Sink - Design time validation scenarios Then Enter input plugin property: "referenceName" with value: "targetRef" Then Click on the Validate button Then Verify that the Plugin is displaying an error message: "invalid.host.message" on the header + + @Mysql_Required + Scenario: Verify required fields missing validation messages + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "MySQL" + Then Click on the Validate button + Then Verify mandatory property error for below listed properties: + | jdbcPluginName | + | referenceName | + | database | + | tableName | + + @Mysql_Required + Scenario: Verify the validation error message with missing jdbc plugin name + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click plugin property: "switch-useConnection" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.jdbcPluginName.message" on the header + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required + Scenario: Verify the validation error message with blank password value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required + Scenario: Verify the validation error message with blank host value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature index 833b6b946..504ba7b77 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature @@ -147,3 +147,57 @@ Feature: MySQL Sink - Run time scenarios Then Open and capture logs Then Verify the pipeline status is "Succeeded" Then Validate the values of records transferred to target table is equal to the values from source table + + @BQ_SOURCE_TEST @MYSQL_TARGET_TABLE @CONNECTION @Mysql_Required + Scenario: To verify data is getting transferred from BigQuery to Mysql successfully with use connection + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "bqOutputDatatypesSchema" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "MySQL" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + And Select Mysql Connection + And Enter input plugin property: "name" with value: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sinkRef" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + And Use new connection + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Click plugin property: "useCompression" + Then Click plugin property: "autoReconnect" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for MySQL sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target MySQL table is equal to the values from source BigQuery table diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature index 9f2c5d301..a4cc32d6e 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature @@ -139,3 +139,55 @@ Feature: MySQL Sink - Run time scenarios (macro) And Open and capture logs And Verify the pipeline status is "Failed" And Close the pipeline logs + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required + Scenario: To verify data is getting transferred from Mysql to Mysql successfully with connection arguments, databasename and tablename macro enabled + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + And Click on the Macro button of Property: "database" and set the value to: "databaseName" + And Click on the Macro button of Property: "tableName" and set the value to: "tableName" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "databaseName" for key "databaseName" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "targetTable" for key "tableName" + And Run the preview of pipeline with runtime arguments + Then Verify the preview of pipeline is "success" + And Close the preview + And Deploy the pipeline + And Run the Pipeline in Runtime + Then Enter runtime argument value "databaseName" for key "databaseName" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "targetTable" for key "tableName" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties index e3985cd61..4a7188ff8 100644 --- a/mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -14,3 +14,5 @@ blank.username.message=Username is required when password is given. errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. +blank.connection.message=Exception while trying to validate schema of database table +blank.jdbcPluginName.message=Required property 'jdbcPluginName' has no value. From 4bd59716ef8622e559999919339c7bb6102ec89d Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Wed, 15 Jan 2025 14:03:53 +0000 Subject: [PATCH 27/62] fix e2e workflow to run on release branches --- .github/workflows/e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c710ef929..ab3be13ff 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -65,6 +65,7 @@ jobs: with: repository: cdapio/cdap-e2e-tests path: e2e + ref: release/6.11 - name: Cache uses: actions/cache@v4 From 10288ce385d33f56996c4e703c62ac925270be9a Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 27 Dec 2024 15:05:22 +0530 Subject: [PATCH 28/62] Add ExternalDocumentationLink in abstract sink --- .../cloudsql/mysql/CloudSQLMySQLSink.java | 5 +++ .../postgres/CloudSQLPostgreSQLSink.java | 5 +++ .../cdap/plugin/db/sink/AbstractDBSink.java | 31 +++++++++++++++++-- .../plugin/db/source/AbstractDBSource.java | 6 ++++ .../java/io/cdap/plugin/mysql/MysqlSink.java | 5 +++ .../io/cdap/plugin/postgres/PostgresSink.java | 5 +++ 6 files changed, 55 insertions(+), 2 deletions(-) diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index 86a8e6f52..da1414ff3 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -108,6 +108,11 @@ protected String getErrorDetailsProviderClassName() { return CloudSQLMySQLErrorDetailsProvider.class.getName(); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL; + } + @Override protected LineageRecorder getLineageRecorder(BatchSinkContext context) { String host; diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index c8ca0d6dc..65b86366c 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -152,6 +152,11 @@ protected String getErrorDetailsProviderClassName() { return CloudSQLPostgreSQLErrorDetailsProvider.class.getName(); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL; + } + /** CloudSQL PostgreSQL sink config. */ public static class CloudSQLPostgreSQLSinkConfig extends AbstractDBSpecificSinkConfig { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 26a95405b..8d7264f1c 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -25,6 +25,10 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCodeType; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; @@ -175,6 +179,16 @@ protected String getErrorDetailsProviderClassName() { return DBErrorDetailsProvider.class.getName(); } + /** + * Returns the external documentation link. + * Override this method to provide a custom external documentation link. + * + * @return external documentation link + */ + protected String getExternalDocumentationLink() { + return null; + } + @Override public void prepareRun(BatchSinkContext context) { String connectionString = dbSinkConfig.getConnectionString(); @@ -296,8 +310,21 @@ private Schema inferSchema(Class driverClass) { inferredFields.addAll(getSchemaReader().getSchemaFields(rs)); } } catch (SQLException e) { - throw new InvalidStageException("Error while reading table metadata", e); - + // wrap exception to ensure SQLException-child instances not exposed to contexts w/o jdbc driver in classpath + String errorMessageWithDetails = String.format("Error while reading table metadata." + + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessageWithDetails.endsWith(".")) { + errorMessageWithDetails = errorMessageWithDetails + "."; + } + errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, + externalDocumentationLink); + } + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + e.getMessage(), errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), + e.getSQLState(), e.getErrorCode())); } } catch (IllegalAccessException | InstantiationException | SQLException e) { throw new InvalidStageException("JDBC Driver unavailable: " + dbSinkConfig.getJdbcPluginName(), e); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 559985758..70fc12e4b 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -373,6 +373,12 @@ protected Class getDBRecordType() { return DBRecord.class; } + /** + * Returns the external documentation link. + * Override this method to provide a custom external documentation link. + * + * @return external documentation link + */ protected String getExternalDocumentationLink() { return null; } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index 42488b31e..fee0a31fa 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -114,6 +114,11 @@ protected String getErrorDetailsProviderClassName() { return MysqlErrorDetailsProvider.class.getName(); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MYSQL_SUPPORTED_DOC_URL; + } + /** * MySQL action configuration. */ diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index 3becf5f27..7682b8b0b 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -121,6 +121,11 @@ protected String getErrorDetailsProviderClassName() { return PostgresErrorDetailsProvider.class.getName(); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.POSTGRES_SUPPORTED_DOC_URL; + } + /** * PostgreSQL action configuration. */ From 7fdc40ec89202e3e85bb888013bc466be29bb31e Mon Sep 17 00:00:00 2001 From: psainics Date: Thu, 23 Jan 2025 05:50:25 +0530 Subject: [PATCH 29/62] Return null for ErrorDetailsProvider by default --- .../main/java/io/cdap/plugin/db/sink/AbstractDBSink.java | 6 ++++-- .../java/io/cdap/plugin/db/source/AbstractDBSource.java | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 8d7264f1c..3252f70bd 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -176,7 +176,7 @@ public void validateOperations(FailureCollector collector, T dbSinkConfig, @Null * @return ErrorDetailsProvider class name */ protected String getErrorDetailsProviderClassName() { - return DBErrorDetailsProvider.class.getName(); + return null; } /** @@ -254,7 +254,9 @@ public void prepareRun(BatchSinkContext context) { context.getArguments().get(ETLDBOutputFormat.COMMIT_BATCH_SIZE)); } // set error details provider - context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); + if (!Strings.isNullOrEmpty(getErrorDetailsProviderClassName())) { + context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); + } addOutputContext(context); } protected void addOutputContext(BatchSinkContext context) { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 70fc12e4b..e3340fb87 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -239,7 +239,7 @@ protected SchemaReader getSchemaReader() { * @return ErrorDetailsProvider class name */ protected String getErrorDetailsProviderClassName() { - return DBErrorDetailsProvider.class.getName(); + return null; } private DriverCleanup loadPluginClassAndGetDriver(Class driverClass) @@ -299,7 +299,9 @@ public void prepareRun(BatchSourceContext context) throws Exception { schema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList())); } // set error details provider - context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); + if (!Strings.isNullOrEmpty(getErrorDetailsProviderClassName())) { + context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); + } context.setInput(Input.of(sourceConfig.getReferenceName(), new SourceInputFormatProvider( DataDrivenETLDBInputFormat.class, connectionConfigAccessor.getConfiguration()))); } From 4137fe9b9182bf1a487532cfce342df76747060d Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 27 Jan 2025 18:34:05 +0530 Subject: [PATCH 30/62] Bump cdap.plugin.version to 2.13.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fa4724a47..a3dacc4bf 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ UTF-8 6.11.0-SNAPSHOT - 2.11.1 + 2.13.0-SNAPSHOT 13.0.1 3.3.6 2.2.4 From 35607b5e476da4840a8e3e68c20b31e30aeaedec Mon Sep 17 00:00:00 2001 From: psainics Date: Tue, 28 Jan 2025 16:46:02 +0530 Subject: [PATCH 31/62] Revert cdap.plugin.version to 2.13.0-SNAPSHOT --- .../cdap/plugin/db/CommonSchemaReaderTest.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java b/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java index 0f5a3ca4a..cbe1361d0 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java @@ -17,6 +17,7 @@ package io.cdap.plugin.db; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.ProgramFailureException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -162,49 +163,49 @@ public void testGetSchemaThrowsExceptionOnNumericWithZeroPrecision() throws SQLE reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnArray() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.ARRAY); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnDatalink() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.DATALINK); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnDistinct() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.DISTINCT); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnJavaObject() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.JAVA_OBJECT); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnOther() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.OTHER); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnRef() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.REF); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnSQLXML() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.SQLXML); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnStruct() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.STRUCT); reader.getSchema(metadata, 1); From 1d8b9680023f6cdc8c1d055532beb643eccabb52 Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 29 Jan 2025 10:39:22 +0530 Subject: [PATCH 32/62] Remove DBErrorDetailsProvider --- .../plugin/db/DBErrorDetailsProvider.java | 131 ------------------ .../cdap/plugin/db/sink/AbstractDBSink.java | 13 +- .../plugin/db/source/AbstractDBSource.java | 13 +- .../mysql/MysqlErrorDetailsProvider.java | 4 +- .../PostgresErrorDetailsProvider.java | 4 +- 5 files changed, 18 insertions(+), 147 deletions(-) delete mode 100644 database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java b/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java deleted file mode 100644 index cc731d6ac..000000000 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright © 2024 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.db; - -import com.google.common.base.Strings; -import com.google.common.base.Throwables; -import io.cdap.cdap.api.exception.ErrorCategory; -import io.cdap.cdap.api.exception.ErrorCodeType; -import io.cdap.cdap.api.exception.ErrorType; -import io.cdap.cdap.api.exception.ErrorUtils; -import io.cdap.cdap.api.exception.ProgramFailureException; -import io.cdap.cdap.etl.api.exception.ErrorContext; -import io.cdap.cdap.etl.api.exception.ErrorDetailsProvider; - -import java.sql.SQLException; -import java.util.List; - -/** - * A custom ErrorDetailsProvider for Database plugins. - */ -public class DBErrorDetailsProvider implements ErrorDetailsProvider { - - public ProgramFailureException getExceptionDetails(Exception e, ErrorContext errorContext) { - List causalChain = Throwables.getCausalChain(e); - for (Throwable t : causalChain) { - if (t instanceof ProgramFailureException) { - // if causal chain already has program failure exception, return null to avoid double wrap. - return null; - } - if (t instanceof SQLException) { - return getProgramFailureException((SQLException) t, errorContext); - } - if (t instanceof IllegalArgumentException) { - return getProgramFailureException((IllegalArgumentException) t, errorContext); - } - if (t instanceof IllegalStateException) { - return getProgramFailureException((IllegalStateException) t, errorContext); - } - } - return null; - } - - /** - * Get a ProgramFailureException with the given error - * information from {@link SQLException}. - * - * @param e The SQLException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(SQLException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String sqlState = e.getSQLState(); - int errorCode = e.getErrorCode(); - String errorMessageWithDetails = String.format( - "Error occurred in the phase: '%s' with sqlState: '%s', errorCode: '%s', errorMessage: %s", - errorContext.getPhase(), sqlState, errorCode, errorMessage); - String externalDocumentationLink = getExternalDocumentationLink(); - if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessageWithDetails.endsWith(".")) { - errorMessageWithDetails = errorMessageWithDetails + "."; - } - errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, - externalDocumentationLink); - } - return ErrorUtils.getProgramFailureException(Strings.isNullOrEmpty(sqlState) ? - new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN) : getErrorCategoryFromSqlState(sqlState), - errorMessage, errorMessageWithDetails, getErrorTypeFromErrorCode(errorCode, sqlState), true, - ErrorCodeType.SQLSTATE, sqlState, externalDocumentationLink, e); - } - - /** - * Get a ProgramFailureException with the given error - * information from {@link IllegalArgumentException}. - * - * @param e The IllegalArgumentException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(IllegalArgumentException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.USER, false, e); - } - - /** - * Get a ProgramFailureException with the given error - * information from {@link IllegalStateException}. - * - * @param e The IllegalStateException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(IllegalStateException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); - } - - /** - * Get the external documentation link for the client errors if available. - * - * @return The external documentation link as a {@link String}. - */ - protected String getExternalDocumentationLink() { - return null; - } - - protected ErrorType getErrorTypeFromErrorCode(int errorCode, String sqlState) { - return ErrorType.UNKNOWN; - } - - protected ErrorCategory getErrorCategoryFromSqlState(String sqlState) { - return new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN); - } -} diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 3252f70bd..797abfc23 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -47,7 +47,6 @@ import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.DBConfig; -import io.cdap.plugin.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.Operation; import io.cdap.plugin.db.SchemaReader; @@ -313,18 +312,20 @@ private Schema inferSchema(Class driverClass) { } } catch (SQLException e) { // wrap exception to ensure SQLException-child instances not exposed to contexts w/o jdbc driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), + e.getSQLState(), e.getErrorCode()); String errorMessageWithDetails = String.format("Error while reading table metadata." + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); String externalDocumentationLink = getExternalDocumentationLink(); if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessageWithDetails.endsWith(".")) { - errorMessageWithDetails = errorMessageWithDetails + "."; + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; } - errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, - externalDocumentationLink); + errorMessage = String.format("%s For more details, see %s", errorMessageWithDetails, errorMessage); } throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - e.getMessage(), errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index e3340fb87..3fd490ada 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -46,7 +46,6 @@ import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.DBConfig; -import io.cdap.plugin.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.TransactionIsolationLevel; @@ -201,18 +200,20 @@ private Schema loadSchemaFromDB(Class driverClass) } catch (SQLException e) { // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), + e.getSQLState(), e.getErrorCode()); String errorMessageWithDetails = String.format("Error occurred while trying to get schema from database." + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); String externalDocumentationLink = getExternalDocumentationLink(); if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessageWithDetails.endsWith(".")) { - errorMessageWithDetails = errorMessageWithDetails + "."; + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; } - errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, - externalDocumentationLink); + errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); } throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - e.getMessage(), errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); } finally { diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java index 251f0fc74..ca9a2b928 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java @@ -17,7 +17,7 @@ package io.cdap.plugin.mysql; import io.cdap.cdap.api.exception.ErrorType; -import io.cdap.plugin.db.DBErrorDetailsProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; /** @@ -31,7 +31,7 @@ protected String getExternalDocumentationLink() { } @Override - protected ErrorType getErrorTypeFromErrorCode(int errorCode, String sqlState) { + protected ErrorType getErrorTypeFromErrorCodeAndSqlState(int errorCode, String sqlState) { // https://dev.mysql.com/doc/refman/9.0/en/error-message-elements.html#error-code-ranges if (errorCode >= 1000 && errorCode <= 5999) { return ErrorType.USER; diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java index 3202a3e28..a7de4e5dc 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java @@ -19,7 +19,7 @@ import com.google.common.base.Strings; import io.cdap.cdap.api.exception.ErrorCategory; import io.cdap.cdap.api.exception.ErrorType; -import io.cdap.plugin.db.DBErrorDetailsProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; import java.util.HashMap; @@ -77,7 +77,7 @@ protected String getExternalDocumentationLink() { } @Override - protected ErrorType getErrorTypeFromErrorCode(int errorCode, String sqlState) { + protected ErrorType getErrorTypeFromErrorCodeAndSqlState(int errorCode, String sqlState) { if (!Strings.isNullOrEmpty(sqlState) && sqlState.length() >= 2 && ERROR_CODE_TO_ERROR_TYPE.containsKey(sqlState.substring(0, 2))) { return ERROR_CODE_TO_ERROR_TYPE.get(sqlState.substring(0, 2)); From 35b1f6bc80effb002fcfe68216012a80df25cdc9 Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 24 Jan 2025 11:27:20 +0530 Subject: [PATCH 33/62] Add OracleErrorDetailsProvider --- .../java/io/cdap/plugin/util/DBUtils.java | 1 + .../features/source/OracleRunTime.feature | 2 ++ .../oracle/OracleErrorDetailsProvider.java | 31 +++++++++++++++++++ .../io/cdap/plugin/oracle/OracleSink.java | 9 ++++++ .../io/cdap/plugin/oracle/OracleSource.java | 10 ++++++ 5 files changed, 53 insertions(+) create mode 100644 oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index ffcfdc375..6fad9a4ab 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -64,6 +64,7 @@ public final class DBUtils { public static final String CLOUDSQLMYSQL_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/mysql/error-messages"; public static final String POSTGRES_SUPPORTED_DOC_URL = "https://www.postgresql.org/docs/current/errcodes-appendix.html"; + public static final String ORACLE_SUPPORTED_DOC_URL = "https://docs.oracle.com/en/error-help/db/ora-index.html"; public static final String CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/postgres/error-messages"; diff --git a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature index 2d1ca9ad1..fca0b3981 100644 --- a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature @@ -338,7 +338,9 @@ Feature: Oracle - Verify data transfer from Oracle source to BigQuery sink And Save and Deploy Pipeline And Run the Pipeline in Runtime And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidBoundingQuery | diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java new file mode 100644 index 000000000..e0c770c9f --- /dev/null +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for Oracle plugin. + */ +public class OracleErrorDetailsProvider extends DBErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.ORACLE_SUPPORTED_DOC_URL; + } +} diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java index 45afbcb51..511281e9d 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java @@ -82,6 +82,15 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getErrorDetailsProviderClassName() { + return OracleErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.ORACLE_SUPPORTED_DOC_URL; + } /** * Oracle action configuration. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 6df62e63e..53f75613b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -71,6 +71,16 @@ protected Class getDBRecordType() { return OracleSourceDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.ORACLE_SUPPORTED_DOC_URL; + } + + @Override + protected String getErrorDetailsProviderClassName() { + return OracleErrorDetailsProvider.class.getName(); + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("oracle", From 34e08e0857989bfa65ae2f4775b47cecfdcc4413 Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 24 Jan 2025 09:46:33 +0530 Subject: [PATCH 34/62] Add SqlServerErrorDetailsProvider --- .../java/io/cdap/plugin/util/DBUtils.java | 2 ++ .../mssql/SqlServerErrorDetailsProvider.java | 31 +++++++++++++++++++ .../io/cdap/plugin/mssql/SqlServerSink.java | 10 ++++++ .../io/cdap/plugin/mssql/SqlServerSource.java | 10 ++++++ 4 files changed, 53 insertions(+) create mode 100644 mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index 6fad9a4ab..34043d513 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -61,6 +61,8 @@ public final class DBUtils { public static final Calendar PURE_GREGORIAN_CALENDAR = createPureGregorianCalender(); public static final String MYSQL_SUPPORTED_DOC_URL = "https://dev.mysql.com/doc/mysql-errors/9.0/en/"; + public static final String MSSQL_SUPPORTED_DOC_URL = + "https://docs.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors"; public static final String CLOUDSQLMYSQL_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/mysql/error-messages"; public static final String POSTGRES_SUPPORTED_DOC_URL = "https://www.postgresql.org/docs/current/errcodes-appendix.html"; diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java new file mode 100644 index 000000000..90d1ce7b7 --- /dev/null +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mssql; + +import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for SQL Server plugins. + */ +public class SqlServerErrorDetailsProvider extends DBErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MSSQL_SUPPORTED_DOC_URL; + } +} diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index 0fa8991c5..223785f6c 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -88,6 +88,16 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getErrorDetailsProviderClassName() { + return SqlServerErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MSSQL_SUPPORTED_DOC_URL; + } + /** * MSSQL action configuration. */ diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java index 9603b24db..01dd6c9ca 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java @@ -75,6 +75,11 @@ protected Class getDBRecordType() { return SqlServerSourceDBRecord.class; } + @Override + protected String getErrorDetailsProviderClassName() { + return SqlServerErrorDetailsProvider.class.getName(); + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("mssql", @@ -85,6 +90,11 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MSSQL_SUPPORTED_DOC_URL; + } + /** * MSSQL source config. */ From 9e86b11d6ade7a6098e8353c83680a6dc4dd5264 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 27 Jan 2025 15:19:40 +0530 Subject: [PATCH 35/62] Show correct DB name in error string --- .../src/e2e-test/resources/errorMessage.properties | 2 +- .../plugin/cloudsql/mysql/CloudSQLMySQLAction.java | 3 ++- .../plugin/cloudsql/mysql/CloudSQLMySQLSink.java | 3 ++- .../plugin/cloudsql/mysql/CloudSQLMySQLSource.java | 3 ++- .../postgres/CloudSQLPostgreSQLAction.java | 3 ++- .../cloudsql/postgres/CloudSQLPostgreSQLSink.java | 3 ++- .../postgres/CloudSQLPostgreSQLSource.java | 3 ++- .../java/io/cdap/plugin/util/CloudSQLUtil.java | 14 +++++++++----- 8 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 5ff3357f2..1c0b76971 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -12,7 +12,7 @@ errorMessageNumberOfSplits=Split-By Field Name must be specified if Number of Sp errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits is not set to 1. Specify the Bounding Query. errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'URLDecoder: Illegal hex characters in escape (%) pattern - For input string: "$^"' errorMessageInvalidTableName=Table 'Invalidtable' does not exist. Ensure table 'Invalidtable' is set correctly and -errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL PostgreSQL instance. +errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL MySQL instance. validationSuccessMessage=No errors found. validationErrorMessage=COUNT ERROR found errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java index 0608edb75..770dd9030 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java @@ -55,7 +55,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlActionConfig.instanceType, - cloudsqlMysqlActionConfig.connectionName); + cloudsqlMysqlActionConfig.connectionName, + CloudSQLUtil.CLOUDSQL_MYSQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index da1414ff3..6cd1b0031 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -74,7 +74,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlSinkConfig.connection.getInstanceType(), - cloudsqlMysqlSinkConfig.connection.getConnectionName()); + cloudsqlMysqlSinkConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_MYSQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index 8273169c0..201360c67 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -70,7 +70,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlSourceConfig.connection.getInstanceType(), - cloudsqlMysqlSourceConfig.connection.getConnectionName()); + cloudsqlMysqlSourceConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_MYSQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java index 1a3f8ad7b..5b13759f6 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java @@ -55,7 +55,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlActionConfig.instanceType, - cloudsqlPostgresqlActionConfig.connectionName); + cloudsqlPostgresqlActionConfig.connectionName, + CloudSQLUtil.CLOUDSQL_POSTGRESQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index 65b86366c..060b67f82 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -81,7 +81,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlSinkConfig.connection.getInstanceType(), - cloudsqlPostgresqlSinkConfig.connection.getConnectionName()); + cloudsqlPostgresqlSinkConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_POSTGRESQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java index e32651a9a..db3f2d708 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java @@ -70,7 +70,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlSourceConfig.connection.getInstanceType(), - cloudsqlPostgresqlSourceConfig.connection.getConnectionName()); + cloudsqlPostgresqlSourceConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_POSTGRESQL); } super.configurePipeline(pipelineConfigurer); diff --git a/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java b/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java index 11595ac06..f704f2ad5 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java @@ -31,6 +31,9 @@ public class CloudSQLUtil { public static final String INSTANCE_TYPE = "instanceType"; public static final String PUBLIC_INSTANCE = "public"; public static final String PRIVATE_INSTANCE = "private"; + public static final String CLOUDSQL_POSTGRESQL = "CloudSQL PostgreSQL"; + public static final String CLOUDSQL_MYSQL = "CloudSQL MySQL"; + /** * Utility method to check the Connection Name format of a CloudSQL instance. @@ -38,9 +41,10 @@ public class CloudSQLUtil { * @param failureCollector {@link FailureCollector} for the pipeline * @param instanceType CloudSQL instance type * @param connectionName Connection Name for the CloudSQL instance + * @param databaseType Type of CloudSQL instance- CloudSQL PostgreSQL, CLoudSQL MySQL */ public static void checkConnectionName( - FailureCollector failureCollector, String instanceType, String connectionName) { + FailureCollector failureCollector, String instanceType, String connectionName, String databaseType) { if (PUBLIC_INSTANCE.equalsIgnoreCase(instanceType)) { Pattern connectionNamePattern = @@ -50,16 +54,16 @@ public static void checkConnectionName( if (!matcher.matches()) { failureCollector .addFailure( - "Connection Name must be in the format :: to connect to " - + "a public CloudSQL PostgreSQL instance.", null) + String.format("Connection Name must be in the format :: to connect to " + + "a public %s instance.", databaseType), null) .withConfigProperty(CONNECTION_NAME); } } else { if (!InetAddresses.isInetAddress(connectionName)) { failureCollector .addFailure( - "Enter the internal IP address of the Compute Engine VM cloudsql proxy " - + "is running on, to connect to a private CloudSQL PostgreSQL instance.", null) + String.format("Enter the internal IP address of the Compute Engine VM cloudsql proxy " + + "is running on, to connect to a private %s instance.", databaseType), null) .withConfigProperty(CONNECTION_NAME); } } From fd55012c8f1526f2b802eae60e2d90889136b7c5 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Fri, 31 Jan 2025 12:27:52 +0530 Subject: [PATCH 36/62] e2e tests pipeline preview error fix --- .../src/e2e-test/resources/errorMessage.properties | 10 ++++++---- .../src/e2e-test/resources/errorMessage.properties | 12 ++++++------ .../e2e-test/features/mysqlsource/RunTime.feature | 2 +- .../e2e-test/features/source/OracleRunTime.feature | 2 +- .../src/e2e-test/resources/errorMessage.properties | 6 ++++-- .../src/e2e-test/resources/errorMessage.properties | 4 ++-- 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 1c0b76971..d4fc1de28 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -15,7 +15,9 @@ errorMessageInvalidTableName=Table 'Invalidtable' does not exist. Ensure table ' errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL MySQL instance. validationSuccessMessage=No errors found. validationErrorMessage=COUNT ERROR found -errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ - Table -errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Errors were encountered during validation. -errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'table' at line 1. Please check the system logs for more details. +errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \ + Table 'Table123' does not exist.. Please check the system logs for more details. +errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \ + Exception while trying to validate schema of database table +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : java.io.IOException: You have an error in your SQL syntax; \ + check the manual that corresponds to your MySQL server version for the right syntax to use near 'table' at line 1. Please check the system logs for more details. diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index 00721f148..c752d6ec1 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -13,11 +13,11 @@ errorMessagenumofSplit=Split-By Field Name must be specified if Number of Splits errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table errorMessageInvalidSinkTableName=Table 'Table123@' does not exist. errormessageBlankHost=Exception while trying to validate schema of database table -errorMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ - Table 'Table123@' does not exist.. Please check the system logs for more details. +errorMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'SQL Server2' encountered : io.cdap.cdap.etl.api.validation.ValidationException: \ + Errors were encountered during validation. Table 'Table123@' does not exist.. Please check the system logs for more details. errorMessageInvalidCredentials=Spark program 'phase-1' failed with error: Unable to create config for batchsink SqlServer \ 'connection' is invalid: Failed to assign value -errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Incorrect syntax near the keyword 'table'.. \ - Please check the system logs for more details. -errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Plugin with id SQL \ - Server:source.jdbc.sqlserver does not exist in program phase-1 of application +errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : io.cdap.cdap.api.exception.ProgramFailureException: \ + Error occurred while trying to get schema from database.Error message: 'Incorrect syntax near the keyword 'table'.'. +errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.lang.IllegalArgumentException: \ + Plugin with id SQL Server:source.jdbc.sqlserver does not exist in program phase-1 of application diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature index 1ad2f8cc1..0ea426da0 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature @@ -142,7 +142,7 @@ Feature: MySQL Source - Run time scenarios Then Close the Plugin Properties page Then Save the pipeline Then Preview and run the pipeline - Then Wait till pipeline preview is in running state + Then Wait till pipeline preview is in running state and check if any error occurs Then Open and capture pipeline preview logs Then Verify the preview run status of pipeline in the logs is "failed" diff --git a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature index fca0b3981..d6ad85cd4 100644 --- a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature @@ -296,7 +296,7 @@ Feature: Oracle - Verify data transfer from Oracle source to BigQuery sink Then Close the Plugin Properties page Then Save the pipeline Then Preview and run the pipeline - Then Wait till pipeline preview is in running state + Then Wait till pipeline preview is in running state and check if any error occurs Then Verify the preview run status of pipeline in the logs is "failed" @ORACLE_SOURCE_TEST @BQ_SINK_TEST diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index b981faf81..e44f4c00a 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -15,5 +15,7 @@ errorMessageBlankUsername=Username is required when password is given. errorMessageInvalidTableName=Exception while trying to validate schema of database table '"table"' for connection errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"TARGETTABLE_ errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection -errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: ORA-00936: missing expression . \ - Please check the system logs for more details. +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ + java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. +blank.database.message=Required property 'database' has no value. +blank.connection.message=Exception while trying to validate schema of database table diff --git a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties index 6e1929245..f793e3be7 100644 --- a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -18,5 +18,5 @@ errorMessageInvalidSourceHost=SQL error while getting query schema: The connecti errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"targettable_ errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection -errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: The column index is out of range: 1, \ - number of columns: 0.. Please check the system logs for more details. +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'PostgreSQL' encountered : \ + java.io.IOException: The column index is out of range: 1, number of columns: 0.. Please check the system logs for more details. From 858a7cc795fa1cdf375710b715f66a6b1ad7e19a Mon Sep 17 00:00:00 2001 From: Aakash Nayak Date: Mon, 10 Mar 2025 00:39:14 +0530 Subject: [PATCH 37/62] Removing snapshot for 6.11 release --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 4 ++-- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 4 ++-- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 21 insertions(+), 21 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 17aa5e48b..f48bc6ccd 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index df38e7267..ddb3b4e9b 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index cb803d1ba..3154001e8 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 44dd1fe8c..95f278118 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 CloudSQL MySQL plugin @@ -45,7 +45,7 @@ io.cdap.plugin mysql-plugin - 1.12.0-SNAPSHOT + 1.12.0 diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 8faab79ba..3357deb03 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 683dd2f43..33d30657e 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 39c3fcd52..0c059f9bf 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index b823356d8..2f6c83d2b 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index d8a78cd4d..a8c1f88d0 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 7ece99f31..c7cbbdbf2 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index c9dbaf035..f98bc9bbd 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index f5fc81a93..8a8316f13 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index bb5caf4c0..53b3e75f0 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index f7b559439..835948cf3 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index b21152fcd..3137b38df 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Oracle plugin diff --git a/pom.xml b/pom.xml index a3dacc4bf..2e40cac37 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.0-SNAPSHOT + 1.12.0 pom Database Plugins Collection of database plugins @@ -724,7 +724,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.4.0-SNAPSHOT + 0.4.0 test diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 8f086e482..e1b2076db 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 26ce2e396..7e8c8cff8 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index c351a9d96..6d311cc52 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 teradata-plugin From a3a0613415bf39dc0c88e84f7827213fe4dbb540 Mon Sep 17 00:00:00 2001 From: Aakash Nayak Date: Thu, 13 Mar 2025 14:54:13 +0530 Subject: [PATCH 38/62] Updating cdap version to 6.11 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2e40cac37..a2985a578 100644 --- a/pom.xml +++ b/pom.xml @@ -61,8 +61,8 @@ true UTF-8 - 6.11.0-SNAPSHOT - 2.13.0-SNAPSHOT + 6.11.0 + 2.13.0 13.0.1 3.3.6 2.2.4 From 27216859b29977ac7f9259ec549bf6ca372690ce Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 2 Apr 2025 10:23:11 +0530 Subject: [PATCH 39/62] Bump 1.12.1-SNAPSHOT --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 4 ++-- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 20 insertions(+), 20 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index f48bc6ccd..f4251bb32 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index ddb3b4e9b..9ca5bef36 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 3154001e8..70e946e3d 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 95f278118..b692902e5 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT CloudSQL MySQL plugin @@ -45,7 +45,7 @@ io.cdap.plugin mysql-plugin - 1.12.0 + ${project.version} diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 3357deb03..1bc542518 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 33d30657e..58812af4d 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 0c059f9bf..0c53ce6a7 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 2f6c83d2b..e30dca17d 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index a8c1f88d0..e229e104b 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index c7cbbdbf2..b69aa03f4 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index f98bc9bbd..06491d4d4 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 8a8316f13..75d967686 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 53b3e75f0..7619593e9 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 835948cf3..232e70336 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 3137b38df..4e9800c7a 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index a2985a578..53f611119 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.0 + 1.12.1-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index e1b2076db..edf87555d 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 7e8c8cff8..7b4382341 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 6d311cc52..ab09612a5 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT teradata-plugin From 30bf808842f9041a4eb175dcb494dc9b6fe0365b Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 2 Apr 2025 09:38:06 +0530 Subject: [PATCH 40/62] Bumped to 2.13.1-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 53f611119..c58fec1f3 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ UTF-8 6.11.0 - 2.13.0 + 2.13.1-SNAPSHOT 13.0.1 3.3.6 2.2.4 From 525cf2e8d9e57389452a750f6b69b92a2a4b10c7 Mon Sep 17 00:00:00 2001 From: psainics Date: Tue, 18 Mar 2025 03:49:11 +0530 Subject: [PATCH 41/62] Error Management AbstractDBAction --- .../cdap/plugin/db/action/AbstractDBAction.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java index a2be9cbf0..0eaac3148 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java @@ -16,12 +16,15 @@ package io.cdap.plugin.db.action; +import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.action.Action; import io.cdap.cdap.etl.api.action.ActionContext; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; import java.sql.Driver; +import java.sql.SQLException; /** * Action that runs a db command. @@ -40,7 +43,18 @@ public AbstractDBAction(QueryConfig config, Boolean enableAutoCommit) { public void run(ActionContext context) throws Exception { Class driverClass = context.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); - executeQuery.run(); + try { + executeQuery.run(); + } catch (Exception e) { + if (e instanceof SQLException) { + DBErrorDetailsProvider dbe = new DBErrorDetailsProvider(); + throw dbe.getProgramFailureException((SQLException) e, null); + } + FailureCollector collector = context.getFailureCollector(); + collector.addFailure("Failed to execute query with message: " + e.getMessage(), null) + .withStacktrace(e.getStackTrace()); + collector.getOrThrowException(); + } } @Override From adf1d235a1d645556fbeaae8d020797343e0a0a5 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 14 Apr 2025 16:16:58 +0530 Subject: [PATCH 42/62] Add MariadbDBRecord implementations UPDATE DOCS --- mariadb-plugin/docs/Mariadb-batchsink.md | 68 +++++++++--------- mariadb-plugin/docs/Mariadb-batchsource.md | 70 +++++++++---------- mariadb-plugin/pom.xml | 5 ++ .../cdap/plugin/mariadb/MariadbDBRecord.java | 40 +++++++++++ .../plugin/mariadb/MariadbSchemaReader.java | 36 ++++++++++ .../io/cdap/plugin/mariadb/MariadbSink.java | 15 ++++ .../io/cdap/plugin/mariadb/MariadbSource.java | 46 ++++++++++++ 7 files changed, 208 insertions(+), 72 deletions(-) create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java diff --git a/mariadb-plugin/docs/Mariadb-batchsink.md b/mariadb-plugin/docs/Mariadb-batchsink.md index 11176c0db..e4541fe67 100644 --- a/mariadb-plugin/docs/Mariadb-batchsink.md +++ b/mariadb-plugin/docs/Mariadb-batchsink.md @@ -60,41 +60,39 @@ connections. Data Types Mapping ---------- - +--------------------------------+-----------------------+------------------------------------+ - | MariaDB Data Type | CDAP Schema Data Type | Comment | - +--------------------------------+-----------------------+------------------------------------+ - | TINYINT | int | | - | BOOLEAN, BOOL | boolean | | - | SMALLINT | int | | - | MEDIUMINT | int | | - | INT, INTEGER | int | | - | BIGINT | long | | - | DECIMAL, DEC, NUMERIC, FIXED | decimal | | - | FLOAT | float | | - | DOUBLE, DOUBLE PRECISION, REAL | decimal | | - | BIT | boolean | | - | CHAR | string | | - | VARCHAR | string | | - | BINARY | bytes | | - | CHAR BYTE | bytes | | - | VARBINARY | bytes | | - | TINYBLOB | bytes | | - | BLOB | bytes | | - | MEDIUMBLOB | bytes | | - | LONGBLOB | bytes | | - | TINYTEXT | string | | - | TEXT | string | | - | MEDIUMTEXT | string | | - | LONGTEXT | string | | - | JSON | string | In MariaDB it is alias to LONGTEXT | - | ENUM | string | Mapping to String by default | - | SET | string | | - | DATE | date | | - | TIME | time_micros | | - | DATETIME | timestamp_micros | | - | TIMESTAMP | timestamp_micros | | - | YEAR | date | | - +--------------------------------+-----------------------+------------------------------------+ + | MariaDB Data Type | CDAP Schema Data Type | Comment | + |--------------------------------|-----------------------|---------------------------------------------------------| + | TINYINT | int | | + | BOOLEAN, BOOL | boolean | | + | SMALLINT | int | | + | MEDIUMINT | int | | + | INT, INTEGER | int | | + | BIGINT | long | | + | DECIMAL, DEC, NUMERIC, FIXED | decimal | | + | FLOAT | float | | + | DOUBLE, DOUBLE PRECISION, REAL | decimal | | + | BIT | boolean | | + | CHAR | string | | + | VARCHAR | string | | + | BINARY | bytes | | + | CHAR BYTE | bytes | | + | VARBINARY | bytes | | + | TINYBLOB | bytes | | + | BLOB | bytes | | + | MEDIUMBLOB | bytes | | + | LONGBLOB | bytes | | + | TINYTEXT | string | | + | TEXT | string | | + | MEDIUMTEXT | string | | + | LONGTEXT | string | | + | JSON | string | In MariaDB it is alias to LONGTEXT | + | ENUM | string | Mapping to String by default | + | SET | string | | + | DATE | date | | + | TIME | time_micros | | + | DATETIME | timestamp_micros | | + | TIMESTAMP | timestamp_micros | | + | YEAR | int | Users can manually set output schema to map it to Date. | Example ------- diff --git a/mariadb-plugin/docs/Mariadb-batchsource.md b/mariadb-plugin/docs/Mariadb-batchsource.md index 2b1fe3944..713af2ee8 100644 --- a/mariadb-plugin/docs/Mariadb-batchsource.md +++ b/mariadb-plugin/docs/Mariadb-batchsource.md @@ -78,43 +78,39 @@ with the tradeoff of higher memory usage. Data Types Mapping ---------- - - +--------------------------------+-----------------------+------------------------------------+ - | MariaDB Data Type | CDAP Schema Data Type | Comment | - +--------------------------------+-----------------------+------------------------------------+ - | TINYINT | int | | - | BOOLEAN, BOOL | boolean | | - | SMALLINT | int | | - | MEDIUMINT | int | | - | INT, INTEGER | int | | - | BIGINT | long | | - | DECIMAL, DEC, NUMERIC, FIXED | decimal | | - | FLOAT | float | | - | DOUBLE, DOUBLE PRECISION, REAL | decimal | | - | BIT | boolean | | - | CHAR | string | | - | VARCHAR | string | | - | BINARY | bytes | | - | CHAR BYTE | bytes | | - | VARBINARY | bytes | | - | TINYBLOB | bytes | | - | BLOB | bytes | | - | MEDIUMBLOB | bytes | | - | LONGBLOB | bytes | | - | TINYTEXT | string | | - | TEXT | string | | - | MEDIUMTEXT | string | | - | LONGTEXT | string | | - | JSON | string | In MariaDB it is alias to LONGTEXT | - | ENUM | string | Mapping to String by default | - | SET | string | | - | DATE | date | | - | TIME | time_micros | | - | DATETIME | timestamp_micros | | - | TIMESTAMP | timestamp_micros | | - | YEAR | date | | - +--------------------------------+-----------------------+------------------------------------+ - + | MariaDB Data Type | CDAP Schema Data Type | Comment | + |--------------------------------|-----------------------|---------------------------------------------------------| + | TINYINT | int | | + | BOOLEAN, BOOL | boolean | | + | SMALLINT | int | | + | MEDIUMINT | int | | + | INT, INTEGER | int | | + | BIGINT | long | | + | DECIMAL, DEC, NUMERIC, FIXED | decimal | | + | FLOAT | float | | + | DOUBLE, DOUBLE PRECISION, REAL | decimal | | + | BIT | boolean | | + | CHAR | string | | + | VARCHAR | string | | + | BINARY | bytes | | + | CHAR BYTE | bytes | | + | VARBINARY | bytes | | + | TINYBLOB | bytes | | + | BLOB | bytes | | + | MEDIUMBLOB | bytes | | + | LONGBLOB | bytes | | + | TINYTEXT | string | | + | TEXT | string | | + | MEDIUMTEXT | string | | + | LONGTEXT | string | | + | JSON | string | In MariaDB it is alias to LONGTEXT | + | ENUM | string | Mapping to String by default | + | SET | string | | + | DATE | date | | + | TIME | time_micros | | + | DATETIME | timestamp_micros | | + | TIMESTAMP | timestamp_micros | | + | YEAR | int | Users can manually set output schema to map it to Date. | Example ------ diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index b69aa03f4..a387ee041 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -83,6 +83,11 @@ RELEASE compile + + io.cdap.plugin + mysql-plugin + ${project.version} + diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java new file mode 100644 index 000000000..94498c787 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.plugin.db.ColumnType; +import io.cdap.plugin.mysql.MysqlDBRecord; +import java.util.List; + +/** + * Writable class for MariaDB Source/Sink. + */ +public class MariadbDBRecord extends MysqlDBRecord { + + /** + * Used in map-reduce. Do not remove. + */ + @SuppressWarnings("unused") + public MariadbDBRecord() { + // Required by Hadoop DBRecordReader to create an instance + } + + public MariadbDBRecord(StructuredRecord record, List columnTypes) { + super(record, columnTypes); + } +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java new file mode 100644 index 000000000..37ac12a93 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + + +import io.cdap.plugin.mysql.MysqlSchemaReader; +import java.util.Map; + +/** + * Schema reader for mapping Maria DB type + */ +public class MariadbSchemaReader extends MysqlSchemaReader { + + public MariadbSchemaReader (String sessionID) { + super(sessionID); + } + + public MariadbSchemaReader (String sessionID, Map connectionArguments) { + super(sessionID, connectionArguments); + } + +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index ab20f3c5d..9dbf0c7d4 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -19,10 +19,14 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.etl.api.batch.BatchSink; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSinkConfig; import io.cdap.plugin.db.sink.AbstractDBSink; +import io.cdap.plugin.mysql.MysqlDBRecord; import java.util.Map; import javax.annotation.Nullable; @@ -45,6 +49,17 @@ public MariadbSink(MariadbSinkConfig mariadbSinkConfig) { this.mariadbSinkConfig = mariadbSinkConfig; } + @Override + protected DBRecord getDBRecord(StructuredRecord output) { + return new MariadbDBRecord(output, columnTypes); + } + + @Override + protected SchemaReader getSchemaReader() { + return new MariadbSchemaReader(null); + } + + /** * MariaDB Sink Config. */ diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index d5ffcb290..4a4d689bb 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -20,9 +20,16 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.Asset; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.DBUtils; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -53,10 +60,36 @@ protected String createConnectionString() { mariadbSourceConfig.host, mariadbSourceConfig.port, mariadbSourceConfig.database); } + @Override + protected Class getDBRecordType() { + return MariadbDBRecord.class; + } + + @Override + protected LineageRecorder getLineageRecorder(BatchSourceContext context) { + String fqn = DBUtils.constructFQN("mariadb", + mariadbSourceConfig.host, + mariadbSourceConfig.port, + mariadbSourceConfig.database, + mariadbSourceConfig.getReferenceName()); + Asset asset = Asset.builder(mariadbSourceConfig.getReferenceName()).setFqn(fqn).build(); + return new LineageRecorder(context, asset); + } + + @Override + protected SchemaReader getSchemaReader() { + return new MariadbSchemaReader(null, mariadbSourceConfig.getConnectionArguments()); + } + /** * MaraiDB source mariadbSourceConfig. */ public static class MariadbSourceConfig extends DBSpecificSourceConfig { + private static final String JDBC_PROPERTY_CONNECT_TIMEOUT = "connectTimeout"; + private static final String JDBC_PROPERTY_SOCKET_TIMEOUT = "socketTimeout"; + private static final String JDBC_REWRITE_BATCHED_STATEMENTS = "rewriteBatchedStatements"; + + private static final String MARIADB_TINYINT1_IS_BIT = "tinyInt1isBit"; @Name(MariadbConstants.AUTO_RECONNECT) @Description("Should the driver try to re-establish stale and/or dead connections") @@ -116,5 +149,18 @@ public Map getDBSpecificArguments() { public List getInitQueries() { return MariadbUtil.composeDbInitQueries(useAnsiQuotes); } + + @Override + public Map getConnectionArguments() { + Map arguments = new HashMap<>(super.getConnectionArguments()); + // the unit below is millisecond + arguments.putIfAbsent(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); + arguments.putIfAbsent(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); + arguments.putIfAbsent(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); + // MariaDB property to ensure that TINYINT(1) type data is not converted to MariaDB Bit/Boolean type in the + // ResultSet. + arguments.putIfAbsent(MARIADB_TINYINT1_IS_BIT, "false"); + return arguments; + } } } From 0805c0587425f86b9d99f56d19ad49275250a2ae Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 11 Apr 2025 09:53:41 +0530 Subject: [PATCH 43/62] Add MariadbErrorDetailsProvider --- .../java/io/cdap/plugin/util/DBUtils.java | 1 + .../mariadb/MariadbErrorDetailsProvider.java | 33 +++++++++++++++++++ .../io/cdap/plugin/mariadb/MariadbSink.java | 12 ++++++- .../io/cdap/plugin/mariadb/MariadbSource.java | 10 ++++++ 4 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index 34043d513..b125a7214 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -61,6 +61,7 @@ public final class DBUtils { public static final Calendar PURE_GREGORIAN_CALENDAR = createPureGregorianCalender(); public static final String MYSQL_SUPPORTED_DOC_URL = "https://dev.mysql.com/doc/mysql-errors/9.0/en/"; + public static final String MARIADB_SUPPORTED_DOC_URL = "https://mariadb.com/kb/en/mariadb-error-codes/"; public static final String MSSQL_SUPPORTED_DOC_URL = "https://docs.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors"; public static final String CLOUDSQLMYSQL_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/mysql/error-messages"; diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java new file mode 100644 index 000000000..38405225d --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + + +import io.cdap.plugin.mysql.MysqlErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for MariaDb plugins. + */ +public class MariadbErrorDetailsProvider extends MysqlErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MARIADB_SUPPORTED_DOC_URL; + } + +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index 9dbf0c7d4..589da2953 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -25,8 +25,8 @@ import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSinkConfig; import io.cdap.plugin.db.sink.AbstractDBSink; +import io.cdap.plugin.util.DBUtils; -import io.cdap.plugin.mysql.MysqlDBRecord; import java.util.Map; import javax.annotation.Nullable; @@ -60,6 +60,16 @@ protected SchemaReader getSchemaReader() { } + @Override + protected String getErrorDetailsProviderClassName() { + return MariadbErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MARIADB_SUPPORTED_DOC_URL; + } + /** * MariaDB Sink Config. */ diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index 4a4d689bb..098145ab9 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -81,6 +81,16 @@ protected SchemaReader getSchemaReader() { return new MariadbSchemaReader(null, mariadbSourceConfig.getConnectionArguments()); } + @Override + protected String getErrorDetailsProviderClassName() { + return MariadbErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MARIADB_SUPPORTED_DOC_URL; + } + /** * MaraiDB source mariadbSourceConfig. */ From 8777c32c5fe56c52db26b96245f81483799d9b25 Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 16 Apr 2025 14:12:00 +0530 Subject: [PATCH 44/62] Remove snapshot to release 1.12.1 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index f4251bb32..2eab54191 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 9ca5bef36..8648ca695 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 70e946e3d..fee4ef048 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index b692902e5..b2f41ed7f 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 1bc542518..c5d9997c1 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 58812af4d..7cc5f1622 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 0c53ce6a7..8018b783a 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index e30dca17d..ba8dd82f5 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index e229e104b..c5e73943f 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index a387ee041..b080138db 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 06491d4d4..3b193c0bd 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 75d967686..72b20d78e 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 7619593e9..08d4fc015 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 232e70336..e884e6d23 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 4e9800c7a..378521afd 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Oracle plugin diff --git a/pom.xml b/pom.xml index c58fec1f3..f52f6bb18 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.1-SNAPSHOT + 1.12.1 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index edf87555d..5d45cfa40 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 7b4382341..57083ad10 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index ab09612a5..f40f39a42 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 teradata-plugin From cf99b4db7b299ab2e04ea70785b61563bda50711 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 5 May 2025 15:45:14 +0530 Subject: [PATCH 45/62] Bump version to 1.12.2-SNAPSHOT in pom.xml for all database plugins --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 2eab54191..d1a9c0a96 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 8648ca695..65c6b693f 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index fee4ef048..39e0cb9dc 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index b2f41ed7f..f5798c468 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index c5d9997c1..8ee68aada 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 7cc5f1622..778524126 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 8018b783a..0ef1249cd 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index ba8dd82f5..124260892 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index c5e73943f..5d2d8cc5a 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index b080138db..e82c7db1c 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 3b193c0bd..cd1c9fe57 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 72b20d78e..c00a12e44 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 08d4fc015..bacdd9567 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index e884e6d23..fedd84455 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 378521afd..891b78537 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index f52f6bb18..e8edf4027 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.1 + 1.12.2-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 5d45cfa40..5092392e4 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 57083ad10..2313be26e 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index f40f39a42..0c32586c6 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT teradata-plugin From 1c0c4b689939700fb96c9868cb037e8009f9ab91 Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 2 May 2025 08:33:23 +0530 Subject: [PATCH 46/62] Skip field validation for compatiblity --- .../plugin/db/source/AbstractDBSource.java | 24 ++++++++++------- .../db/source/AbstractDBSourceTest.java | 12 ++++++--- .../mariadb/MariadbFieldsValidator.java | 25 +++++++++++++++++ .../io/cdap/plugin/mariadb/MariadbSink.java | 7 +++++ .../io/cdap/plugin/mariadb/MariadbSource.java | 27 +++++++++++++++++++ 5 files changed, 83 insertions(+), 12 deletions(-) create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 3fd490ada..54d1e2ab6 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -529,7 +529,7 @@ public void validateSchema(Schema actualSchema, FailureCollector collector) { } @VisibleForTesting - static void validateSchema(Schema actualSchema, Schema configSchema, FailureCollector collector) { + void validateSchema(Schema actualSchema, Schema configSchema, FailureCollector collector) { if (configSchema == null) { collector.addFailure("Schema should not be null or empty.", null) .withConfigProperty(SCHEMA); @@ -550,14 +550,20 @@ static void validateSchema(Schema actualSchema, Schema configSchema, FailureColl Schema expectedFieldSchema = field.getSchema().isNullable() ? field.getSchema().getNonNullable() : field.getSchema(); - if (actualFieldSchema.getType() != expectedFieldSchema.getType() || - actualFieldSchema.getLogicalType() != expectedFieldSchema.getLogicalType()) { - collector.addFailure( - String.format("Schema field '%s' has type '%s but found '%s'.", - field.getName(), expectedFieldSchema.getDisplayName(), - actualFieldSchema.getDisplayName()), null) - .withOutputSchemaField(field.getName()); - } + validateField(collector, field, actualFieldSchema, expectedFieldSchema); + } + } + + protected void validateField(FailureCollector collector, Schema.Field field, Schema actualFieldSchema, + Schema expectedFieldSchema) { + if (actualFieldSchema.getType() != expectedFieldSchema.getType() || + actualFieldSchema.getLogicalType() != expectedFieldSchema.getLogicalType()) { + collector.addFailure( + String.format("Schema field '%s' is expected to have type '%s but found '%s'.", field.getName(), + expectedFieldSchema.getDisplayName(), actualFieldSchema.getDisplayName()), + String.format("Change the data type of field %s to %s.", field.getName(), + actualFieldSchema.getDisplayName())) + .withOutputSchemaField(field.getName()); } } diff --git a/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java b/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java index 3dc7a2d1c..a8be38b46 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java @@ -43,11 +43,17 @@ public class AbstractDBSourceTest { Schema.Field.of("double_column", Schema.nullableOf(Schema.of(Schema.Type.DOUBLE))), Schema.Field.of("boolean_column", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))) ); + private static final AbstractDBSource.DBSourceConfig TEST_CONFIG = new AbstractDBSource.DBSourceConfig() { + @Override + public String getConnectionString() { + return ""; + } + }; @Test public void testValidateSourceSchemaCorrectSchema() { MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(SCHEMA, SCHEMA, collector); + TEST_CONFIG.validateSchema(SCHEMA, SCHEMA, collector); Assert.assertEquals(0, collector.getValidationFailures().size()); } @@ -65,7 +71,7 @@ public void testValidateSourceSchemaMismatchFields() { ); MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(actualSchema, SCHEMA, collector); + TEST_CONFIG.validateSchema(actualSchema, SCHEMA, collector); assertPropertyValidationFailed(collector, "boolean_column"); } @@ -84,7 +90,7 @@ public void testValidateSourceSchemaInvalidFieldType() { ); MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(actualSchema, SCHEMA, collector); + TEST_CONFIG.validateSchema(actualSchema, SCHEMA, collector); assertPropertyValidationFailed(collector, "boolean_column"); } diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java new file mode 100644 index 000000000..71ccb0d06 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java @@ -0,0 +1,25 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + +import io.cdap.plugin.mysql.MysqlFieldsValidator; + +/** + * Field validator for maraidb + */ +public class MariadbFieldsValidator extends MysqlFieldsValidator { +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index 589da2953..52a73344a 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -25,6 +25,8 @@ import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSinkConfig; import io.cdap.plugin.db.sink.AbstractDBSink; +import io.cdap.plugin.db.sink.FieldsValidator; +import io.cdap.plugin.mysql.MysqlFieldsValidator; import io.cdap.plugin.util.DBUtils; import java.util.Map; @@ -70,6 +72,11 @@ protected String getExternalDocumentationLink() { return DBUtils.MARIADB_SUPPORTED_DOC_URL; } + @Override + protected FieldsValidator getFieldsValidator() { + return new MariadbFieldsValidator(); + } + /** * MariaDB Sink Config. */ diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index 098145ab9..28204100c 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -19,6 +19,8 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.plugin.common.Asset; @@ -172,5 +174,30 @@ public Map getConnectionArguments() { arguments.putIfAbsent(MARIADB_TINYINT1_IS_BIT, "false"); return arguments; } + + @Override + protected void validateField(FailureCollector collector, + Schema.Field field, + Schema actualFieldSchema, + Schema expectedFieldSchema) { + // Backward compatibility changes to support MySQL YEAR to Date type conversion + if (Schema.LogicalType.DATE.equals(expectedFieldSchema.getLogicalType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + + // Backward compatibility change to support MySQL MEDIUMINT UNSIGNED to Long type conversion + if (Schema.Type.LONG.equals(expectedFieldSchema.getType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + + // Backward compatibility change to support MySQL TINYINT(1) to Bool type conversion + if (Schema.Type.BOOLEAN.equals(expectedFieldSchema.getType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + super.validateField(collector, field, actualFieldSchema, expectedFieldSchema); + } } } From 9e3be1de5e950e79368a0f2ff528ca6e1dbb1f1a Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 21 Apr 2025 15:27:42 +0530 Subject: [PATCH 47/62] Set TRANSACTION_ISOLATION_LEVEL config in MySQL, PostgreSQL & MSSQL plugins Add transaction isolation level to PostgreSQL, MYSQL and MSSQL Plugins. --- .../io/cdap/plugin/db/ConnectionConfig.java | 1 + .../AbstractDBSpecificConnectorConfig.java | 26 ++++++++++++++++++- mssql-plugin/docs/SQL Server-connector.md | 8 ++++++ mssql-plugin/docs/SqlServer-batchsink.md | 8 ++++++ mssql-plugin/docs/SqlServer-batchsource.md | 8 ++++++ .../io/cdap/plugin/mssql/SqlServerSink.java | 5 ++++ .../io/cdap/plugin/mssql/SqlServerSource.java | 5 ++++ .../widgets/SQL Server-connector.json | 14 ++++++++++ mssql-plugin/widgets/SqlServer-batchsink.json | 18 +++++++++++++ .../widgets/SqlServer-batchsource.json | 18 +++++++++++++ mysql-plugin/docs/MySQL-connector.md | 8 ++++++ mysql-plugin/docs/Mysql-batchsink.md | 8 ++++++ mysql-plugin/docs/Mysql-batchsource.md | 8 ++++++ .../java/io/cdap/plugin/mysql/MysqlSink.java | 5 ++++ .../io/cdap/plugin/mysql/MysqlSource.java | 5 ++++ mysql-plugin/widgets/MySQL-connector.json | 14 ++++++++++ mysql-plugin/widgets/Mysql-batchsink.json | 18 +++++++++++++ mysql-plugin/widgets/Mysql-batchsource.json | 18 +++++++++++++ .../plugin/oracle/OracleConnectorConfig.java | 18 ++----------- .../docs/PostgreSQL-connector.md | 8 ++++++ postgresql-plugin/docs/Postgres-batchsink.md | 8 ++++++ .../docs/Postgres-batchsource.md | 8 ++++++ .../io/cdap/plugin/postgres/PostgresSink.java | 5 ++++ .../cdap/plugin/postgres/PostgresSource.java | 5 ++++ .../widgets/PostgreSQL-connector.json | 13 ++++++++++ .../widgets/Postgres-batchsink.json | 17 ++++++++++++ .../widgets/Postgres-batchsource.json | 17 ++++++++++++ 27 files changed, 277 insertions(+), 17 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index 588ed78b8..c5320e25e 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -45,6 +45,7 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String CONNECTION_ARGUMENTS = "connectionArguments"; public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; + public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; @Name(JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index 5c6b08031..8de0e4d70 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -20,8 +20,9 @@ import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.TransactionIsolationLevel; -import java.util.Collections; +import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; @@ -42,6 +43,12 @@ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnec @Nullable protected Integer port; + @Name(ConnectionConfig.TRANSACTION_ISOLATION_LEVEL) + @Description("The transaction isolation level for the database session.") + @Macro + @Nullable + protected String transactionIsolationLevel; + public String getHost() { return host; } @@ -55,4 +62,21 @@ public int getPort() { public boolean canConnect() { return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && !containsMacro(ConnectionConfig.PORT); } + + @Override + public Map getAdditionalArguments() { + Map additonalArguments = new HashMap<>(); + if (getTransactionIsolationLevel() != null) { + additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); + } + return additonalArguments; + } + + public String getTransactionIsolationLevel() { + if (transactionIsolationLevel == null) { + return null; + } + return TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); + } } + diff --git a/mssql-plugin/docs/SQL Server-connector.md b/mssql-plugin/docs/SQL Server-connector.md index cb72161f5..6f0038715 100644 --- a/mssql-plugin/docs/SQL Server-connector.md +++ b/mssql-plugin/docs/SQL Server-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Authentication Type:** Indicates which authentication method will be used for the connection. Use 'SQL Login'. to connect to a SQL Server using username and password properties. Use 'Active Directory Password' to connect to an Azure SQL Database/Data Warehouse using an Azure AD principal name and password. diff --git a/mssql-plugin/docs/SqlServer-batchsink.md b/mssql-plugin/docs/SqlServer-batchsink.md index 5d10b4bb6..b4ca1cbc5 100644 --- a/mssql-plugin/docs/SqlServer-batchsink.md +++ b/mssql-plugin/docs/SqlServer-batchsink.md @@ -46,6 +46,14 @@ an Azure SQL Database/Data Warehouse using an Azure AD principal name and passwo **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Instance Name:** SQL Server instance name to connect to. When it is not specified, a connection is made to the default instance. For the case where both the instanceName and port are specified, see the notes for port. If you specify a Virtual Network Name in the Server connection property, you cannot diff --git a/mssql-plugin/docs/SqlServer-batchsource.md b/mssql-plugin/docs/SqlServer-batchsource.md index c8e30f77e..5c917621c 100644 --- a/mssql-plugin/docs/SqlServer-batchsource.md +++ b/mssql-plugin/docs/SqlServer-batchsource.md @@ -56,6 +56,14 @@ an Azure SQL Database/Data Warehouse using an Azure AD principal name and passwo **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Instance Name:** SQL Server instance name to connect to. When it is not specified, a connection is made to the default instance. For the case where both the instanceName and port are specified, see the notes for port. If you specify a Virtual Network Name in the Server connection property, you cannot diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index 223785f6c..dc442d200 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -177,6 +177,11 @@ public Map getDBSpecificArguments() { packetSize, queryTimeout); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public String getConnectionString() { return String.format(SqlServerConstants.SQL_SERVER_CONNECTION_STRING_FORMAT, diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java index 01dd6c9ca..004532064 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java @@ -198,6 +198,11 @@ public List getInitQueries() { return Collections.emptyList(); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/mssql-plugin/widgets/SQL Server-connector.json b/mssql-plugin/widgets/SQL Server-connector.json index 171076295..c326cd81d 100644 --- a/mssql-plugin/widgets/SQL Server-connector.json +++ b/mssql-plugin/widgets/SQL Server-connector.json @@ -64,6 +64,20 @@ "widget-type": "password", "label": "Password", "name": "password" + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } } ] }, diff --git a/mssql-plugin/widgets/SqlServer-batchsink.json b/mssql-plugin/widgets/SqlServer-batchsink.json index 260c66259..fb20cad9d 100644 --- a/mssql-plugin/widgets/SqlServer-batchsink.json +++ b/mssql-plugin/widgets/SqlServer-batchsink.json @@ -84,6 +84,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -280,6 +294,10 @@ { "type": "property", "name": "connectionArguments" + }, + { + "type": "property", + "name": "transactionIsolationLevel" } ] }, diff --git a/mssql-plugin/widgets/SqlServer-batchsource.json b/mssql-plugin/widgets/SqlServer-batchsource.json index dad5f4708..b3494e485 100644 --- a/mssql-plugin/widgets/SqlServer-batchsource.json +++ b/mssql-plugin/widgets/SqlServer-batchsource.json @@ -84,6 +84,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -316,6 +330,10 @@ { "type": "property", "name": "connectionArguments" + }, + { + "type": "property", + "name": "transactionIsolationLevel" } ] }, diff --git a/mysql-plugin/docs/MySQL-connector.md b/mysql-plugin/docs/MySQL-connector.md index fb5c1fbb8..f586084c1 100644 --- a/mysql-plugin/docs/MySQL-connector.md +++ b/mysql-plugin/docs/MySQL-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments will be passed to the JDBC driver, as connection arguments, for JDBC drivers that may need additional configurations. This is a semicolon-separated list of key-value pairs, where each pair is separated by a equals '=' and specifies diff --git a/mysql-plugin/docs/Mysql-batchsink.md b/mysql-plugin/docs/Mysql-batchsink.md index b28a28618..46a763f9d 100644 --- a/mysql-plugin/docs/Mysql-batchsink.md +++ b/mysql-plugin/docs/Mysql-batchsink.md @@ -39,6 +39,14 @@ You also can use the macro function ${conn(connection-name)}. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/docs/Mysql-batchsource.md b/mysql-plugin/docs/Mysql-batchsource.md index 010e08216..552bb5504 100644 --- a/mysql-plugin/docs/Mysql-batchsource.md +++ b/mysql-plugin/docs/Mysql-batchsource.md @@ -49,6 +49,14 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index fee0a31fa..0a9257a0a 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -194,6 +194,11 @@ public Map getDBSpecificArguments() { trustCertificateKeyStorePassword, false); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public MysqlConnectorConfig getConnection() { return connection; diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index 971b76809..38642468c 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -197,6 +197,11 @@ public MysqlConnectorConfig getConnection() { return connection; } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/mysql-plugin/widgets/MySQL-connector.json b/mysql-plugin/widgets/MySQL-connector.json index 9064d1bf6..f60f5526f 100644 --- a/mysql-plugin/widgets/MySQL-connector.json +++ b/mysql-plugin/widgets/MySQL-connector.json @@ -30,6 +30,20 @@ "widget-attributes": { "default": "3306" } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } } ] }, diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index c525ead40..58596aae2 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -65,6 +65,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -225,6 +239,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "host" diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index 9175bd5ed..506e837f7 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -65,6 +65,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -277,6 +291,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "host" diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 10022364a..c3ce051e5 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -81,12 +81,6 @@ public String getConnectionString() { @Macro private String database; - @Name(OracleConstants.TRANSACTION_ISOLATION_LEVEL) - @Description("The transaction isolation level for the database session.") - @Macro - @Nullable - private String transactionIsolationLevel; - @Name(OracleConstants.USE_SSL) @Description("Turns on SSL encryption. Connection will fail if SSL is not available") @Nullable @@ -124,6 +118,7 @@ public Properties getConnectionArgumentsProperties() { return prop; } + @Override public String getTransactionIsolationLevel() { //if null default to the highest isolation level possible if (transactionIsolationLevel == null) { @@ -133,16 +128,7 @@ public String getTransactionIsolationLevel() { //This ensures that the role is mapped to the right serialization level, even w/ incorrect user input //if role is SYSDBA or SYSOP it will map to read_committed. else serialized return (!getRole().equals(ROLE_NORMAL)) ? TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name() : - TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); - } - - @Override - public Map getAdditionalArguments() { - Map additonalArguments = new HashMap<>(); - if (getTransactionIsolationLevel() != null) { - additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); - } - return additonalArguments; + TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); } @Override diff --git a/postgresql-plugin/docs/PostgreSQL-connector.md b/postgresql-plugin/docs/PostgreSQL-connector.md index 739c678e3..fe442cbf1 100644 --- a/postgresql-plugin/docs/PostgreSQL-connector.md +++ b/postgresql-plugin/docs/PostgreSQL-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Database:** The name of the database to connect to. **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments diff --git a/postgresql-plugin/docs/Postgres-batchsink.md b/postgresql-plugin/docs/Postgres-batchsink.md index b8a996463..82065e0fd 100644 --- a/postgresql-plugin/docs/Postgres-batchsink.md +++ b/postgresql-plugin/docs/Postgres-batchsink.md @@ -39,6 +39,14 @@ You also can use the macro function ${conn(connection-name)}. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/postgresql-plugin/docs/Postgres-batchsource.md b/postgresql-plugin/docs/Postgres-batchsource.md index af359022d..559723526 100644 --- a/postgresql-plugin/docs/Postgres-batchsource.md +++ b/postgresql-plugin/docs/Postgres-batchsource.md @@ -49,6 +49,14 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index 7682b8b0b..73430c1e2 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -175,6 +175,11 @@ public Map getDBSpecificArguments() { return ImmutableMap.of(PostgresConstants.CONNECTION_TIMEOUT, String.valueOf(connectionTimeout)); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override protected PostgresConnectorConfig getConnection() { return connection; diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index 8e3c091f9..b230f3d1e 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -143,6 +143,11 @@ protected PostgresConnectorConfig getConnection() { return connection; } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/postgresql-plugin/widgets/PostgreSQL-connector.json b/postgresql-plugin/widgets/PostgreSQL-connector.json index 091afc972..9a7a02e14 100644 --- a/postgresql-plugin/widgets/PostgreSQL-connector.json +++ b/postgresql-plugin/widgets/PostgreSQL-connector.json @@ -31,6 +31,19 @@ "default": "5432" } }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "textbox", "label": "Database", diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index 6aa2dad8a..14e6f8154 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -65,6 +65,19 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -186,6 +199,10 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "database" diff --git a/postgresql-plugin/widgets/Postgres-batchsource.json b/postgresql-plugin/widgets/Postgres-batchsource.json index 0e4ba28c1..60de4725f 100644 --- a/postgresql-plugin/widgets/Postgres-batchsource.json +++ b/postgresql-plugin/widgets/Postgres-batchsource.json @@ -65,6 +65,19 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -206,6 +219,10 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "database" From 829e0e7a58ad5345d5c176f2a9edaad885ceef79 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Wed, 23 Apr 2025 20:02:59 +0530 Subject: [PATCH 48/62] Added changes for New committer to have commit/rollback in commit/abort task methods. --- .../cdap/plugin/db/sink/AbstractDBSink.java | 1 + .../plugin/db/sink/ETLDBOutputFormat.java | 97 +++++++++++++++++-- 2 files changed, 90 insertions(+), 8 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 797abfc23..0bb4bf123 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -228,6 +228,7 @@ public void prepareRun(BatchSinkContext context) { configAccessor.setInitQueries(dbSinkConfig.getInitQueries()); configAccessor.getConfiguration().set(DBConfiguration.DRIVER_CLASS_PROPERTY, driverClass.getName()); configAccessor.getConfiguration().set(DBConfiguration.URL_PROPERTY, connectionString); + configAccessor.getConfiguration().set(ETLDBOutputFormat.STAGE_NAME, context.getStageName()); String fullyQualifiedTableName = dbSchemaName == null ? dbSinkConfig.getEscapedTableName() : dbSinkConfig.getEscapedDbSchemaName() + "." + dbSinkConfig.getEscapedTableName(); configAccessor.getConfiguration().set(DBConfiguration.OUTPUT_TABLE_NAME_PROPERTY, fullyQualifiedTableName); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java index ad2b91ab1..ad196386c 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java @@ -25,6 +25,8 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.util.DBUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; @@ -43,6 +45,7 @@ import java.sql.Statement; import java.util.Map; import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; import static io.cdap.plugin.db.ConnectionConfigAccessor.OPERATION_NAME; import static io.cdap.plugin.db.ConnectionConfigAccessor.RELATION_TABLE_KEY; @@ -56,15 +59,92 @@ public class ETLDBOutputFormat extends DBOutputFormat { // Batch size before submitting a batch to the SQL engine. If set to 0, no batches will be submitted until commit. public static final String COMMIT_BATCH_SIZE = "io.cdap.plugin.db.output.commit.batch.size"; + public static final String STAGE_NAME = "io.cdap.plugin.db.output.stage_name"; public static final int DEFAULT_COMMIT_BATCH_SIZE = 1000; private static final Character ESCAPE_CHAR = '"'; + // Format for connection map's key will be "taskAttemptId_stageName" + private static final String CONNECTION_MAP_KEY_FORMAT = "%s_%s"; + + // CONNECTION_MAP will be used to store connections with "taskAttemptId_stageName" as key and + // connection object as value. Making it static to be accessed from multiple task attempts within same executor. + private static final Map CONNECTION_MAP = new ConcurrentHashMap<>(); private static final Logger LOG = LoggerFactory.getLogger(ETLDBOutputFormat.class); private Configuration conf; private Driver driver; private JDBCDriverShim driverShim; + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext context) + throws IOException, InterruptedException { + return new OutputCommitter() { + @Override + public void setupJob(JobContext jobContext) throws IOException { + // do nothing + } + + @Override + public void setupTask(TaskAttemptContext taskContext) throws IOException { + // do nothing + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException { + return true; + } + + @Override + public void commitTask(TaskAttemptContext taskContext) throws IOException { + conf = context.getConfiguration(); + String stageName = conf.get(STAGE_NAME); + String connectionId = getConnectionMapKeyFormat(context.getTaskAttemptID().toString(), stageName); + Connection connection; + if ((connection = CONNECTION_MAP.remove(connectionId)) != null) { + try { + connection.commit(); + } catch (SQLException e) { + try { + connection.rollback(); + } catch (SQLException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + throw new IOException(e); + } finally { + try { + connection.close(); + LOG.debug("Connection Closed after committing the task with taskAttemptId {}", connectionId); + } catch (SQLException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + } + } + } + + @Override + public void abortTask(TaskAttemptContext taskContext) throws IOException { + conf = context.getConfiguration(); + String stageName = conf.get(STAGE_NAME); + String connectionId = getConnectionMapKeyFormat(context.getTaskAttemptID().toString(), stageName); + Connection connection; + if ((connection = CONNECTION_MAP.remove(connectionId)) != null) { + try { + connection.rollback(); + } catch (SQLException e) { + throw new IOException(e); + } finally { + try { + connection.close(); + LOG.debug("Connection Closed after rollback the task with taskAttemptId {}", connectionId); + } catch (SQLException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + } + } + } + }; + } + @Override public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException { conf = context.getConfiguration(); @@ -81,6 +161,11 @@ public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOE try { Connection connection = getConnection(conf); + String stageName = conf.get(STAGE_NAME); + // If using multiple sinks, task attemptID can be same in that case, appending stage in the end for uniqueness. + String connectionId = getConnectionMapKeyFormat(context.getTaskAttemptID().toString(), stageName); + CONNECTION_MAP.put(connectionId, connection); + LOG.debug("Connection Added to the map with connectionId : {}", connectionId); PreparedStatement statement = connection.prepareStatement(constructQueryOnOperation(tableName, fieldNames, operationName, listKeys)); return new DBRecordWriter(connection, statement) { @@ -98,23 +183,15 @@ public void close(TaskAttemptContext context) throws IOException { if (!emptyData) { getStatement().executeBatch(); } - getConnection().commit(); } catch (SQLException e) { - try { - getConnection().rollback(); - } catch (SQLException ex) { - LOG.warn(StringUtils.stringifyException(ex)); - } throw new IOException(e); } finally { try { getStatement().close(); - getConnection().close(); } catch (SQLException ex) { throw new IOException(ex); } } - try { DriverManager.deregisterDriver(driverShim); } catch (SQLException e) { @@ -298,4 +375,8 @@ public String constructUpdateQuery(String table, String[] fieldNames, String[] l return query.toString(); } } + + private String getConnectionMapKeyFormat(String taskAttemptId, String stageName) { + return String.format(CONNECTION_MAP_KEY_FORMAT, taskAttemptId, stageName); + } } From 213bb8d143c5c489afa88d5ecfdd7f81fc64aceb Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 7 May 2025 11:02:26 +0530 Subject: [PATCH 49/62] Update version to 1.12.2 in pom.xml for all database plugins --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index d1a9c0a96..2f3aa906d 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 65c6b693f..58bd6fb9e 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 39e0cb9dc..708d55615 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index f5798c468..58a30b41a 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 8ee68aada..65f8233ec 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 778524126..3ad3e0704 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 0ef1249cd..579d49f96 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 124260892..6009d73e9 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 5d2d8cc5a..1a17ae6b8 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index e82c7db1c..1335d993e 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index cd1c9fe57..bc8bda62e 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index c00a12e44..1bbfe839c 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index bacdd9567..c0e0a9dc2 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index fedd84455..8c6059dde 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 891b78537..5011b2f84 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Oracle plugin diff --git a/pom.xml b/pom.xml index e8edf4027..e7cf3f934 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.2-SNAPSHOT + 1.12.2 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 5092392e4..28a801545 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 2313be26e..6996cd8a0 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 0c32586c6..125871dcd 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 teradata-plugin From a96b7dafd9fd071c6ce32662c4b7d54d36c3cd0f Mon Sep 17 00:00:00 2001 From: sahusanket Date: Tue, 3 Jun 2025 15:08:21 +0530 Subject: [PATCH 50/62] [PLUGIN-1893] Adding fields in Oracle source and connector which acts like a flag for Backward compatibility issues for Timestamp and number (precisionless) --- .../cdap/plugin/oracle/OracleConnector.java | 3 +- .../plugin/oracle/OracleConnectorConfig.java | 28 +++++++++-- .../cdap/plugin/oracle/OracleConstants.java | 2 + .../io/cdap/plugin/oracle/OracleSource.java | 13 ++++-- .../oracle/OracleSourceSchemaReader.java | 44 ++++++++++++------ oracle-plugin/widgets/Oracle-batchsource.json | 46 +++++++++++++++++++ oracle-plugin/widgets/Oracle-connector.json | 38 +++++++++++++++ 7 files changed, 153 insertions(+), 21 deletions(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 3d2f7399a..16371d5c1 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -112,7 +112,8 @@ protected DBConnectorPath getDBConnectorPath(String path) { @Override protected SchemaReader getSchemaReader(String sessionID) { - return new OracleSourceSchemaReader(sessionID); + return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(), + config.getTreatPrecisionlessNumAsDeci()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index c3ce051e5..cbc1e5ed2 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -22,8 +22,6 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.db.connector.AbstractDBSpecificConnectorConfig; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; import javax.annotation.Nullable; @@ -43,12 +41,14 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { - this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null); + this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null, + null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, - String role, Boolean useSSL) { + String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp, + @Nullable Boolean treatPrecisionlessNumAsDeci) { this.host = host; this.port = port; @@ -60,6 +60,8 @@ public OracleConnectorConfig(String host, int port, String user, String password this.database = database; this.role = role; this.useSSL = useSSL; + this.treatAsOldTimestamp = treatAsOldTimestamp; + this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci; } @Override @@ -86,6 +88,16 @@ public String getConnectionString() { @Nullable public Boolean useSSL; + @Name(OracleConstants.TREAT_AS_OLD_TIMESTAMP) + @Description("A hidden field to handle timestamp as CDAP's timestamp micros or string as per old behavior.") + @Nullable + public Boolean treatAsOldTimestamp; + + @Name(OracleConstants.TREAT_PRECISIONLESSNUM_AS_DECI) + @Description("A hidden field to handle precision less number as CDAP's decimal per old behavior.") + @Nullable + public Boolean treatPrecisionlessNumAsDeci; + @Override protected int getDefaultPort() { return 1521; @@ -108,6 +120,14 @@ public Boolean getSSlMode() { return useSSL != null && useSSL; } + public Boolean getTreatAsOldTimestamp() { + return Boolean.TRUE.equals(treatAsOldTimestamp); + } + + public Boolean getTreatPrecisionlessNumAsDeci() { + return Boolean.TRUE.equals(treatPrecisionlessNumAsDeci); + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index dc38f80ac..cbd411175 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -43,6 +43,8 @@ private OracleConstants() { public static final String TNS_CONNECTION_TYPE = "tns"; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; public static final String USE_SSL = "useSSL"; + public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp"; + public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci"; /** * Constructs the Oracle connection string based on the provided connection type, host, port, and database. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 53f75613b..1488a084b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -63,7 +63,12 @@ protected String createConnectionString() { @Override protected SchemaReader getSchemaReader() { - return new OracleSourceSchemaReader(); + // PLUGIN-1893 : Based on field/properties from Oracle source and Oracle connection we will pass the flag to control + // handle schema to make it backward compatible. + boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp(); + boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci(); + + return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci); } @Override @@ -127,9 +132,11 @@ public OracleSourceConfig(String host, int port, String user, String password, S String connectionArguments, String connectionType, String database, String role, int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, - String boundingQuery, String splitBy, Boolean useSSL) { + String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp, + Boolean treatPrecisionlessNumAsDeci) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, - connectionType, database, role, useSSL); + connectionType, database, role, useSSL, treatAsOldTimestamp, + treatPrecisionlessNumAsDeci); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index 7d35f9bc7..dd17d2e84 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -26,6 +26,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.Set; +import javax.annotation.Nullable; /** * Oracle Source schema reader. @@ -65,14 +66,17 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { ); private final String sessionID; + private final Boolean isTimestampOldBehavior; + private final Boolean isPrecisionlessNumAsDecimal; public OracleSourceSchemaReader() { - this(null); + this(null, false, false); } - - public OracleSourceSchemaReader(String sessionID) { - super(); + public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior, + boolean isPrecisionlessNumAsDecimal) { this.sessionID = sessionID; + this.isTimestampOldBehavior = isTimestampOldBehavior; + this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal; } @Override @@ -81,10 +85,12 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti switch (sqlType) { case TIMESTAMP_TZ: - return Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); - case Types.TIMESTAMP: + return isTimestampOldBehavior ? Schema.of(Schema.Type.STRING) : Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); case TIMESTAMP_LTZ: - return Schema.of(Schema.LogicalType.DATETIME); + return isTimestampOldBehavior ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) + : Schema.of(Schema.LogicalType.DATETIME); + case Types.TIMESTAMP: + return isTimestampOldBehavior ? super.getSchema(metadata, index) : Schema.of(Schema.LogicalType.DATETIME); case BINARY_FLOAT: return Schema.of(Schema.Type.FLOAT); case BINARY_DOUBLE: @@ -107,12 +113,24 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti // For a Number type without specified precision and scale, precision will be 0 and scale will be -127 if (precision == 0) { // reference : https://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832 - LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " - + "converting into STRING type to avoid any precision loss.", - metadata.getColumnName(index), - metadata.getColumnTypeName(index), - metadata.getColumnName(index))); - return Schema.of(Schema.Type.STRING); + if (isPrecisionlessNumAsDecimal) { + precision = 38; + scale = 0; + LOG.warn(String.format("%s type with undefined precision and scale is detected, " + + "there may be a precision loss while running the pipeline. " + + "Please define an output precision and scale for field '%s' to avoid " + + "precision loss.", + metadata.getColumnTypeName(index), + metadata.getColumnName(index))); + return Schema.decimalOf(precision, scale); + } else { + LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " + + "converting into STRING type to avoid any precision loss.", + metadata.getColumnName(index), + metadata.getColumnTypeName(index), + metadata.getColumnName(index))); + return Schema.of(Schema.Type.STRING); + } } return Schema.decimalOf(precision, scale); } diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 5eca20cc4..404262fb2 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -120,6 +120,44 @@ ] } }, + { + "widget-type": "hidden", + "label": "Treat as old timestamp", + "name": "treatAsOldTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat precision less number as Decimal(old behavior)", + "name": "treatPrecisionlessNumAsDeci", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", @@ -326,6 +364,14 @@ { "type": "property", "name": "transactionIsolationLevel" + }, + { + "type": "property", + "name": "getTreatAsOldTimestampConn" + }, + { + "type": "property", + "name": "treatPrecisionlessNumAsDeci" } ] }, diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 628027caf..013f3b240 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -129,6 +129,44 @@ } ] } + }, + { + "widget-type": "hidden", + "label": "Treat as old timestamp", + "name": "treatAsOldTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat precision less number as Decimal(old behavior)", + "name": "treatPrecisionlessNumAsDeci", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] }, From 13520c415fe7e163bcfc3632dc28ae59ee5cff67 Mon Sep 17 00:00:00 2001 From: sahusanket Date: Tue, 3 Jun 2025 20:30:45 +0530 Subject: [PATCH 51/62] Bumping Snapshot. --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 2f3aa906d..acad60eb7 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 58bd6fb9e..9047cb04b 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 708d55615..70c98fc78 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 58a30b41a..c81b8f71c 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 65f8233ec..f412f5e2d 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 3ad3e0704..2c339b245 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 579d49f96..bffa9056e 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 6009d73e9..d378fbb8d 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 1a17ae6b8..d80e2e96c 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 1335d993e..4cb8acb79 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index bc8bda62e..d7e4a4586 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 1bbfe839c..a3553635b 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index c0e0a9dc2..0c78be65c 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 8c6059dde..a55398341 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 5011b2f84..a6b359529 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index e7cf3f934..44612edfc 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.2 + 1.12.3-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 28a801545..4303d5468 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 6996cd8a0..6e9053d36 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 125871dcd..cf962ad21 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT teradata-plugin From 525a91df0e59f0f0ce4afec65188e4815ff6339d Mon Sep 17 00:00:00 2001 From: sahusanket Date: Wed, 4 Jun 2025 16:55:17 +0530 Subject: [PATCH 52/62] removing Snapshot for hub release --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index acad60eb7..17a18caa9 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 9047cb04b..654bde42b 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 70c98fc78..fd508c8fd 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index c81b8f71c..e5d0f896b 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index f412f5e2d..ebe89f33f 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 2c339b245..f7cbe44d5 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index bffa9056e..920ed89c7 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index d378fbb8d..08f979397 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index d80e2e96c..70c4414dc 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 4cb8acb79..682cc153f 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index d7e4a4586..f3ae24c38 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index a3553635b..bf8239beb 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 0c78be65c..763a26b28 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index a55398341..f141c7371 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index a6b359529..59aeeb067 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Oracle plugin diff --git a/pom.xml b/pom.xml index 44612edfc..9d231e198 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.3-SNAPSHOT + 1.12.3 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 4303d5468..22ae8b535 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 6e9053d36..c40736e07 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index cf962ad21..0201805c5 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 teradata-plugin From 59e94841ce7867cb4c7f4876dcabab4deab99a41 Mon Sep 17 00:00:00 2001 From: dj-smart Date: Fri, 11 Jul 2025 11:00:52 +0000 Subject: [PATCH 53/62] CDAP OSS migration for CDF 6.11 --- pom.xml | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index 9d231e198..579dd8240 100644 --- a/pom.xml +++ b/pom.xml @@ -78,23 +78,12 @@ - - sonatype - https://oss.sonatype.org/content/groups/public - sonatype-snapshots - https://oss.sonatype.org/content/repositories/snapshots + https://central.sonatype.com/repository/maven-snapshots - - - sonatype - https://oss.sonatype.org/content/groups/public/ - - - @@ -349,16 +338,6 @@ - - - sonatype.release - https://oss.sonatype.org/service/local/staging/deploy/maven2 - - - sonatype.snapshots - https://oss.sonatype.org/content/repositories/snapshots - - ${testSourceLocation} @@ -532,14 +511,14 @@ - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.2 + org.sonatype.central + central-publishing-maven-plugin + 0.8.0 true - https://oss.sonatype.org - sonatype.release - 655dc88dc770c3 + sonatype.release + false + true From c78ec825098344410e9b5a3db4ac2dc1cb0cbd1e Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Mon, 21 Jul 2025 09:36:49 +0000 Subject: [PATCH 54/62] mark dependencies as test scope --- cloudsql-mysql-plugin/pom.xml | 3 +++ cloudsql-postgresql-plugin/pom.xml | 3 +++ mssql-plugin/pom.xml | 3 +++ mysql-plugin/pom.xml | 3 +++ oracle-plugin/pom.xml | 3 +++ postgresql-plugin/pom.xml | 1 + 6 files changed, 16 insertions(+) diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index e5d0f896b..f2ccb2442 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -59,6 +59,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap @@ -68,6 +69,7 @@ junit junit + test io.cdap.cdap @@ -77,6 +79,7 @@ org.mockito mockito-core + test org.jetbrains diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index ebe89f33f..eeff25572 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -63,6 +63,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap @@ -72,6 +73,7 @@ junit junit + test io.cdap.cdap @@ -81,6 +83,7 @@ org.mockito mockito-core + test org.jetbrains diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index bf8239beb..b485279ed 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -57,6 +57,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap @@ -65,10 +66,12 @@ junit junit + test org.mockito mockito-core + test com.microsoft.sqlserver diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 763a26b28..d87b80ccb 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -57,6 +57,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap @@ -65,6 +66,7 @@ junit junit + test io.cdap.cdap @@ -74,6 +76,7 @@ org.mockito mockito-core + test mysql diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 59aeeb067..4d7125448 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -57,6 +57,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap @@ -65,6 +66,7 @@ junit junit + test org.hsqldb @@ -80,6 +82,7 @@ org.mockito mockito-core + test io.cdap.cdap diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 22ae8b535..d641628dd 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -63,6 +63,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap From 768575e7817168f2e7afeed7c057dcde2ac60110 Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Wed, 6 Aug 2025 02:20:50 +0000 Subject: [PATCH 55/62] Add required for maven central publishing --- cloudsql-mysql-plugin/pom.xml | 44 ++++++++++++++++++++++++---- cloudsql-postgresql-plugin/pom.xml | 44 ++++++++++++++++++++++++---- database-commons/pom.xml | 39 +++++++++++++++++++++++++ mssql-plugin/pom.xml | 46 +++++++++++++++++++++++++---- mysql-plugin/pom.xml | 46 +++++++++++++++++++++++++---- oracle-plugin/pom.xml | 47 ++++++++++++++++++++++++++---- pom.xml | 19 +++++++++++- postgresql-plugin/pom.xml | 46 +++++++++++++++++++++++++---- 8 files changed, 300 insertions(+), 31 deletions(-) diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index f2ccb2442..012acc177 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -26,11 +26,45 @@ CloudSQL MySQL plugin cloudsql-mysql-plugin 4.0.0 + CloudSQL MySQL database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-api + ${cdap.version} provided @@ -41,6 +75,7 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} io.cdap.plugin @@ -59,26 +94,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} test junit junit + ${junit.version} test - - io.cdap.cdap - cdap-api - provided - org.mockito mockito-core + ${mockito.version} test diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index eeff25572..d219d5f65 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -26,11 +26,45 @@ CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin 4.0.0 + CloudSQL PostgreSQL database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-api + ${cdap.version} provided @@ -41,6 +75,7 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} io.cdap.plugin @@ -63,26 +98,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} test junit junit + ${junit.version} test - - io.cdap.cdap - cdap-api - provided - org.mockito mockito-core + ${mockito.version} test diff --git a/database-commons/pom.xml b/database-commons/pom.xml index f7cbe44d5..8ee1e295b 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -26,33 +26,70 @@ Database Commons database-commons 4.0.0 + Database Commons + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + provided io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} io.cdap.cdap hydrator-test + ${cdap.version} + test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} + test com.mockrunner @@ -63,6 +100,8 @@ org.mockito mockito-core + ${mockito.version} + test diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index b485279ed..376a6bc3f 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -26,11 +26,45 @@ Microsoft SQL Server plugin mssql-plugin 4.0.0 + Microsoft SQL Server plugin database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -57,20 +93,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} test org.mockito mockito-core + ${mockito.version} test @@ -79,11 +120,6 @@ 8.2.1.jre8 test - - io.cdap.cdap - cdap-api - provided - org.jetbrains annotations diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index d87b80ccb..d2ea17f41 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -26,11 +26,45 @@ Mysql plugin mysql-plugin 4.0.0 + Mysql database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -57,25 +93,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} test - - io.cdap.cdap - cdap-api - provided - org.mockito mockito-core + ${mockito.version} test diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 4d7125448..30e4b5911 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -26,11 +26,45 @@ Oracle plugin oracle-plugin 4.0.0 + Oracle database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -57,20 +93,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} test org.hsqldb hsqldb + ${hsql.version} test @@ -82,13 +123,9 @@ org.mockito mockito-core + ${mockito.version} test - - io.cdap.cdap - cdap-api - provided - org.glassfish jakarta.json diff --git a/pom.xml b/pom.xml index 579dd8240..1d4baa4b4 100644 --- a/pom.xml +++ b/pom.xml @@ -56,6 +56,22 @@ + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + + 7 true @@ -591,6 +607,7 @@ src/e2e-test/java TestRunner.java + 31.1-jre @@ -716,7 +733,7 @@ ch.qos.logback logback-classic - 1.2.8 + 1.3.15 runtime diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index d641628dd..2eeb641bf 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -26,11 +26,45 @@ PostgreSQL plugin postgresql-plugin 4.0.0 + PostgreSQL database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -63,16 +99,14 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 - - - io.cdap.cdap - cdap-api - provided + ${cdap.version} + test org.jetbrains @@ -83,11 +117,13 @@ org.mockito mockito-core + ${mockito.version} test junit junit + ${junit.version} test From c60c68878790a303f9f9d890e1066f31b9f2f367 Mon Sep 17 00:00:00 2001 From: Komal Yadav Date: Wed, 20 Aug 2025 12:39:38 +0000 Subject: [PATCH 56/62] Remove -SNAPSHOT from Version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1d4baa4b4..3a50542e0 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ UTF-8 6.11.0 - 2.13.1-SNAPSHOT + 2.13.1 13.0.1 3.3.6 2.2.4 From 86f19de7eeb67ad35a41ea92112aca26691a1fed Mon Sep 17 00:00:00 2001 From: AnkitCLI Date: Wed, 10 Sep 2025 20:40:11 +0530 Subject: [PATCH 57/62] Verify Goal Removal --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3a50542e0..a85b645e6 100644 --- a/pom.xml +++ b/pom.xml @@ -665,7 +665,8 @@ integration-test - verify + + From f10bbf9f0a0367a4318ac17ed0e966ee8ecda18c Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Tue, 7 Oct 2025 11:25:02 +0000 Subject: [PATCH 58/62] bump up to 1.12.4-SNAPSHOT --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 17a18caa9..9a545ef6b 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 654bde42b..b9a542c3d 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index fd508c8fd..0f31154ca 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 012acc177..8061b4ca0 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index d219d5f65..f147961e6 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 8ee1e295b..ebd4b8bab 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 920ed89c7..868269a8a 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 08f979397..39cb543d1 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 70c4414dc..912528d4a 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 682cc153f..3f7b9a58b 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index f3ae24c38..53e10ed78 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 376a6bc3f..768e5d4c6 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index d2ea17f41..7c7dd054a 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index f141c7371..824a7d6ec 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 30e4b5911..988cd424b 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index a85b645e6..f59a221d6 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.3 + 1.12.4-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 2eeb641bf..73912e656 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index c40736e07..6e541ddf2 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 0201805c5..6fa152ad7 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT teradata-plugin From 7a3fca681cf883c9519f0d60236ae13f59ac20f8 Mon Sep 17 00:00:00 2001 From: AMit-Cloudsufi Date: Tue, 30 Sep 2025 06:23:06 +0000 Subject: [PATCH 59/62] Add hidden treatTimestampLTZAsTimestamp --- .../cdap/plugin/oracle/OracleConnector.java | 3 +- .../plugin/oracle/OracleConnectorConfig.java | 15 ++- .../cdap/plugin/oracle/OracleConstants.java | 1 + .../io/cdap/plugin/oracle/OracleSource.java | 8 +- .../oracle/OracleSourceSchemaReader.java | 16 +++- .../plugin/oracle/OracleSchemaReaderTest.java | 94 +++++++++++++++++++ oracle-plugin/widgets/Oracle-batchsource.json | 19 ++++ oracle-plugin/widgets/Oracle-connector.json | 19 ++++ 8 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 16371d5c1..3094e7152 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -113,7 +113,8 @@ protected DBConnectorPath getDBConnectorPath(String path) { @Override protected SchemaReader getSchemaReader(String sessionID) { return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(), - config.getTreatPrecisionlessNumAsDeci()); + config.getTreatPrecisionlessNumAsDeci(), + config.getTreatTimestampLTZAsTimestamp()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index cbc1e5ed2..79d14215b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -42,13 +42,14 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null, - null); + null, null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp, - @Nullable Boolean treatPrecisionlessNumAsDeci) { + @Nullable Boolean treatPrecisionlessNumAsDeci, + @Nullable Boolean treatTimestampLTZAsTimestamp) { this.host = host; this.port = port; @@ -62,6 +63,7 @@ public OracleConnectorConfig(String host, int port, String user, String password this.useSSL = useSSL; this.treatAsOldTimestamp = treatAsOldTimestamp; this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci; + this.treatTimestampLTZAsTimestamp = treatTimestampLTZAsTimestamp; } @Override @@ -98,6 +100,11 @@ public String getConnectionString() { @Nullable public Boolean treatPrecisionlessNumAsDeci; + @Name(OracleConstants.TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP) + @Description("A hidden field to handle mapping of Oracle Timestamp_LTZ data type to BQ Timestamp.") + @Nullable + public Boolean treatTimestampLTZAsTimestamp; + @Override protected int getDefaultPort() { return 1521; @@ -128,6 +135,10 @@ public Boolean getTreatPrecisionlessNumAsDeci() { return Boolean.TRUE.equals(treatPrecisionlessNumAsDeci); } + public Boolean getTreatTimestampLTZAsTimestamp() { + return Boolean.TRUE.equals(treatTimestampLTZAsTimestamp); + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index cbd411175..31c7620ff 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -45,6 +45,7 @@ private OracleConstants() { public static final String USE_SSL = "useSSL"; public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp"; public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci"; + public static final String TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP = "treatTimestampLTZAsTimestamp"; /** * Constructs the Oracle connection string based on the provided connection type, host, port, and database. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 1488a084b..978155fc5 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -67,8 +67,10 @@ protected SchemaReader getSchemaReader() { // handle schema to make it backward compatible. boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp(); boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci(); + boolean treatTimestampLTZAsTimestamp = oracleSourceConfig.getConnection().getTreatTimestampLTZAsTimestamp(); - return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci); + return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci, + treatTimestampLTZAsTimestamp); } @Override @@ -133,10 +135,10 @@ public OracleSourceConfig(String host, int port, String user, String password, S int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp, - Boolean treatPrecisionlessNumAsDeci) { + Boolean treatPrecisionlessNumAsDeci, Boolean treatTimestampLTZAsTimestamp) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, role, useSSL, treatAsOldTimestamp, - treatPrecisionlessNumAsDeci); + treatPrecisionlessNumAsDeci, treatTimestampLTZAsTimestamp); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index dd17d2e84..208b70410 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableSet; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.CommonSchemaReader; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,15 +69,17 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { private final String sessionID; private final Boolean isTimestampOldBehavior; private final Boolean isPrecisionlessNumAsDecimal; + private final Boolean isTimestampLtzFieldTimestamp; public OracleSourceSchemaReader() { - this(null, false, false); + this(null, false, false, false); } public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior, - boolean isPrecisionlessNumAsDecimal) { + boolean isPrecisionlessNumAsDecimal, boolean isTimestampLtzFieldTimestamp) { this.sessionID = sessionID; this.isTimestampOldBehavior = isTimestampOldBehavior; this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal; + this.isTimestampLtzFieldTimestamp = isTimestampLtzFieldTimestamp; } @Override @@ -87,8 +90,7 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti case TIMESTAMP_TZ: return isTimestampOldBehavior ? Schema.of(Schema.Type.STRING) : Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); case TIMESTAMP_LTZ: - return isTimestampOldBehavior ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) - : Schema.of(Schema.LogicalType.DATETIME); + return getTimestampLtzSchema(); case Types.TIMESTAMP: return isTimestampOldBehavior ? super.getSchema(metadata, index) : Schema.of(Schema.LogicalType.DATETIME); case BINARY_FLOAT: @@ -139,6 +141,12 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti } } + private @NotNull Schema getTimestampLtzSchema() { + return isTimestampOldBehavior || isTimestampLtzFieldTimestamp + ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) + : Schema.of(Schema.LogicalType.DATETIME); + } + @Override public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { if (sessionID == null) { diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java new file mode 100644 index 000000000..1ff77c533 --- /dev/null +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java @@ -0,0 +1,94 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import com.google.common.collect.Lists; +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.List; + +public class OracleSchemaReaderTest { + + @Test + public void getSchema_timestampLTZFieldTrue_returnTimestamp() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, true); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(2); + // -101 is for TIMESTAMP_TZ + Mockito.when(metadata.getColumnType(1)).thenReturn(-101); + Mockito.when(metadata.getColumnName(1)).thenReturn("column1"); + + // -102 is for TIMESTAMP_LTZ + Mockito.when(metadata.getColumnType(2)).thenReturn(-102); + Mockito.when(metadata.getColumnName(2)).thenReturn("column2"); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("column1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + expectedSchemaFields.add(Schema.Field.of("column2", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema()); + Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); + Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema()); + + } + + @Test + public void getSchema_timestampLTZFieldFalse_returnDatetime() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(2); + // -101 is for TIMESTAMP_TZ + Mockito.when(metadata.getColumnType(1)).thenReturn(-101); + Mockito.when(metadata.getColumnName(1)).thenReturn("column1"); + + // -102 is for TIMESTAMP_LTZ + Mockito.when(metadata.getColumnType(2)).thenReturn(-102); + Mockito.when(metadata.getColumnName(2)).thenReturn("column2"); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("column1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + expectedSchemaFields.add(Schema.Field.of("column2", Schema.of(Schema.LogicalType.DATETIME))); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema()); + Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); + Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema()); + } +} diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 404262fb2..ab35f3e8c 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -158,6 +158,25 @@ ] } }, + { + "widget-type": "hidden", + "label": "Treat Timestamp_LTZ as Timestamp", + "name": "treatTimestampLTZAsTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 013f3b240..005c3ffbd 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -167,6 +167,25 @@ } ] } + }, + { + "widget-type": "hidden", + "label": "Treat Timestamp_LTZ as Timestamp", + "name": "treatTimestampLTZAsTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] }, From 7a62d47bc3597eccb5950626f677559acabfd5d0 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 30 Mar 2026 10:25:27 +0530 Subject: [PATCH 60/62] oracle sink update upsert --- .../main/java/io/cdap/plugin/db/DBRecord.java | 5 +- .../sink/OracleDesignTimeValidation.feature | 137 +++++++++++ .../features/sink/OracleRunTime.feature | 226 ++++++++++++++++++ .../features/sink/OracleRunTimeMacro.feature | 132 ++++++++++ .../java/io.cdap.plugin/OracleClient.java | 32 ++- .../common.stepsdesign/TestSetupHooks.java | 32 +++ .../resources/errorMessage.properties | 4 +- .../resources/pluginParameters.properties | 8 + .../oracle/OracleETLDBOutputFormat.java | 126 ++++++++++ .../io/cdap/plugin/oracle/OracleSink.java | 13 +- .../plugin/oracle/OracleSinkDBRecord.java | 15 +- .../plugin/oracle/OracleSourceDBRecord.java | 2 +- .../oracle/OracleETLDBOutputFormatTest.java | 129 ++++++++++ oracle-plugin/widgets/Oracle-batchsink.json | 28 ++- 14 files changed, 877 insertions(+), 12 deletions(-) create mode 100644 oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java create mode 100644 oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java index a5a9fcf5f..b187c7670 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java @@ -314,10 +314,7 @@ protected void upsertOperation(PreparedStatement stmt) throws SQLException { } private boolean fillUpdateParams(List updatedKeyList, ColumnType columnType) { - if (operationName.equals(Operation.UPDATE) && updatedKeyList.contains(columnType.getName())) { - return true; - } - return false; + return operationName.equals(Operation.UPDATE) && updatedKeyList.contains(columnType.getName()); } private Schema getNonNullableSchema(Schema.Field field) { diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature b/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature index 936802e66..d9cb71e38 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature @@ -173,3 +173,140 @@ Feature: Oracle sink- Verify Oracle sink plugin design time validation scenarios Then Enter input plugin property: "referenceName" with value: "sourceRef" Then Click on the Validate button Then Verify that the Plugin is displaying an error message: "errorMessageInvalidHost" on the header + + @Oracle_Required + Scenario: Verify the validation error message on header with blank database value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click plugin property: "switch-useConnection" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.database.message" on the header + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario: Verify the validation error message with blank password value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario: Verify the validation error message with blank host value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.HostBlank.message" on the header + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario Outline: To verify Oracle sink plugin validation error message for update, upsert operation name and table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Validate button + Then Verify that the Plugin Property: "operationName" is displaying an in-line error message: "errorMessageUpdateUpsertOperationName" + Then Verify that the Plugin Property: "relationTableKey" is displaying an in-line error message: "errorMessageUpdateUpsertOperationName" + Examples: + | options | + | upsert | + | update | diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature index 70b1bdba6..e7cb104c5 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature @@ -218,3 +218,229 @@ Feature: Oracle - Verify data transfer from BigQuery source to Oracle sink Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @BQ_SOURCE_TEST @ORACLE_TEST_TABLE @CONNECTION + Scenario: To verify data is getting transferred from BigQuery source to Oracle sink successfully with use connection + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Oracle" to establish connection + Then Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "bqOutputDatatypesSchema" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + Then Click plugin property: "connector-Oracle" + And Enter input plugin property: "name" with value: "connection.name" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Replace input plugin property: "database" with value: "databaseName" + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + Then Select connection: "connection.name" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Oracle sink + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table with record counts of BigQuery table + Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario Outline: To verify pipeline preview failed with invalid table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Add Button of the property: "relationTableKey" with value: + | invalidOracleTableKey | + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "Failed" + Examples: + | options | + | upsert | + | update | + + @ORACLE_SOURCE_TEST @ORACLE_UPDATE_TABLE @Oracle_Required + Scenario Outline: To verify data is getting transferred from Oracle to Oracle successfully using update operation with table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Add Button of the property: "relationTableKey" with value: + | oracleTableKey | + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on the Preview Data link on the Sink plugin node: "Oracle" + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + Examples: + | options | + | update | + + @ORACLE_SOURCE_TEST @ORACLE_UPSERT_TABLE @Oracle_Required + Scenario Outline: To verify data is getting transferred from Oracle to Oracle successfully using upsert operation with table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Add Button of the property: "relationTableKey" with value: + | oracleTableKey | + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on the Preview Data link on the Sink plugin node: "Oracle" + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + Examples: + | options | + | upsert | diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature index 78130655f..74e5302c6 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature @@ -88,3 +88,135 @@ Feature: Oracle - Verify data transfer to Oracle sink with macro arguments Then Close the pipeline logs Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @ORACLE_SOURCE_TEST @ORACLE_UPSERT_TABLE @Oracle_Required + Scenario: To verify data is getting transferred from Oracle to Oracle successfully with connection argument, transaction isolation, operationName with upsert macro enabled + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Click on the Macro button of Property: "transactionIsolationLevel" and set the value to: "transactionIsolationLevel" + Then Click on the Macro button of Property: "operationName" and set the value to: "oracleOperationName" + Then Click on the Macro button of Property: "relationTableKey" and set the value to: "oracleTableKey" + Then Validate "Oracle2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "upsertOperationName" for key "oracleOperationName" + Then Enter runtime argument value "upsertRelationTableKey" for key "oracleTableKey" + Then Run the preview of pipeline with runtime arguments + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "upsertOperationName" for key "oracleOperationName" + Then Enter runtime argument value "upsertRelationTableKey" for key "oracleTableKey" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate the values of records transferred to target table is equal to the values from source table + + @ORACLE_SOURCE_TEST @ORACLE_UPDATE_TABLE @Oracle_Required + Scenario: To verify data is getting transferred from Oracle source to Oracle sink using macro arguments for operation name with update + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Click on the Macro button of Property: "transactionIsolationLevel" and set the value to: "transactionIsolationLevel" + Then Click on the Macro button of Property: "operationName" and set the value to: "oracleOperationName" + Then Click on the Macro button of Property: "relationTableKey" and set the value to: "oracleTableKey" + Then Validate "Oracle2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "operationName" for key "oracleOperationName" + Then Enter runtime argument value "relationTableKey" for key "oracleTableKey" + Then Run the preview of pipeline with runtime arguments + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "operationName" for key "oracleOperationName" + Then Enter runtime argument value "relationTableKey" for key "oracleTableKey" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate the values of records transferred to target table is equal to the values from source table diff --git a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/OracleClient.java b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/OracleClient.java index 815d5309c..3976972db 100644 --- a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/OracleClient.java +++ b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/OracleClient.java @@ -73,8 +73,8 @@ public static int countRecord(String table, String schema) throws SQLException, */ public static boolean validateRecordValues(String schema, String sourceTable, String targetTable) throws SQLException, ClassNotFoundException { - String getSourceQuery = "SELECT * FROM " + schema + "." + sourceTable; - String getTargetQuery = "SELECT * FROM " + schema + "." + targetTable; + String getSourceQuery = "SELECT * FROM " + schema + "." + sourceTable + " ORDER BY ID"; + String getTargetQuery = "SELECT * FROM " + schema + "." + targetTable + " ORDER BY ID"; try (Connection connect = getOracleConnection()) { connect.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT); Statement statement1 = connect.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE, @@ -193,6 +193,34 @@ public static void createTargetTable(String targetTable, String schema) throws S } } + public static void createOracleSinkUpdateTable(String sourceTable, String schema) throws SQLException, + ClassNotFoundException { + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + String createSourceTableQuery = "CREATE TABLE " + schema + "." + sourceTable + + "(ID number(38), LASTNAME varchar2(100))"; + statement.executeUpdate(createSourceTableQuery); + + // Insert dummy data. + statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " (ID, LASTNAME)" + + " VALUES (1, 'Maria')"); + statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " (ID, LASTNAME)" + + " VALUES (2, 'Shelly')"); + } + } + + public static void createOracleSinkUpsertTable(String sourceTable, String schema) throws SQLException, + ClassNotFoundException { + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + String createSourceTableQuery = "CREATE TABLE " + schema + "." + sourceTable + + "(ID number(38), LASTNAME varchar2(100))"; + statement.executeUpdate(createSourceTableQuery); + + // Insert dummy data. + statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " (ID, LASTNAME)" + + " VALUES (1, 'John')"); + } + } + public static void createSourceDatatypesTable(String sourceTable, String schema) throws SQLException, ClassNotFoundException { try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { diff --git a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java index 757b9c9f7..eb53de3cd 100644 --- a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java +++ b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java @@ -292,6 +292,38 @@ public static void dropOracleTargetTestTable() throws SQLException, ClassNotFoun + " deleted successfully"); } + @Before(order = 2, value = "@ORACLE_UPDATE_TABLE") + public static void createOracleTargetUpdateTable() throws SQLException, ClassNotFoundException { + OracleClient.createOracleSinkUpdateTable(PluginPropertyUtils.pluginProp("targetTable"), + PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("Oracle Target Table - " + PluginPropertyUtils.pluginProp("targetTable") + + " created successfully"); + } + + @After(order = 2, value = "@ORACLE_UPDATE_TABLE") + public static void dropOracleTargetUpdateTable() throws SQLException, ClassNotFoundException { + OracleClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("targetTable")); + BeforeActions.scenario.write("Oracle Target Table - " + PluginPropertyUtils.pluginProp("targetTable") + + " deleted successfully"); + } + + @Before(order = 2, value = "@ORACLE_UPSERT_TABLE") + public static void createOracleTargetUpsertTable() throws SQLException, ClassNotFoundException { + OracleClient.createOracleSinkUpsertTable(PluginPropertyUtils.pluginProp("targetTable"), + PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("Oracle Target Table - " + PluginPropertyUtils.pluginProp("targetTable") + + " created successfully"); + } + + @After(order = 2, value = "@ORACLE_UPSERT_TABLE") + public static void dropOracleTargetUpsertTable() throws SQLException, ClassNotFoundException { + OracleClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("targetTable")); + BeforeActions.scenario.write("Oracle Target Table - " + PluginPropertyUtils.pluginProp("targetTable") + + " deleted successfully"); + } + @Before(order = 1, value = "@BQ_SINK_TEST") public static void setTempTargetBQTableName() { String bqTargetTableName = "E2E_TARGET_" + UUID.randomUUID().toString().replaceAll("-", "_"); diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index e44f4c00a..2a1cea51e 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -18,4 +18,6 @@ errorMessageInvalidHost=Exception while trying to validate schema of database ta errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. blank.database.message=Required property 'database' has no value. -blank.connection.message=Exception while trying to validate schema of database table +blank.connection.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '72000', errorCode: '1005', errorMessage: SQL Exception occurred: [Message='ORA-01005: null password given; logon denied ', SQLState='72000', ErrorCode='1005'].' +blank.HostBlank.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: The Network Adapter could not establish the connection', SQLState='08006', ErrorCode='17002'].' +errorMessageUpdateUpsertOperationName=Table key must be set if the operation is 'Update' or 'Upsert'. diff --git a/oracle-plugin/src/e2e-test/resources/pluginParameters.properties b/oracle-plugin/src/e2e-test/resources/pluginParameters.properties index c71e82b53..d16a7b450 100644 --- a/oracle-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/oracle-plugin/src/e2e-test/resources/pluginParameters.properties @@ -84,6 +84,7 @@ invalidPassword=testPassword invalidBoundingQuery=SELECT MIN(id),MAX(id) FROM table invalidBoundingQueryValue=select; invalidTable=table +invalidOracleTableKey=asdas #ORACLE Valid Properties connectionArgumentsList=[{"key":"queryTimeout","value":"-1"}] @@ -93,6 +94,13 @@ numberOfSplits=2 zeroValue=0 splitByColumn=ID importQuery=where $CONDITIONS +connectionArguments=queryTimeout=50 +transactionIsolationLevel=TRANSACTION_READ_COMMITTED +operationName=update +oracleTableKey=ID +relationTableKey=ID +upsertOperationName=upsert +upsertRelationTableKey=ID #bq properties projectId=cdf-athena diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java new file mode 100644 index 000000000..4a7b18278 --- /dev/null +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java @@ -0,0 +1,126 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import io.cdap.plugin.db.sink.ETLDBOutputFormat; + +/** + * Class that extends {@link ETLDBOutputFormat} to implement the abstract methods + */ +public class OracleETLDBOutputFormat extends ETLDBOutputFormat { + + /** + * This method is used to construct the upsert query for Oracle using MERGE statement. + * Example - MERGE INTO my_table target + * USING (SELECT ? AS id, ? AS name, ? AS age FROM dual) source + * ON (target.id = source.id) + * WHEN MATCHED THEN UPDATE SET target.name = source.name, target.age = source.age + * WHEN NOT MATCHED THEN INSERT (id, name, age) VALUES (source.id, source.name, source.age) + * @param table - Name of the table + * @param fieldNames - All the columns of the table + * @param listKeys - The columns used as keys for matching + * @return Upsert query in the form of string + */ + @Override + public String constructUpsertQuery(String table, String[] fieldNames, String[] listKeys) { + if (listKeys == null) { + throw new IllegalArgumentException( + "'Relation Table Key' must be specified for upsert operations. " + + "Please provide the list of key columns used to match records in the target table."); + } else if (fieldNames == null) { + throw new IllegalArgumentException( + "'Field Names' must be specified for upsert operations. " + + "Please provide the list of columns to be written to the target table."); + } else { + StringBuilder query = new StringBuilder(); + + // MERGE INTO target_table target + query.append("MERGE INTO ").append(table).append(" target "); + + // USING (SELECT ? AS col1, ? AS col2, ... FROM dual) source + query.append("USING (SELECT "); + for (int i = 0; i < fieldNames.length; ++i) { + query.append("? AS ").append(fieldNames[i]); + if (i != fieldNames.length - 1) { + query.append(", "); + } + } + query.append(" FROM dual) source "); + + // ON (target.key1 = source.key1 AND target.key2 = source.key2 ...) + query.append("ON ("); + for (int i = 0; i < listKeys.length; ++i) { + query.append("target.").append(listKeys[i]).append(" = source.").append(listKeys[i]); + if (i != listKeys.length - 1) { + query.append(" AND "); + } + } + query.append(") "); + + // WHEN MATCHED THEN UPDATE SET target.col1 = source.col1, target.col2 = source.col2 ... + // Only update non-key columns + query.append("WHEN MATCHED THEN UPDATE SET "); + boolean firstUpdateColumn = true; + for (String fieldName : fieldNames) { + boolean isKeyColumn = false; + for (String listKey : listKeys) { + String listKeyNoQuote = listKey.replace("\"", ""); + if (listKeyNoQuote.equals(fieldName)) { + isKeyColumn = true; + break; + } + } + if (!isKeyColumn) { + if (!firstUpdateColumn) { + query.append(", "); + } + query.append("target.").append(fieldName).append(" = source.").append(fieldName); + firstUpdateColumn = false; + } + } + + // WHEN NOT MATCHED THEN INSERT (col1, col2, ...) VALUES (source.col1, source.col2, ...) + query.append(" WHEN NOT MATCHED THEN INSERT ("); + for (int i = 0; i < fieldNames.length; ++i) { + query.append(fieldNames[i]); + if (i != fieldNames.length - 1) { + query.append(", "); + } + } + query.append(") VALUES ("); + for (int i = 0; i < fieldNames.length; ++i) { + query.append("source.").append(fieldNames[i]); + if (i != fieldNames.length - 1) { + query.append(", "); + } + } + query.append(")"); + + return query.toString(); + } + } + + @Override + public String constructUpdateQuery(String table, String[] fieldNames, String[] listKeys) { + // Oracle JDBC does not accept a trailing semicolon in prepared statements. + String query = super.constructUpdateQuery(table, fieldNames, listKeys); + if (query.endsWith(";")) { + return query.substring(0, query.length() - 1); + } + return query; + } +} diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java index 511281e9d..415cfa660 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java @@ -23,6 +23,7 @@ import io.cdap.cdap.api.annotation.MetadataProperty; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Output; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSink; @@ -31,6 +32,8 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.common.batch.sink.SinkOutputFormatProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSinkConfig; @@ -59,7 +62,8 @@ public OracleSink(OracleSinkConfig oracleSinkConfig) { @Override protected DBRecord getDBRecord(StructuredRecord output) { - return new OracleSinkDBRecord(output, columnTypes); + return new OracleSinkDBRecord(output, columnTypes, oracleSinkConfig.getOperationName(), + oracleSinkConfig.getRelationTableKey()); } @Override @@ -71,6 +75,13 @@ protected FieldsValidator getFieldsValidator() { protected SchemaReader getSchemaReader() { return new OracleSinkSchemaReader(); } + + @Override + protected void addOutputContext(BatchSinkContext context) { + context.addOutput(Output.of(oracleSinkConfig.getReferenceName(), + new SinkOutputFormatProvider(OracleETLDBOutputFormat.class, getConfiguration()))); + } + @Override protected LineageRecorder getLineageRecorder(BatchSinkContext context) { String fqn = DBUtils.constructFQN("oracle", diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java index 01b9a8247..ee77a7fc9 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java @@ -19,6 +19,7 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.ColumnType; +import io.cdap.plugin.db.Operation; import io.cdap.plugin.db.SchemaReader; import java.sql.PreparedStatement; @@ -31,9 +32,12 @@ */ public class OracleSinkDBRecord extends OracleSourceDBRecord { - public OracleSinkDBRecord(StructuredRecord record, List columnTypes) { + public OracleSinkDBRecord(StructuredRecord record, List columnTypes, Operation operationName, + String relationTableKey) { this.record = record; this.columnTypes = columnTypes; + this.operationName = operationName; + this.relationTableKey = relationTableKey; } @Override @@ -50,4 +54,13 @@ protected void insertOperation(PreparedStatement stmt) throws SQLException { writeToDB(stmt, field, fieldIndex); } } + + @Override + protected void upsertOperation(PreparedStatement stmt) throws SQLException { + for (int fieldIndex = 0; fieldIndex < columnTypes.size(); fieldIndex++) { + ColumnType columnType = columnTypes.get(fieldIndex); + Schema.Field field = record.getSchema().getField(columnType.getName()); + writeToDB(stmt, field, fieldIndex); + } + } } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java index 7d7c69d2b..44131a01b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java @@ -116,8 +116,8 @@ protected void handleField(ResultSet resultSet, StructuredRecord.Builder recordB @Override protected void writeNonNullToDB(PreparedStatement stmt, Schema fieldSchema, String fieldName, int fieldIndex) throws SQLException { - int sqlType = columnTypes.get(fieldIndex).getType(); int sqlIndex = fieldIndex + 1; + int sqlType = modifiableColumnTypes.get(fieldIndex).getType(); // TIMESTAMP and TIMESTAMPTZ types needs to be handled using the specific oracle types to ensure that the data // inserted matches with the provided value. As Oracle driver internally alters the values provided diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java new file mode 100644 index 000000000..994c86b55 --- /dev/null +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java @@ -0,0 +1,129 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import org.junit.Assert; +import org.junit.Test; + +public class OracleETLDBOutputFormatTest { + + private final OracleETLDBOutputFormat outputFormat = new OracleETLDBOutputFormat(); + + @Test + public void testConstructUpsertQueryBasic() { + String[] fieldNames = {"id", "name", "age"}; + String[] listKeys = {"id"}; + String table = "my_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO my_table target " + + "USING (SELECT ? AS id, ? AS name, ? AS age FROM dual) source " + + "ON (target.id = source.id) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name, target.age = source.age " + + "WHEN NOT MATCHED THEN INSERT (id, name, age) VALUES (source.id, source.name, source.age)"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testConstructUpsertQueryMultipleKeys() { + String[] fieldNames = {"id", "code", "name", "value"}; + String[] listKeys = {"id", "code"}; + String table = "composite_key_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO composite_key_table target " + + "USING (SELECT ? AS id, ? AS code, ? AS name, ? AS value FROM dual) source " + + "ON (target.id = source.id AND target.code = source.code) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name, target.value = source.value " + + "WHEN NOT MATCHED THEN INSERT (id, code, name, value) VALUES (source.id, source.code, source.name, source" + + ".value)"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testConstructUpsertQuerySingleField() { + String[] fieldNames = {"id", "name"}; + String[] listKeys = {"id"}; + String table = "single_field_update_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO single_field_update_table target " + + "USING (SELECT ? AS id, ? AS name FROM dual) source " + + "ON (target.id = source.id) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name " + + "WHEN NOT MATCHED THEN INSERT (id, name) VALUES (source.id, source.name)"; + + Assert.assertEquals(expected, result); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructUpsertQueryNullListKeys() { + String[] fieldNames = {"id", "name", "age"}; + String table = "my_table"; + + outputFormat.constructUpsertQuery(table, fieldNames, null); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructUpsertQueryNullFieldNames() { + String[] listKeys = {"id"}; + String table = "my_table"; + + outputFormat.constructUpsertQuery(table, null, listKeys); + } + + @Test + public void testConstructUpsertQueryAllFieldsAreKeys() { + String[] fieldNames = {"id", "code"}; + String[] listKeys = {"id", "code"}; + String table = "all_keys_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + // When all fields are keys, the UPDATE SET clause will be empty after "SET " + // Note: There's an extra space before "WHEN NOT MATCHED" due to implementation + String expected = "MERGE INTO all_keys_table target " + + "USING (SELECT ? AS id, ? AS code FROM dual) source " + + "ON (target.id = source.id AND target.code = source.code) " + + "WHEN MATCHED THEN UPDATE SET " + + "WHEN NOT MATCHED THEN INSERT (id, code) VALUES (source.id, source.code)"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testConstructUpsertQueryWithSpecialTableName() { + String[] fieldNames = {"id", "name"}; + String[] listKeys = {"id"}; + String table = "SCHEMA.MY_TABLE"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO SCHEMA.MY_TABLE target " + + "USING (SELECT ? AS id, ? AS name FROM dual) source " + + "ON (target.id = source.id) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name " + + "WHEN NOT MATCHED THEN INSERT (id, name) VALUES (source.id, source.name)"; + + Assert.assertEquals(expected, result); + } +} diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index 8d6168780..e3537e2ba 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -192,6 +192,29 @@ "label": "Schema Name", "name": "dbSchemaName" }, + { + "widget-type": "radio-group", + "label": "Operation Name", + "name": "operationName", + "widget-attributes": { + "default": "insert", + "layout": "inline", + "options": [ + { + "id": "insert", + "label": "INSERT" + }, + { + "id": "update", + "label": "UPDATE" + }, + { + "id": "upsert", + "label": "UPSERT" + } + ] + } + }, { "widget-type": "hidden", "label": "Operation Name", @@ -201,9 +224,10 @@ } }, { - "widget-type": "hidden", + "name": "relationTableKey", + "widget-type": "csv", "label": "Table Key", - "name": "relationTableKey" + "widget-attributes": {} } ] }, From 99951d337d40aa96629e491ab21297749992571e Mon Sep 17 00:00:00 2001 From: suryakumari Date: Mon, 6 Apr 2026 19:34:03 +0530 Subject: [PATCH 61/62] cdap e2e framework version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f59a221d6..c7259d221 100644 --- a/pom.xml +++ b/pom.xml @@ -721,7 +721,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.4.0 + 0.4.1 test From 081c0d5d8f012f5beee0814fab8d7f854325ff19 Mon Sep 17 00:00:00 2001 From: vedanshugarg04 Date: Thu, 21 May 2026 17:11:30 +0530 Subject: [PATCH 62/62] updated cdap-e2e-framework from 0.4.1 to 0.4.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c7259d221..fa94b54b0 100644 --- a/pom.xml +++ b/pom.xml @@ -721,7 +721,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.4.1 + 0.4.2 test