From c399e9beef8c62b031184c00741ff78233e35fb9 Mon Sep 17 00:00:00 2001 From: AnkitCLI Date: Mon, 4 Nov 2024 11:39:48 +0530 Subject: [PATCH] wrangler ui features --- .../WranglerUI/DataTypeParsers.feature | 6 +-- .../WranglerUI/ParseAsFixedLength.feature | 2 +- .../features/WranglerUI/ParseAsHl7.feature | 4 +- .../features/WranglerUI/ParseAsJson.feature | 2 +- .../features/WranglerUI/ParseAsLog.feature | 2 +- .../WranglerUI/ParseAsXmlToJson.feature | 2 +- .../pluginDataTestIdAttributes.properties | 49 ++++++++++++++++++- .../resources/pluginParameters.properties | 1 + 8 files changed, 57 insertions(+), 11 deletions(-) diff --git a/wrangler-transform/src/e2e-test/features/WranglerUI/DataTypeParsers.feature b/wrangler-transform/src/e2e-test/features/WranglerUI/DataTypeParsers.feature index bebcc1090..c531fcc17 100644 --- a/wrangler-transform/src/e2e-test/features/WranglerUI/DataTypeParsers.feature +++ b/wrangler-transform/src/e2e-test/features/WranglerUI/DataTypeParsers.feature @@ -31,8 +31,8 @@ Feature: datatype parsers Then Select connection data row with name: "dataset" Then Select connection data row with name: "bqSourceTable" Then Verify connection datatable is displayed for the data: "bqSourceTable" - Then Expand dropdown column: "create_date" and apply directive: "Parse" as "Simple date" with: "yyyy-MM-dd" option - Then Expand dropdown column: "update_date" and apply directive: "Parse" as "Simple date" with: "yyyy-MM-dd" option + Then Expand dropdown column: "update_date" and apply directive: "Parse" as "SIMPLEDATE" with: "yyyy-MM-dd" option + Then Expand dropdown column: "create_date" and apply directive: "Parse" as "SIMPLEDATE" with: "yyyy-MM-dd" option Then Enter directive from CLI "parse-timestamp :time" Then Enter directive from CLI "parse-as-currency :price :newprice" Then Enter directive from CLI "format-as-currency :newprice :format_price" @@ -85,7 +85,7 @@ Feature: datatype parsers Then Select connection data row with name: "dataset" Then Select connection data row with name: "bqSourceTable" Then Verify connection datatable is displayed for the data: "bqSourceTable" - Then Expand dropdown column: "timestamp" and apply directive: "Parse" with directive type: "Datetime" and select: "Custom format" and enter: "yyyy-MM-dd'T'HH:mm:ssX'['z']'" + Then Expand dropdown column: "timestamp" and apply directive: "Parse" with directive type: "DATETIME" and select: "Custom_Format" and enter: "yyyy-MM-dd'T'HH:mm:ssX'['z']'" Then Enter directive from CLI "current-datetime :create_date" Then Enter directive from CLI "datetime-to-timestamp :timestamp" Then Enter directive from CLI "format-datetime :create_date 'y'" diff --git a/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsFixedLength.feature b/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsFixedLength.feature index 625ff7747..90a9cd17b 100644 --- a/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsFixedLength.feature +++ b/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsFixedLength.feature @@ -31,7 +31,7 @@ Feature: parse as fixed length Then Select connection data row with name: "dataset" Then Select connection data row with name: "bqSourceTable" Then Verify connection datatable is displayed for the data: "bqSourceTable" - Then Expand dropdown column: "fixedlength" and apply directive: "Parse" as "Fixed length" with: "2,4,5,3" option + Then Expand dropdown column: "fixedlength" and apply directive: "Parse" as "FIXEDLENGTH" with: "2,4,5,3" option Then Enter directive from CLI "split-url url" Then Enter directive from CLI "write-as-csv :url_protocol" Then Enter directive from CLI "url-encode :url" diff --git a/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsHl7.feature b/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsHl7.feature index bed600cec..337978e83 100644 --- a/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsHl7.feature +++ b/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsHl7.feature @@ -36,8 +36,8 @@ Feature: parse as HL7 Then Enter directive from CLI "set-type :Body string" Then Enter directive from CLI "keep address,Body,Body_hl7_MSH_12,Body_hl7_MSH_9_1" Then Expand dropdown column: "address" and apply directive: "FindAndReplace" and select: "address1" and enter: "test" - Then Expand dropdown column: "Body_hl7_MSH_9_1" and apply directive: "Maskdata" as "By shuffling" - Then Expand dropdown column: "address" and apply directive: "Sendtoerror" as "value is empty" + Then Expand dropdown column: "Body_hl7_MSH_9_1" and apply directive: "MaskData" as "By_shuffling" + Then Expand dropdown column: "address" and apply directive: "SendToError" as "value_is_empty" Then Enter directive from CLI "rename :Body_hl7_MSH_12 :id " Then Click Create Pipeline button and choose the type of pipeline as: "Batch pipeline" Then Verify plugin: "BigQueryTable" node is displayed on the canvas with a timeout of 120 seconds diff --git a/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsJson.feature b/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsJson.feature index b22f71996..d4a395d5a 100644 --- a/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsJson.feature +++ b/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsJson.feature @@ -32,7 +32,7 @@ Feature: parse as Json Then Select connection data row with name: "bqSourceTable" Then Verify connection datatable is displayed for the data: "bqSourceTable" Then Expand dropdown column: "json" and apply directive: "Parse" as "JSON" with: "1" option - Then Expand dropdown column: "body" and apply directive: "Format" as "Trim leading whitespace" + Then Expand dropdown column: "body" and apply directive: "Format" as "Trim_leading_whitespace" Then Enter directive from CLI "set-column :desc concat(json_pet,body)" Then Expand dropdown column: "json_name" and apply directive: "CopyColumn" as "copied" Then Select checkbox on two columns: "json_id" and "json_age" diff --git a/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsLog.feature b/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsLog.feature index f5113be88..911f1f833 100644 --- a/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsLog.feature +++ b/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsLog.feature @@ -31,7 +31,7 @@ Feature: Wrangler - Run time scenarios for Parse Log Then Select connection data row with name: "dataset" Then Select connection data row with name: "bqSourceTable" Then Verify connection datatable is displayed for the data: "bqSourceTable" - Then Expand dropdown column: "body" and apply directive: "Parse" as "Log" with: "Common" option + Then Expand dropdown column: "body" and apply directive: "Parse" as "LOG" with: "Common" option Then Expand dropdown column: "number_connection_client_logname_last" and apply directive: "DeleteColumn" Then Expand dropdown column: "number_connection_client_logname" and apply directive: "DeleteColumn" Then Expand dropdown column: "http_querystring_request_firstline_uri_query" and apply directive: "DeleteColumn" diff --git a/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsXmlToJson.feature b/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsXmlToJson.feature index 6819f38f2..b71bdd519 100644 --- a/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsXmlToJson.feature +++ b/wrangler-transform/src/e2e-test/features/WranglerUI/ParseAsXmlToJson.feature @@ -31,7 +31,7 @@ Feature: parse as XmlToJson Then Select connection data row with name: "dataset" Then Select connection data row with name: "bqSourceTable" Then Verify connection datatable is displayed for the data: "bqSourceTable" - Then Expand dropdown column: "xmldata" and apply directive: "Parse" as "XML to JSON" with: "1" option + Then Expand dropdown column: "xmldata" and apply directive: "Parse" as "XMLTOJSON" with: "1" option Then Enter directive from CLI "split-email :email" Then Enter directive from CLI "text-distance block email email_account distance" Then Enter directive from CLI "text-metric longest-common-subsequence email email_account distance2" diff --git a/wrangler-transform/src/e2e-test/resources/pluginDataTestIdAttributes.properties b/wrangler-transform/src/e2e-test/resources/pluginDataTestIdAttributes.properties index a8fd6f034..fe2cbde25 100644 --- a/wrangler-transform/src/e2e-test/resources/pluginDataTestIdAttributes.properties +++ b/wrangler-transform/src/e2e-test/resources/pluginDataTestIdAttributes.properties @@ -1,10 +1,10 @@ ChangeDataType=changeDataType -Sendtoerror=markAsError +SendToError=markAsError SwapTwoColumnNames=swapColumns DeleteColumn=dropColumn Hash=hash Parse=parse -Maskdata=maskData +MaskData=maskData FindAndReplace=findAndReplace Format=format Calculate=calculate @@ -17,3 +17,48 @@ ExtractFields=extractFields Explode=explode DefineVariable=defineVariable SetCounter=setCounter +Concatenate=CONCATENATE +Always=ALWAYS +Integer=integer +Comma=COMMA +Common=COMMON +Tab=TAB +Space=SPACE +Pipe=PIPE +CONTROL_A=^A +CONTROL_D=^D +CSV=csv +Avro=AVRO +EXCEL=excel +JSON=singleField +XMLTOJSON=xmlToJson +LOG=log +SIMPLEDATE=dateFormats +DATETIME=dateFormats +FIXEDLENGTH=singleField +HL7=HL7 +Decimal=decimal +lowercase=LOWERCASE +Trim_Whitespace=TRIM_WHITESPACE +Character_count=CHARCOUNT +Using_patterns=patterns +Using_delimiters=delimiters +Delimited_text=delimited +Array_(by_flattening)=arrayFlattening +Record_(by_flattening)=recordFlattening +Show_last_4_characters_only=last4Chars +Show_last_2_characters_only=last2Chars +Custom_selection=customSelection +By_shuffling=shuffling +value_is_empty=EMPTY +value_is=TEXTEXACTLY +value_contains=TEXTCONTAINS +value_starts_with=TEXTSTARTSWITH +Trim_leading_whitespace=TRIM_LEADING_WHITESPACE +Custom_Format=CUSTOM +yyyy-MM-dd=OPTION5 + + + + + diff --git a/wrangler-transform/src/e2e-test/resources/pluginParameters.properties b/wrangler-transform/src/e2e-test/resources/pluginParameters.properties index 5d21ece3d..08a8e7696 100644 --- a/wrangler-transform/src/e2e-test/resources/pluginParameters.properties +++ b/wrangler-transform/src/e2e-test/resources/pluginParameters.properties @@ -46,6 +46,7 @@ dataset=Wrangler_Test dataset2=Wrangler #expectedBQFiles ExpectedDirective_GroupBy=BQValidationExpectedFiles/Directive_wrangler_GroupBy +filterEmptyProperty=value is empty ExpectedDirective_parse_FixedLength=BQValidationExpectedFiles/Directive_parse_fixedlength ExpectedDirective_parse_hl7=BQValidationExpectedFiles/Directive_parse_hl7