[NU-1821][NU-1822] BASE64 helper and from/to JSON conversions (#6995)

TouK · Oct 11, 2024 · e6e3f23 · e6e3f23
1 parent 2224d33
commit e6e3f23
Show file tree

Hide file tree

Showing 11 changed files with 280 additions and 17 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1126,7 +1126,7 @@ lazy val defaultHelpers = (project in utils("default-helpers"))
   .settings(
     name := "nussknacker-default-helpers"
   )
-  .dependsOn(mathUtils, testUtils % Test, scenarioCompiler % "test->test;test->compile")
+  .dependsOn(mathUtils, commonUtils, testUtils % Test, scenarioCompiler % "test->test;test->compile")
 
 lazy val testUtils = (project in utils("test-utils"))
   .settings(commonSettings)

diff --git a/defaultModel/src/main/scala/pl/touk/nussknacker/defaultmodel/DefaultConfigCreator.scala b/defaultModel/src/main/scala/pl/touk/nussknacker/defaultmodel/DefaultConfigCreator.scala
@@ -18,6 +18,7 @@ class DefaultConfigCreator extends EmptyProcessConfigCreator {
         "DATE_FORMAT" -> anyCategory(dateFormat),
         "UTIL"        -> anyCategory(util),
         "RANDOM"      -> anyCategory(random),
+        "BASE64"      -> anyCategory(base64)
       ),
       List()
     )

diff --git a/docs/Changelog.md b/docs/Changelog.md
@@ -47,9 +47,11 @@
   * toBigIntegerOrNull
   * toBigDecimal
   * toBigDecimalOrNull
+* [#6995](https://github.com/TouK/nussknacker/pull/6995) Add `toJson` and `toJsonString` conversions (in the `#CONV` helper)
+* [#6995](https://github.com/TouK/nussknacker/pull/6995) Add `#BASE64` helper to decode/encode Base64 values
 * [#6826](https://github.com/TouK/nussknacker/pull/6826) Security fix: added validation of expression used inside 
   indexer for Maps and Lists (for example `{1,2,3}[#otherList.remove(1) == null ? 0 : 0]`). This allowed executing 
-  some types of unallowed expressions.
+  some types of not allowed expressions.
 * [#6880](https://github.com/TouK/nussknacker/pull/6880) Performance optimization of generating Avro messages with unions
   - shorter message in logs
 * [#6766](https://github.com/TouK/nussknacker/pull/6766) Scenario labels support - you can assign labels to scenarios and use them to filter the scenario list

diff --git a/docs/scenarios_authoring/Spel.md b/docs/scenarios_authoring/Spel.md
@@ -289,17 +289,17 @@ Explicit conversions are available in utility classes and build-in java conversi
 
 Nussknacker comes with the following helpers:
 
-| Helper        | Functions                                      |
-|---------------|------------------------------------------------|
-| `COLLECTION`  | Operations on collections                      |
-| `CONV`        | General conversion functions                   |
-| `DATE`        | Date operations (conversions, useful helpers)  |
-| `DATE_FORMAT` | Date formatting/parsing operations             |
-| `GEO`         | Simple distance measurements                   |
-| `NUMERIC`     | Number parsing                                 |
-| `RANDOM`      | Random value generators                        |
-| `UTIL`        | Various utilities (e.g. identifier generation) |
-
+| Helper        | Functions                                                          |
+|---------------|--------------------------------------------------------------------|
+| `COLLECTION`  | Operations on collections                                          |
+| `CONV`        | General conversion functions                                       |
+| `DATE`        | Date operations (conversions, useful helpers)                      |
+| `DATE_FORMAT` | Date formatting/parsing operations                                 |
+| `GEO`         | Simple distance measurements                                       |
+| `NUMERIC`     | Number parsing                                                     |
+| `RANDOM`      | Random value generators                                            |
+| `UTIL`        | Various utilities (e.g. identifier generation)                     |
+| `BASE64`      | Encoding & decoding [Base64](https://en.wikipedia.org/wiki/Base64) |
 
 ## Handling date/time.
 

diff --git a/...t-utils/src/main/scala/pl/touk/nussknacker/engine/flink/test/ClassDiscoveryBaseTest.scala b/...t-utils/src/main/scala/pl/touk/nussknacker/engine/flink/test/ClassDiscoveryBaseTest.scala
@@ -2,6 +2,7 @@ package pl.touk.nussknacker.engine.flink.test
 
 import cats.data.NonEmptyList
 import cats.implicits.toFunctorOps
+import com.typesafe.scalalogging.LazyLogging
 import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
 import io.circe.parser.parse
 import io.circe.syntax.EncoderOps
@@ -13,7 +14,7 @@ import org.scalatest.matchers.should.Matchers
 import org.springframework.util.ClassUtils
 import pl.touk.nussknacker.engine.ModelData
 import pl.touk.nussknacker.engine.api.generics.{MethodTypeInfo, Parameter}
-import pl.touk.nussknacker.engine.api.typed.typing.{TypedClass, TypingResult, Unknown}
+import pl.touk.nussknacker.engine.api.typed.typing.{TypedClass, TypingResult}
 import pl.touk.nussknacker.engine.api.typed.{TypeEncoders, TypingResultDecoder}
 import pl.touk.nussknacker.engine.definition.clazz.{
   ClassDefinition,
@@ -30,7 +31,7 @@ import pl.touk.nussknacker.engine.api.CirceUtil._
 
 import scala.util.Properties
 
-trait ClassDiscoveryBaseTest extends AnyFunSuite with Matchers with Inside {
+trait ClassDiscoveryBaseTest extends AnyFunSuite with Matchers with Inside with LazyLogging {
 
   protected def model: ModelData
 
@@ -68,6 +69,7 @@ trait ClassDiscoveryBaseTest extends AnyFunSuite with Matchers with Inside {
     val types = model.modelDefinitionWithClasses.classDefinitions.all
     if (Option(System.getenv("CLASS_EXTRACTION_PRINT")).exists(_.toBoolean)) {
       val fileName = s"${Properties.tmpDir}/${getClass.getSimpleName}-result.json"
+      logger.info(s"CLASS_EXTRACTION_PRINT is set. The file JSON file will be stored in '$fileName'")
       FileUtils.write(new File(fileName), encode(types), StandardCharsets.UTF_8)
     }
     val parsed  = parse(ResourceLoader.load(outputResource)).toOption.get

diff --git a/engine/flink/tests/src/test/resources/extractedTypes/defaultModel.json b/engine/flink/tests/src/test/resources/extractedTypes/defaultModel.json
@@ -13124,6 +13124,60 @@
       ]
     }
   },
+  {
+    "clazzName": {"refClazzName": "pl.touk.nussknacker.engine.util.functions.base64$"},
+    "methods": {
+      "decode": [
+        {
+          "description": "Decode Base64 value to String",
+          "name": "decode",
+          "signature": {
+            "noVarArgs": [
+              {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}}
+            ],
+            "result": {"refClazzName": "java.lang.String"}
+          }
+        }
+      ],
+      "encode": [
+        {
+          "description": "Encode String value to Base64",
+          "name": "encode",
+          "signature": {
+            "noVarArgs": [
+              {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}}
+            ],
+            "result": {"refClazzName": "java.lang.String"}
+          }
+        }
+      ],
+      "urlSafeDecode": [
+        {
+          "description": "Decode URL-safe Base64 value to String",
+          "name": "urlSafeDecode",
+          "signature": {
+            "noVarArgs": [
+              {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}}
+            ],
+            "result": {"refClazzName": "java.lang.String"}
+          }
+        }
+      ],
+      "urlSafeEncode": [
+        {
+          "description": "Encode String value to URL-safe Base64",
+          "name": "urlSafeEncode",
+          "signature": {
+            "noVarArgs": [
+              {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}}
+            ],
+            "result": {"refClazzName": "java.lang.String"}
+          }
+        }
+      ]
+    },
+    "staticMethods": {}
+  },
   {
     "clazzName": {"refClazzName": "pl.touk.nussknacker.engine.util.functions.collection$"},
     "methods": {
@@ -14110,6 +14164,42 @@
           }
         }
       ],
+      "toJson": [
+        {
+          "description": "Convert String value to JSON",
+          "name": "toJson",
+          "signature": {
+            "noVarArgs": [
+              {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}}
+            ],
+            "result": {"type": "Unknown"}
+          }
+        }
+      ],
+      "toJsonOrNull": [
+        {
+          "description": "Convert String value to JSON or null in case of failure",
+          "name": "toJsonOrNull",
+          "signature": {
+            "noVarArgs": [
+              {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}}
+            ],
+            "result": {"type": "Unknown"}
+          }
+        }
+      ],
+      "toJsonString": [
+        {
+          "description": "Convert JSON to String",
+          "name": "toJsonString",
+          "signature": {
+            "noVarArgs": [
+              {"name": "value", "refClazz": {"type": "Unknown"}}
+            ],
+            "result": {"refClazzName": "java.lang.String"}
+          }
+        }
+      ],
       "toLong": [
         {
           "description": "Convert any value to Long or throw exception in case of failure",
@@ -15521,4 +15611,4 @@
       ]
     }
   }
-]
+]
diff --git a/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/base64.scala b/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/base64.scala
@@ -0,0 +1,31 @@
+package pl.touk.nussknacker.engine.util.functions
+
+import pl.touk.nussknacker.engine.api.{Documentation, HideToString, ParamName}
+
+import java.util.Base64
+
+object base64 extends Base64Utils
+
+trait Base64Utils extends HideToString {
+
+  @Documentation(description = "Decode Base64 value to String")
+  def decode(@ParamName("value") value: String): String = {
+    new String(Base64.getDecoder.decode(value.getBytes("UTF-8")))
+  }
+
+  @Documentation(description = "Encode String value to Base64")
+  def encode(@ParamName("value") value: String): String = {
+    new String(Base64.getEncoder.encode(value.getBytes("UTF-8")))
+  }
+
+  @Documentation(description = "Decode URL-safe Base64 value to String")
+  def urlSafeDecode(@ParamName("value") value: String): String = {
+    new String(Base64.getUrlDecoder.decode(value.getBytes("UTF-8")))
+  }
+
+  @Documentation(description = "Encode String value to URL-safe Base64")
+  def urlSafeEncode(@ParamName("value") value: String): String = {
+    new String(Base64.getUrlEncoder.withoutPadding().encode(value.getBytes("UTF-8")))
+  }
+
+}
diff --git a/...default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/conversion.scala b/...default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/conversion.scala
@@ -4,6 +4,7 @@ import pl.touk.nussknacker.engine.api.generics.GenericType
 import pl.touk.nussknacker.engine.api.{Documentation, HideToString, ParamName}
 import pl.touk.nussknacker.engine.util.functions.ConversionUtils.{stringToBigInteger, stringToBoolean}
 import pl.touk.nussknacker.engine.util.functions.NumericUtils.ToNumberTypingFunction
+import pl.touk.nussknacker.engine.util.json.{JsonUtils, ToJsonEncoder}
 
 import scala.util.Try
 
@@ -138,6 +139,30 @@ trait ConversionUtils extends HideToString {
     case _                       => null
   }
 
+  @Documentation(description = "Convert String value to JSON")
+  def toJson(@ParamName("value") value: String): Any = {
+    toJsonEither(value).toTry.get
+  }
+
+  @Documentation(description = "Convert String value to JSON or null in case of failure")
+  def toJsonOrNull(@ParamName("value") value: String): Any = {
+    toJsonEither(value).getOrElse(null)
+  }
+
+  @Documentation(description = "Convert JSON to String")
+  def toJsonString(@ParamName("value") value: Any): String = {
+    jsonEncoder.encode(value).noSpaces
+  }
+
+  private def toJsonEither(value: String): Either[Throwable, Any] = {
+    io.circe.parser.parse(value) match {
+      case Right(json) => Right(JsonUtils.jsonToAny(json))
+      case Left(ex)    => Left(new IllegalArgumentException(s"Cannot convert [$value] to JSON", ex))
+    }
+  }
+
+  private lazy val jsonEncoder = new ToJsonEncoder(true, this.getClass.getClassLoader)
+
 }
 
 object ConversionUtils {

diff --git a/...lt-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/Base64UtilsSpec.scala b/...lt-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/Base64UtilsSpec.scala
@@ -0,0 +1,28 @@
+package pl.touk.nussknacker.engine.util.functions
+
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
+
+class Base64UtilsSpec extends AnyFunSuite with BaseSpelSpec with Matchers {
+
+  test("encode") {
+    evaluate[String]("#BASE64.encode('{\"foo\": 1}')") shouldBe "eyJmb28iOiAxfQ=="
+    evaluate[String]("#BASE64.encode('')") shouldBe ""
+  }
+
+  test("decode") {
+    evaluate[String]("#BASE64.decode('eyJmb28iOiAxfQ==')") shouldBe """{"foo": 1}"""
+    evaluate[String]("#BASE64.decode('')") shouldBe ""
+  }
+
+  test("urlSafeEncode") {
+    evaluate[String]("#BASE64.urlSafeEncode('{\"foo\": 1}')") shouldBe "eyJmb28iOiAxfQ"
+    evaluate[String]("#BASE64.urlSafeEncode('')") shouldBe ""
+  }
+
+  test("urlSafeDecode") {
+    evaluate[String]("#BASE64.urlSafeDecode('eyJmb28iOiAxfQ')") shouldBe """{"foo": 1}"""
+    evaluate[String]("#BASE64.urlSafeDecode('')") shouldBe ""
+  }
+
+}
diff --git a/...fault-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/BaseSpelSpec.scala b/...fault-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/BaseSpelSpec.scala
@@ -27,7 +27,8 @@ trait BaseSpelSpec {
     "DATE_FORMAT" -> new DateFormatUtils(Locale.US),
     "UTIL"        -> util,
     "NUMERIC"     -> numeric,
-    "CONV"        -> conversion
+    "CONV"        -> conversion,
+    "BASE64"      -> base64
   )
 
   private val parser = SpelExpressionParser.default(