Skip to content

Commit

Permalink
Remove runtime call (#6)
Browse files Browse the repository at this point in the history
* Chore: Fix CustomPrettifier object name

* Create SparkShowString to enable access to DataFrame#showString

* build.sbt: change release version

* README.md: change release version
  • Loading branch information
fpatin authored Feb 1, 2023
1 parent 4eb3df8 commit 8754008
Show file tree
Hide file tree
Showing 5 changed files with 15 additions and 17 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# SparkTest - 0.2.0
# SparkTest - 0.3.0

**SparkTest** is a Scala library for unit testing with [Spark](https://github.com/apache/spark).
For now, it is only made for DataFrames.
Expand Down
2 changes: 1 addition & 1 deletion build.sbt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
ThisBuild / organization := "com.bedrockstreaming"
ThisBuild / version := "0.2.0"
ThisBuild / version := "0.3.0"
ThisBuild / scalaVersion := "2.12.11"

// ********
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
package com.bedrockstreaming.data.sparktest

import org.apache.spark.sql.DataFrame
import com.bedrockstreaming.data.sparktest.CustomPrettifier.prettyDataFrame
import org.apache.spark.sql.functions.col
import org.apache.spark.sql.{ DataFrame, SparkShowString }
import org.scalactic.Prettifier
import java.lang.reflect.Method

import CustomerPrettifier.prettyDataFrame

trait CustomPrettifier {

Expand All @@ -15,7 +13,7 @@ trait CustomPrettifier {
}
}

object CustomerPrettifier {
object CustomPrettifier {

private[sparktest] def prettyDataFrame(df: DataFrame): String = {
val schemaTitle =
Expand Down Expand Up @@ -48,13 +46,6 @@ object CustomerPrettifier {
numRows: Int = Int.MaxValue,
truncate: Int = 0,
vertical: Boolean = false
): String = {
val methodName = "showString"
val method: Method =
df.getClass.getDeclaredMethod(methodName, numRows.getClass, truncate.getClass, vertical.getClass)
method.setAccessible(true)
method
.invoke(df, numRows.asInstanceOf[Object], truncate.asInstanceOf[Object], vertical.asInstanceOf[Object])
.asInstanceOf[String]
}
): String =
SparkShowString(df, numRows, truncate, vertical)
}
7 changes: 7 additions & 0 deletions src/main/scala/org/apache/spark/sql/SparkShowString.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package org.apache.spark.sql

object SparkShowString {

def apply(df: DataFrame, numRows: Int = Int.MaxValue, truncate: Int = 0, vertical: Boolean = false): String =
df.showString(numRows, truncate, vertical)
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.bedrockstreaming.data.sparktest

import com.bedrockstreaming.data.sparktest.CustomerPrettifier.prettyDataFrame
import com.bedrockstreaming.data.sparktest.CustomPrettifier.prettyDataFrame
import com.bedrockstreaming.data.sparktest.SparkTestTools.SparkSessionOps
import org.apache.spark.sql.types.{ IntegerType, LongType, StringType }
import org.scalatest.flatspec.AnyFlatSpec
Expand Down

0 comments on commit 8754008

Please sign in to comment.