Upload New File

RTU_Sample/src/main/scala/rtu/StatisticsProcess.scala

+package rtu
+
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.{DataFrame, Encoder, SparkSession}
+
+import scala.collection.mutable
+
+trait StatisticsProcess {
+  var name : String
+  var isLocal : Boolean
+  var spark : SparkSession
+  var properties : mutable.Map[String, String]
+
+  def init(_name : String, _isLocal : Boolean, prop : mutable.Map[String, String]): Unit ={
+    name = _name
+    isLocal = _isLocal
+    val sparkBuilder = SparkSession.builder()
+
+    if (_isLocal) {
+      spark = sparkBuilder.appName(name)
+        .master("local[*]")
+        .getOrCreate()
+    }
+    else {
+      spark = sparkBuilder.appName(name)
+        .getOrCreate()
+    }
+
+
+    properties = prop
+  }
+
+  def printlnPartitionSize(name : String, dataFrame : DataFrame): Unit =
+  {
+    val encoder : Encoder[Int] = ExpressionEncoder[Int]()
+    println(name + " partition size : " +  dataFrame.mapPartitions{it=>Iterator(it.toSeq.size)}(encoder).collect.toSeq)
+  }
+
+  def process(date : String)
+}