Develop18

.gitignore

@@ -4,6 +4,8 @@
 *.swo
 *.swn
 
+*.iml
+
 # sbt specific
 .cache/
 .history/
@@ -30,3 +32,5 @@ result.csv
 run_config.cfg
 
 tpchdata/
+
+.idea/

components/src/main/scala/ch/epfl/data/dblab/config/Config.scala

@@ -23,7 +23,7 @@ object Config {
   /** Specifies whether to obtain statistics during schema definition */
   val gatherStats: Boolean = false
   /** Specifies whether to show information about the query plan generation during execution */
-  val debugQueryPlan: Boolean = true
+  val debugQueryPlan: Boolean = false
   /** Specifies whether to specialize the loader or not */
   var specializeLoader: Boolean = true
   /** Specifies whether to specialize the query engine or not */

components/src/main/scala/ch/epfl/data/dblab/dbtoaster/Driver.scala

@@ -0,0 +1,158 @@
+package ch.epfl.data
+package dblab
+package dbtoaster
+
+import schema._
+import frontend.parser._
+import frontend.optimizer._
+import frontend.analyzer._
+import utils.Utilities._
+import java.io.PrintStream
+
+import ch.epfl.data.dblab.frontend.parser.CalcAST._
+import ch.epfl.data.dblab.frontend.parser.DDLAST.UseSchema
+import ch.epfl.data.dblab.frontend.parser.SQLAST._
+import ch.epfl.data.sc.pardis.annotations.mutable
+import ch.epfl.data.sc.pardis.types.{ IntType, Tpe }
+import frontend.parser.OperatorAST._
+import config._
+import schema._
+import sc.pardis.language.Language
+
+import scala.collection.mutable.ArrayBuffer
+
+object Driver {
+  /**
+   * The starting point of DBToaster
+   *
+   * @param args the setting arguments passed through command line
+   */
+  def main(args: Array[String]) {
+    if (args.length < 1) {
+      System.out.println("ERROR: Invalid number (" + args.length + ") of command line arguments!")
+      System.out.println("USAGE: dbtoaster <SQL/Calc query> -l SQL|CALC|PLAN|M3 -O")
+      System.out.println("Example: dbtoaster experimentation/dbtoaster/queries/tpch/query6.sql -l M3")
+      System.exit(1)
+    }
+    val options = nextOption(Map(), args.toList)
+    val queryFiles = options.get('queries) match {
+      case Some(l: List[String]) => l
+      case x                     => throw new Exception(s"No queries provided: $x")
+    }
+    val outputLang: Language = options.get('lang).map(_.toString().toUpperCase()) match {
+      case Some("CALC") => Calc
+      case Some("M3")   => M3
+      case Some("SQL")  => SQL
+      case Some("PLAN") => CalcPlan
+      case _ =>
+        throw new Exception("No proper -l defined!")
+    }
+    val shouldOptimize = options.get('opt).map(_.asInstanceOf[Boolean]).getOrElse(false)
+
+    for (q <- queryFiles) {
+      def getCalc(): List[CalcExpr] = if (q.endsWith(".calc")) {
+        CalcParser.parse(scala.io.Source.fromFile(q).mkString)
+
+      } else {
+        val sqlParserTree = SQLParser.parseStream(scala.io.Source.fromFile(q).mkString)
+        val sqlProgram = sqlParserTree.asInstanceOf[IncludeStatement]
+        val tables = sqlProgram.streams.toList.map(x => x.asInstanceOf[CreateStream]) // ok ?
+        val ddlInterpreter = new DDLInterpreter(new Catalog(scala.collection.mutable.Map()))
+        val query = sqlProgram.body
+        // println(query)
+        val schema = ddlInterpreter.interpret(UseSchema("DBToaster") :: tables)
+
+        def listOfQueries(q: TopLevelStatement): List[TopLevelStatement] = {
+          q match {
+            case u: UnionIntersectSequence if u.connectionType.equals(SEQUENCE) =>
+              List(u.top) ++ listOfQueries(u.bottom)
+            case x => List(x)
+          }
+        }
+        val queries = listOfQueries(query)
+
+        val sqlToCalc = new SQLToCalc(schema)
+
+        // sql_to_calc.init()
+        val namer = new SQLNamer(schema)
+        val typer = new SQLTyper(schema)
+        // val tpchSchema = TPCHSchema.getSchema("experimentation/dbtoaster/queries/sf0.001/", 0.0001)
+        // val calcCoster = new CalcCosting(tpchSchema)
+
+        queries.flatMap({ q =>
+          val namedQuery = namer.nameQuery(q)
+          val typedQuery = typer.typeQuery(namedQuery)
+          val calcExpr = sqlToCalc.calcOfQuery(None, typedQuery)
+
+          // println()
+          // println("Costing : ")
+          // println()
+          //          println(schema)
+          //          println()
+          //          println(tpchSchema)
+
+          // calcExpr.foreach({
+          //   case (name, exp) =>
+          //     println(name + ":")
+          //     println(calcCoster.cost(exp))
+          // })
+          // println()
+          // println()
+          // println()
+
+          calcExpr.map({ case (tgt_name, tgt_calc) => tgt_calc })
+        })
+      }
+      def getCalcPlan(calcExprs: List[CalcExpr]): (Plan, List[CalcQuery]) = {
+        val queries = calcExprs.collect({ case ce: CalcQuery => ce })
+        CalcCompiler.compile(Some(1), queries, Schema(ArrayBuffer(), Statistics()))
+      }
+      outputLang match {
+        case Calc =>
+          val calcExprs = getCalc()
+          for (calcExpr <- calcExprs) {
+            calcExpr match {
+              case CalcQuery(x, y) => println(s"$x:\n${prettyprint(y)}")
+              case _               =>
+            }
+          }
+        case CalcPlan =>
+          println("Outputing PLAN")
+          val calcExprs = getCalc()
+          val (plan, qs) = getCalcPlan(calcExprs)
+          for (cds <- plan.list) {
+            println("description")
+            println(cds.description)
+            println("triggers")
+            println(cds.triggers)
+
+          }
+        // println(plan)
+        // println(qs)
+        case M3 =>
+          println("Outputing M3")
+          val calcExprs = getCalc()
+          val (plan, qs) = getCalcPlan(calcExprs)
+          val m3 = PlanToM3.planToM3(Schema(ArrayBuffer(), Statistics()), plan)
+          println(m3)
+        case lang =>
+          throw new Exception(s"Outputing language $lang is not supported yet!")
+      }
+    }
+  }
+
+  type OptionMap = Map[Symbol, Any]
+
+  def nextOption(map: OptionMap, list: List[String]): OptionMap = {
+    list match {
+      case Nil => map
+      case "-O" :: tail =>
+        nextOption(map ++ Map('opt -> true), tail)
+      case "-l" :: value :: tail =>
+        nextOption(map ++ Map('lang -> value), tail)
+      case string :: tail =>
+        println(s"foo-$string")
+        nextOption(map ++ Map('queries -> (map.getOrElse('queries, List()).asInstanceOf[List[String]] :+ string)), tail)
+    }
+  }
+}

components/src/main/scala/ch/epfl/data/dblab/dbtoaster/QueryLanguages.scala

@@ -0,0 +1,10 @@
+package ch.epfl.data
+package dblab
+package dbtoaster
+
+import sc.pardis.language._
+
+case object SQL extends Language
+case object Calc extends Language
+case object CalcPlan extends Language
+case object M3 extends Language

components/src/main/scala/ch/epfl/data/dblab/frontend/analyzer/CalcCosting.scala

@@ -0,0 +1,134 @@
+package ch.epfl.data
+package dblab
+package frontend
+package analyzer
+
+import ch.epfl.data.dblab.frontend.parser.SQLAST
+import parser.CalcAST._
+import sc.pardis.search.CostingContext
+import sc.pardis.ast.Node
+import schema._
+
+import scala.math._
+
+class CalcCosting(schema: Schema) extends CostingContext {
+  def apply(node: Node): Double = cost(node.asInstanceOf[CalcExpr])
+  var count = 0
+
+  def searchCost(x: Double): Double = max(1, scala.math.log(x) / scala.math.log(2))
+
+  def findVars(exp: CalcExpr): scala.collection.mutable.Map[VarT, CalcExpr] = {
+    val vars = scala.collection.mutable.Map[VarT, CalcExpr]()
+    exp match {
+      case CalcProd(lst) => lst.foldLeft(vars)((a, b) => a ++ findVars(b))
+      case CalcSum(lst)  => lst.foldLeft(vars)((a, b) => a ++ findVars(b))
+      case AggSum(v, e)  => findVars(e)
+      case CalcNeg(e)    => findVars(e)
+      case Lift(vr, e)   => vars += (vr -> e)
+      case _             => vars
+    }
+  }
+
+  def cost(exp: CalcExpr): Double = {
+    count += 1
+    val vars = findVars(exp)
+    costExpr(exp, vars)
+  }
+
+  def costExpr(exp: CalcExpr, vars: scala.collection.mutable.Map[VarT, CalcExpr]): Double = {
+
+    exp match {
+      case CalcProd(List()) => 0.0
+      case CalcProd(lst) =>
+        val headCardinality = cardinality(lst.head, vars)._2
+        lst.map(x => costExpr(x, vars)).sum +
+          lst.foldLeft((1.0, 0.0))((a, b) => {
+            if (cardinality(b, vars)._1 == 2)
+              (a._1 * cardinality(b, vars)._2, a._2 + a._1 * cardinality(b, vars)._2)
+            else if (cardinality(b, vars)._1 == 1 && a._1 == 1)
+              (cardinality(b, vars)._2, a._2 + cardinality(b, vars)._2)
+            else if (cardinality(b, vars)._1 == 1 && a._1 != 1)
+              (a._1, a._2 + a._1)
+            else
+              a
+          })._2 - headCardinality
+      case CalcSum(lst) =>
+        val headCardinality = cardinality(lst.head, vars)._2
+        lst.map(x => costExpr(x, vars)).sum + lst.map(x => searchCost(cardinality(x, vars)._2)).sum +
+          headCardinality - searchCost(headCardinality) //TODO what if they had different rows ?
+      case CalcNeg(e)        => costExpr(e, vars) + cardinality(e, vars)._2
+      case AggSum(List(), e) => costExpr(e, vars) + cardinality(e, vars)._2
+      case AggSum(v, e) => costExpr(e, vars) + cardinality(e, vars)._2 *
+        searchCost(cardinality(e, vars)._2) + cardinality(e, vars)._2 //TODO we should multiply sorting part to a constant if we know what algorithm is used
+      case _: Rel                         => cardinality(exp, vars)._2
+      case Cmp(c, first, second)          => costExpr(first, vars) + costExpr(second, vars) + max(cardinality(first, vars)._2, cardinality(second, vars)._2)
+      case External(_, inps, outs, tp, _) => ???
+      case CmpOrList(v, consts)           => ???
+      case Lift(vr, e)                    => searchCost(vars.size)
+      case Exists(term)                   => costExpr(term, vars) + cardinality(term, vars)._2
+      case CalcValue(v: ArithFunc)        => 5.0
+      case CalcValue(v: ArithVar)         => cardinality(v, vars)._2
+      case CalcValue(_)                   => 1.0
+      case ArithVar(v) =>
+        if (vars.contains(v))
+          costExpr(vars(v), vars)
+        else
+          schema.stats.getCardinalityOrElse("PART", 1) //TODO it should change finding size of R in schema
+      case ArithProd(lst) =>
+        lst.map(x => costExpr(x, vars)).sum +
+          lst.foldLeft((1.0, 0.0))((a, b) => {
+            if (cardinality(b, vars)._1 == 1 && a._1 == 1)
+              (cardinality(b, vars)._2, a._2 + a._1)
+            else if (cardinality(b, vars)._1 == 1 && a._1 != 1)
+              (a._1, a._2 + a._1)
+            else
+              a
+          })._2
+
+      case _: ArithFunc                        => 5.0
+      case ArithConst(s: SQLAST.StringLiteral) => s.v.length.toDouble
+      case _: ArithConst                       => 1.0
+    }
+  }
+
+  def cardinality(exp: CalcExpr, vars: scala.collection.mutable.Map[VarT, CalcExpr]): (Int, Double) = {
+    exp match {
+      case CalcProd(lst) => (lst.foldLeft(1)((a, b) => max(a, cardinality(b, vars)._1)), lst.foldLeft(1.0)((a, b) => {
+        if (cardinality(b, vars)._1 == 2)
+          a * cardinality(b, vars)._2
+        else if (cardinality(b, vars)._1 == 2 && a == 1)
+          cardinality(b, vars)._2
+        else
+          a
+      }))
+      case CalcSum(lst) => (lst.foldLeft(1)((a, b) => max(a, cardinality(b, vars)._1)),
+        lst.foldLeft(0.0)((a, b) => max(a, cardinality(b, vars)._2)))
+      case CalcNeg(e)        => cardinality(e, vars)
+      case AggSum(List(), e) => (1, 1)
+      case AggSum(v, e) => (cardinality(e, vars)._1, min(cardinality(e, vars)._2,
+        v.foldLeft(1)((a, b) => schema.stats.getDistinctAttrValuesOrElse(b.name, cardinality(e, vars)._2.toInt) * a)) * 0.5)
+      case Rel(_, name, _, _)             => (2, schema.stats.getCardinalityOrElse("PART", 1)) //TODO it should change to name
+      case Cmp(c, first, second)          => (1, max(cardinality(first, vars)._2, cardinality(second, vars)._2) * 1) // TODO : +we can have some inference here
+      case External(_, inps, outs, tp, _) => ???
+      case CmpOrList(v, consts)           => ???
+      case Lift(vr, e)                    => (0, 1)
+      case Exists(term)                   => cardinality(term, vars) //TODO we can have inference here
+      case CalcValue(v: ArithVar)         => cardinality(v, vars)
+      case CalcValue(_)                   => (1, 1)
+      case ArithVar(v) =>
+        if (vars.contains(v))
+          cardinality(vars(v), vars)
+        else
+          (1, schema.stats.getCardinalityOrElse("PART", 1)) //TODO it should change finding size of R in schema
+      case _: ArithConst => (1, 1)
+      case _: ArithFunc  => (1, 1)
+      case ArithProd(lst) => (lst.foldLeft(1)((a, b) => max(a, cardinality(b, vars)._1)), lst.foldLeft(1.0)((a, b) => {
+        if (cardinality(b, vars)._1 == 2 && a == 1)
+          cardinality(b, vars)._2
+        else
+          a
+      }))
+
+    }
+  }
+}