Merge branch 'main' of github.com:getkyo/kyo

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "kyo-llm-bench/jvm/BIG-Bench-Hard"]
+	path = kyo-llm-bench/jvm/BIG-Bench-Hard
+	url = https://github.com/suzgunmirac/BIG-Bench-Hard

build.sbt

@@ -200,6 +200,18 @@ lazy val `kyo-cache` =
         libraryDependencies += "com.github.ben-manes.caffeine" % "caffeine" % "3.1.8"
     )
 
+lazy val `kyo-os-lib` =
+  crossProject(JVMPlatform)
+    .withoutSuffixFor(JVMPlatform)
+    .crossType(CrossType.Full)
+    .in(file("kyo-os-lib"))
+    .dependsOn(`kyo-core` % "test->test;compile->compile")
+    .settings(
+        `kyo-settings`,
+        `with-cross-scala`,
+        libraryDependencies += "com.lihaoyi" %% "os-lib" % "0.9.2"
+    )
+
 lazy val `kyo-sttp` =
   crossProject(JSPlatform, JVMPlatform)
     .withoutSuffixFor(JVMPlatform)
@@ -266,6 +278,20 @@ lazy val `kyo-llm` =
     )
     .jsSettings(`js-settings`)
 
+lazy val `kyo-llm-bench` =
+  crossProject(JVMPlatform)
+    .withoutSuffixFor(JVMPlatform)
+    .crossType(CrossType.Full)
+    .in(file("kyo-llm-bench"))
+    .dependsOn(`kyo-llm`)
+    .dependsOn(`kyo-os-lib`)
+    .dependsOn(`kyo-core` % "test->test;compile->compile")
+    .settings(
+        `kyo-settings`,
+        `without-cross-scala`,
+        libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.4.14"
+    )
+
 lazy val `kyo-bench` =
   crossProject(JVMPlatform)
     .withoutSuffixFor(JVMPlatform)

kyo-llm-bench/shared/src/main/scala/kyo/llm/bench/BigBenchHard.scala

@@ -0,0 +1,62 @@
+package kyo.llm.bench
+
+import kyo._
+import kyo.ios._
+import kyo.files._
+import kyo.llm.ais._
+import kyo.llm.KyoLLMApp
+import kyo.direct._
+import kyo.seqs.Seqs
+import kyo.llm.configs.Config
+import scala.util.matching.Regex
+
+object BigBenchHard extends KyoLLMApp {
+
+  val asnwer: Regex = "So the answer is (.*?)\\.".r
+
+  override def config: Config =
+    super.config.apiKey("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")
+
+  case class Task(input: String, target: String)
+
+  case class Bench(name: String, total: Int, successes: Int, percent: Double)
+
+  run {
+    Seqs.run {
+      for {
+        cotPrompts     <- loadCotPrompts
+        (bench, tasks) <- Seqs.get(loadBenchs)
+        task           <- Seqs.get(tasks)
+        output         <- AIs.ask(cotPrompts(bench), task.input)
+      } yield {
+        bench ->
+          (if (output.takeRight(task.target.size + 5).contains(task.target))
+             1
+           else
+             0)
+      }
+    }.map { r =>
+      r.groupBy(_._1).map {
+        case (name, seq) =>
+          val total     = seq.size
+          val successes = seq.map(_._2).sum
+          Bench(name, total, successes, successes.toDouble / total)
+      }
+    }
+  }
+
+  def loadBenchs =
+    Seqs.traverse(
+        Files("BIG-Bench-Hard/bbh").readAll("json").map(_.filter(_._1 == "boolean_expressions"))
+    ) {
+      case (name, content) =>
+        case class Examples(examples: List[Task])
+        Json.decode[Examples](content).map { e =>
+          name -> e.examples
+        }
+    }
+
+  def loadCotPrompts =
+    Files("BIG-Bench-Hard/cot-prompts").readAll("txt").map(_.toMap)
+
+}

kyo-os-lib/shared/src/main/scala/kyo/files.scala

@@ -0,0 +1,144 @@
+package kyo
+
+import kyo._
+import kyo.ios._
+import scala.io._
+import os._
+
+object files {
+
+  type Part = String | Files
+
+  class Files(val path: List[String]) {
+
+    def parts: List[Part] = path
+
+    def osPath = path.foldLeft(os.pwd)(_ / _)
+
+    def read: String < IOs =
+      IOs(os.read(osPath))
+
+    def read(
+        charSet: Codec = java.nio.charset.StandardCharsets.UTF_8,
+        offset: Long = 0,
+        count: Int = Int.MaxValue
+    ): String < IOs =
+      IOs(os.read(osPath, charSet, offset, count))
+
+    def readAll(extension: String): IndexedSeq[(String, String)] < IOs =
+      IOs(os.list(osPath).filter(_.ext == extension).map(p => p.baseName -> os.read(p)))
+
+    def readBytes: Array[Byte] < IOs =
+      IOs(os.read.bytes(osPath))
+
+    def readBytes(
+        offset: Long = 0,
+        count: Int = Int.MaxValue
+    )(path: String*): Array[Byte] < IOs =
+      IOs(os.read.bytes(osPath, offset, count))
+
+    def readLines: IndexedSeq[String] < IOs =
+      IOs(os.read.lines(osPath))
+
+    def readLines(
+        charSet: Codec = java.nio.charset.StandardCharsets.UTF_8
+    )(path: String*): IndexedSeq[String] < IOs =
+      IOs(os.read.lines(osPath, charSet))
+
+    def truncate(size: Long): Unit < IOs =
+      IOs(os.truncate(osPath, size))
+
+    def append(value: String, perms: PermSet = null, createFolders: Boolean = true) =
+      IOs(os.write.append(osPath, value, perms, createFolders))
+
+    def write(value: String, perms: PermSet = null, createFolders: Boolean = true) =
+      IOs(os.write(osPath, value, perms, createFolders))
+
+    def list: IndexedSeq[Files] < IOs =
+      list(true)
+
+    def list(sort: Boolean): IndexedSeq[Files] < IOs =
+      IOs(os.list(osPath, sort).map(p => new Files(p.segments.toList)))
+
+    def list(extension: String): IndexedSeq[Files] < IOs =
+      IOs(os.list(osPath).filter(_.last.endsWith(extension)).map(p => new Files(p.segments.toList)))
+
+    def isDir: Boolean < IOs =
+      IOs(os.isDir(osPath))
+
+    def isFile: Boolean < IOs =
+      IOs(os.isFile(osPath))
+
+    def isLink: Boolean < IOs =
+      IOs(os.isLink(osPath))
+
+    def mkDir: Unit < IOs =
+      IOs(os.makeDir.all(osPath))
+
+    def move(
+        to: Files,
+        replaceExisting: Boolean = false,
+        atomicMove: Boolean = false,
+        createFolders: Boolean = true
+    ) =
+      IOs(os.move(osPath, to.osPath, atomicMove, createFolders))
+
+    def copy(
+        to: Files,
+        followLinks: Boolean = true,
+        replaceExisting: Boolean = false,
+        copyAttributes: Boolean = false,
+        createFolders: Boolean = true,
+        mergeFolders: Boolean = false
+    ): Unit < IOs =
+      IOs(os.copy(
+          osPath,
+          to.osPath,
+          followLinks,
+          replaceExisting,
+          copyAttributes,
+          createFolders,
+          mergeFolders
+      ))
+
+    def remove: Boolean < IOs =
+      remove(false)
+
+    def remove(checkExists: Boolean): Boolean < IOs =
+      IOs(os.remove(osPath, checkExists))
+
+    def removeAll: Unit < IOs =
+      IOs(os.remove.all(osPath))
+
+    def exists: Boolean < IOs =
+      exists(true)
+
+    def exists(followLinks: Boolean): Boolean < IOs =
+      IOs(os.exists(osPath, followLinks))
+
+    override def toString = s"Files(\"${path.mkString("/")}\")"
+  }
+
+  object Files {
+
+    def apply(path: List[Part]): Files = {
+      def loop(path: List[Part], acc: List[String]): List[String] =
+        path match {
+          case Nil =>
+            acc.reverse
+          case h :: t =>
+            h match {
+              case h: String =>
+                loop(t, h.split('/').toList.reverse ::: acc)
+              case h: Files =>
+                loop(h.path ::: t, acc)
+            }
+        }
+      new Files(loop(path, Nil))
+    }
+
+    def apply(path: Part*): Files =
+      apply(path.toList)
+  }
+
+}