diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..2c13d887c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "kyo-llm-bench/jvm/BIG-Bench-Hard"] + path = kyo-llm-bench/jvm/BIG-Bench-Hard + url = https://github.com/suzgunmirac/BIG-Bench-Hard diff --git a/build.sbt b/build.sbt index efb0f2b55..7c06929a3 100644 --- a/build.sbt +++ b/build.sbt @@ -200,6 +200,18 @@ lazy val `kyo-cache` = libraryDependencies += "com.github.ben-manes.caffeine" % "caffeine" % "3.1.8" ) +lazy val `kyo-os-lib` = + crossProject(JVMPlatform) + .withoutSuffixFor(JVMPlatform) + .crossType(CrossType.Full) + .in(file("kyo-os-lib")) + .dependsOn(`kyo-core` % "test->test;compile->compile") + .settings( + `kyo-settings`, + `with-cross-scala`, + libraryDependencies += "com.lihaoyi" %% "os-lib" % "0.9.2" + ) + lazy val `kyo-sttp` = crossProject(JSPlatform, JVMPlatform) .withoutSuffixFor(JVMPlatform) @@ -266,6 +278,20 @@ lazy val `kyo-llm` = ) .jsSettings(`js-settings`) +lazy val `kyo-llm-bench` = + crossProject(JVMPlatform) + .withoutSuffixFor(JVMPlatform) + .crossType(CrossType.Full) + .in(file("kyo-llm-bench")) + .dependsOn(`kyo-llm`) + .dependsOn(`kyo-os-lib`) + .dependsOn(`kyo-core` % "test->test;compile->compile") + .settings( + `kyo-settings`, + `without-cross-scala`, + libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.4.14" + ) + lazy val `kyo-bench` = crossProject(JVMPlatform) .withoutSuffixFor(JVMPlatform) diff --git a/kyo-llm-bench/jvm/BIG-Bench-Hard b/kyo-llm-bench/jvm/BIG-Bench-Hard new file mode 160000 index 000000000..9ee07bd48 --- /dev/null +++ b/kyo-llm-bench/jvm/BIG-Bench-Hard @@ -0,0 +1 @@ +Subproject commit 9ee07bd481feebf959a6b59d61ea57bdcf30964d diff --git a/kyo-llm-bench/shared/src/main/scala/kyo/llm/bench/BigBenchHard.scala b/kyo-llm-bench/shared/src/main/scala/kyo/llm/bench/BigBenchHard.scala new file mode 100644 index 000000000..efd4f050d --- /dev/null +++ b/kyo-llm-bench/shared/src/main/scala/kyo/llm/bench/BigBenchHard.scala @@ -0,0 +1,62 @@ +package kyo.llm.bench + +import kyo._ +import kyo.ios._ +import kyo.files._ +import kyo.llm.ais._ +import kyo.llm.KyoLLMApp +import kyo.direct._ +import kyo.seqs.Seqs +import kyo.llm.configs.Config +import scala.util.matching.Regex + +object BigBenchHard extends KyoLLMApp { + + val asnwer: Regex = "So the answer is (.*?)\\.".r + + override def config: Config = + super.config.apiKey("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") + + case class Task(input: String, target: String) + + case class Bench(name: String, total: Int, successes: Int, percent: Double) + + run { + Seqs.run { + for { + cotPrompts <- loadCotPrompts + (bench, tasks) <- Seqs.get(loadBenchs) + task <- Seqs.get(tasks) + output <- AIs.ask(cotPrompts(bench), task.input) + } yield { + bench -> + (if (output.takeRight(task.target.size + 5).contains(task.target)) + 1 + else + 0) + } + }.map { r => + r.groupBy(_._1).map { + case (name, seq) => + val total = seq.size + val successes = seq.map(_._2).sum + Bench(name, total, successes, successes.toDouble / total) + } + } + } + + def loadBenchs = + Seqs.traverse( + Files("BIG-Bench-Hard/bbh").readAll("json").map(_.filter(_._1 == "boolean_expressions")) + ) { + case (name, content) => + case class Examples(examples: List[Task]) + Json.decode[Examples](content).map { e => + name -> e.examples + } + } + + def loadCotPrompts = + Files("BIG-Bench-Hard/cot-prompts").readAll("txt").map(_.toMap) + +} diff --git a/kyo-os-lib/shared/src/main/scala/kyo/files.scala b/kyo-os-lib/shared/src/main/scala/kyo/files.scala new file mode 100644 index 000000000..05d15a93a --- /dev/null +++ b/kyo-os-lib/shared/src/main/scala/kyo/files.scala @@ -0,0 +1,144 @@ +package kyo + +import kyo._ +import kyo.ios._ +import scala.io._ +import os._ + +object files { + + type Part = String | Files + + class Files(val path: List[String]) { + + def parts: List[Part] = path + + def osPath = path.foldLeft(os.pwd)(_ / _) + + def read: String < IOs = + IOs(os.read(osPath)) + + def read( + charSet: Codec = java.nio.charset.StandardCharsets.UTF_8, + offset: Long = 0, + count: Int = Int.MaxValue + ): String < IOs = + IOs(os.read(osPath, charSet, offset, count)) + + def readAll(extension: String): IndexedSeq[(String, String)] < IOs = + IOs(os.list(osPath).filter(_.ext == extension).map(p => p.baseName -> os.read(p))) + + def readBytes: Array[Byte] < IOs = + IOs(os.read.bytes(osPath)) + + def readBytes( + offset: Long = 0, + count: Int = Int.MaxValue + )(path: String*): Array[Byte] < IOs = + IOs(os.read.bytes(osPath, offset, count)) + + def readLines: IndexedSeq[String] < IOs = + IOs(os.read.lines(osPath)) + + def readLines( + charSet: Codec = java.nio.charset.StandardCharsets.UTF_8 + )(path: String*): IndexedSeq[String] < IOs = + IOs(os.read.lines(osPath, charSet)) + + def truncate(size: Long): Unit < IOs = + IOs(os.truncate(osPath, size)) + + def append(value: String, perms: PermSet = null, createFolders: Boolean = true) = + IOs(os.write.append(osPath, value, perms, createFolders)) + + def write(value: String, perms: PermSet = null, createFolders: Boolean = true) = + IOs(os.write(osPath, value, perms, createFolders)) + + def list: IndexedSeq[Files] < IOs = + list(true) + + def list(sort: Boolean): IndexedSeq[Files] < IOs = + IOs(os.list(osPath, sort).map(p => new Files(p.segments.toList))) + + def list(extension: String): IndexedSeq[Files] < IOs = + IOs(os.list(osPath).filter(_.last.endsWith(extension)).map(p => new Files(p.segments.toList))) + + def isDir: Boolean < IOs = + IOs(os.isDir(osPath)) + + def isFile: Boolean < IOs = + IOs(os.isFile(osPath)) + + def isLink: Boolean < IOs = + IOs(os.isLink(osPath)) + + def mkDir: Unit < IOs = + IOs(os.makeDir.all(osPath)) + + def move( + to: Files, + replaceExisting: Boolean = false, + atomicMove: Boolean = false, + createFolders: Boolean = true + ) = + IOs(os.move(osPath, to.osPath, atomicMove, createFolders)) + + def copy( + to: Files, + followLinks: Boolean = true, + replaceExisting: Boolean = false, + copyAttributes: Boolean = false, + createFolders: Boolean = true, + mergeFolders: Boolean = false + ): Unit < IOs = + IOs(os.copy( + osPath, + to.osPath, + followLinks, + replaceExisting, + copyAttributes, + createFolders, + mergeFolders + )) + + def remove: Boolean < IOs = + remove(false) + + def remove(checkExists: Boolean): Boolean < IOs = + IOs(os.remove(osPath, checkExists)) + + def removeAll: Unit < IOs = + IOs(os.remove.all(osPath)) + + def exists: Boolean < IOs = + exists(true) + + def exists(followLinks: Boolean): Boolean < IOs = + IOs(os.exists(osPath, followLinks)) + + override def toString = s"Files(\"${path.mkString("/")}\")" + } + + object Files { + + def apply(path: List[Part]): Files = { + def loop(path: List[Part], acc: List[String]): List[String] = + path match { + case Nil => + acc.reverse + case h :: t => + h match { + case h: String => + loop(t, h.split('/').toList.reverse ::: acc) + case h: Files => + loop(h.path ::: t, acc) + } + } + new Files(loop(path, Nil)) + } + + def apply(path: Part*): Files = + apply(path.toList) + } + +}