-
Notifications
You must be signed in to change notification settings - Fork 47
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
7 changed files
with
131 additions
and
95 deletions.
There are no files selected for viewing
124 changes: 62 additions & 62 deletions
124
kyo-llm-bench/shared/src/main/scala/kyo/llm/bench/BigBenchHard.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,62 +1,62 @@ | ||
| package kyo.llm.bench | ||
|
|
||
| import kyo._ | ||
| import kyo.ios._ | ||
| import kyo.files._ | ||
| import kyo.llm.ais._ | ||
| import kyo.llm.KyoLLMApp | ||
| import kyo.direct._ | ||
| import kyo.seqs.Seqs | ||
| import kyo.llm.configs.Config | ||
| import scala.util.matching.Regex | ||
|
|
||
| object BigBenchHard extends KyoLLMApp { | ||
|
|
||
| val asnwer: Regex = "So the answer is (.*?)\\.".r | ||
|
|
||
| override def config: Config = | ||
| super.config.apiKey("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") | ||
|
|
||
| case class Task(input: String, target: String) | ||
|
|
||
| case class Bench(name: String, total: Int, successes: Int, percent: Double) | ||
|
|
||
| run { | ||
| Seqs.run { | ||
| for { | ||
| cotPrompts <- loadCotPrompts | ||
| (bench, tasks) <- Seqs.get(loadBenchs) | ||
| task <- Seqs.get(tasks) | ||
| output <- AIs.ask(cotPrompts(bench), task.input) | ||
| } yield { | ||
| bench -> | ||
| (if (output.takeRight(task.target.size + 5).contains(task.target)) | ||
| 1 | ||
| else | ||
| 0) | ||
| } | ||
| }.map { r => | ||
| r.groupBy(_._1).map { | ||
| case (name, seq) => | ||
| val total = seq.size | ||
| val successes = seq.map(_._2).sum | ||
| Bench(name, total, successes, successes.toDouble / total) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| def loadBenchs = | ||
| Seqs.traverse( | ||
| Files("BIG-Bench-Hard/bbh").readAll("json").map(_.filter(_._1 == "boolean_expressions")) | ||
| ) { | ||
| case (name, content) => | ||
| case class Examples(examples: List[Task]) | ||
| Json.decode[Examples](content).map { e => | ||
| name -> e.examples | ||
| } | ||
| } | ||
|
|
||
| def loadCotPrompts = | ||
| Files("BIG-Bench-Hard/cot-prompts").readAll("txt").map(_.toMap) | ||
|
|
||
| } | ||
| // package kyo.llm.bench | ||
|
|
||
| // import kyo._ | ||
| // import kyo.ios._ | ||
| // import kyo.files._ | ||
| // import kyo.llm.ais._ | ||
| // import kyo.llm.KyoLLMApp | ||
| // import kyo.direct._ | ||
| // import kyo.seqs.Seqs | ||
| // import kyo.llm.configs.Config | ||
| // import scala.util.matching.Regex | ||
|
|
||
| // object BigBenchHard extends KyoLLMApp { | ||
|
|
||
| // val asnwer: Regex = "So the answer is (.*?)\\.".r | ||
|
|
||
| // override def config: Config = | ||
| // super.config.apiKey("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") | ||
|
|
||
| // case class Task(input: String, target: String) | ||
|
|
||
| // case class Bench(name: String, total: Int, successes: Int, percent: Double) | ||
|
|
||
| // run { | ||
| // Seqs.run { | ||
| // for { | ||
| // cotPrompts <- loadCotPrompts | ||
| // (bench, tasks) <- Seqs.get(loadBenchs) | ||
| // task <- Seqs.get(tasks) | ||
| // output <- AIs.ask(cotPrompts(bench), task.input) | ||
| // } yield { | ||
| // bench -> | ||
| // (if (output.takeRight(task.target.size + 5).contains(task.target)) | ||
| // 1 | ||
| // else | ||
| // 0) | ||
| // } | ||
| // }.map { r => | ||
| // r.groupBy(_._1).map { | ||
| // case (name, seq) => | ||
| // val total = seq.size | ||
| // val successes = seq.map(_._2).sum | ||
| // Bench(name, total, successes, successes.toDouble / total) | ||
| // } | ||
| // } | ||
| // } | ||
|
|
||
| // def loadBenchs = | ||
| // Seqs.traverse( | ||
| // Files("BIG-Bench-Hard/bbh").readAll("json").map(_.filter(_._1 == "boolean_expressions")) | ||
| // ) { | ||
| // case (name, content) => | ||
| // case class Examples(examples: List[Task]) | ||
| // Json.decode[Examples](content).map { e => | ||
| // name -> e.examples | ||
| // } | ||
| // } | ||
|
|
||
| // def loadCotPrompts = | ||
| // Files("BIG-Bench-Hard/cot-prompts").readAll("txt").map(_.toMap) | ||
|
|
||
| // } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
18 changes: 18 additions & 0 deletions
18
kyo-llm/shared/src/main/scala/kyo/llm/thoughts/Humorize.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package kyo.llm.thoughts | ||
|
|
||
| import kyo.llm.ais._ | ||
|
|
||
| @desc( | ||
| p""" | ||
| The Humorize thought directs the LLM in crafting humor within a given context. | ||
| - Explores the context for potential comedic elements. | ||
| - Develops humor that is engaging, witty, and contextually relevant. | ||
| - Ensures that the humor is appropriate and sensitive to the audience. | ||
| """ | ||
| ) | ||
| case class Humorize( | ||
| `Context exploration for humor`: String, | ||
| `Comedic elements identified`: String, | ||
| `Strategy for humorous content creation`: String, | ||
| `Appropriateness and audience sensitivity`: String | ||
| ) extends Thought.Opening |
6 changes: 6 additions & 0 deletions
6
kyo-llm/shared/src/main/scala/kyo/llm/thoughts/RolePlay.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| package kyo.llm.thoughts | ||
|
|
||
| case class RolePlay[Role <: String]( | ||
| `The user has defined the following const string as my role`: Role, | ||
| `Strategy to assume the role`: String | ||
| ) extends Thought.Opening |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters