Skip to content

Commit

Permalink
add StructuredOutputTest + tool use is a suspended function now (#9)
Browse files Browse the repository at this point in the history
  • Loading branch information
morisil authored Oct 15, 2024
1 parent 9374e62 commit d0d48a1
Show file tree
Hide file tree
Showing 6 changed files with 118 additions and 12 deletions.
8 changes: 7 additions & 1 deletion src/commonMain/kotlin/message/Messages.kt
Original file line number Diff line number Diff line change
Expand Up @@ -302,9 +302,15 @@ data class ToolUse(
) : Content() {

@Transient
@PublishedApi
internal lateinit var toolEntry: Anthropic.ToolEntry<UsableTool>

fun use(): ToolResult {
inline fun <reified T> input(): T = anthropicJson.decodeFromJsonElement(
deserializer = toolEntry.serializer as KSerializer<T>,
element = input
)

suspend fun use(): ToolResult {
val tool = anthropicJson.decodeFromJsonElement(
deserializer = toolEntry.serializer,
element = input
Expand Down
4 changes: 1 addition & 3 deletions src/commonMain/kotlin/tool/Tools.kt
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,7 @@ interface UsableTool {
* @param toolUseId A unique identifier for this particular use of the tool.
* @return A [ToolResult] containing the outcome of executing the tool.
*/
fun use(
toolUseId: String
): ToolResult
suspend fun use(toolUseId: String): ToolResult

}

Expand Down
10 changes: 5 additions & 5 deletions src/commonTest/kotlin/test/AnthropicTestTools.kt
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ data class FibonacciTool(val n: Int): UsableTool {
0 -> a; 1 -> b; else -> fibonacci(n - 1, b, a + b)
}

override fun use(
override suspend fun use(
toolUseId: String,
) = ToolResult(toolUseId, "${fibonacci(n)}")

Expand All @@ -43,20 +43,20 @@ data class Calculator(
DIVIDE({ a, b -> a / b })
}

override fun use(toolUseId: String) = ToolResult(
override suspend fun use(toolUseId: String) = ToolResult(
toolUseId,
operation.calculate(a, b).toString()
)

}

interface Database {
fun execute(query: String): List<String>
suspend fun execute(query: String): List<String>
}

class TestDatabase : Database {
var executedQuery: String? = null
override fun execute(
override suspend fun execute(
query: String
): List<String> {
executedQuery = query
Expand All @@ -75,7 +75,7 @@ data class DatabaseQueryTool(
@Transient
lateinit var database: Database

override fun use(
override suspend fun use(
toolUseId: String
) = ToolResult(
toolUseId,
Expand Down
12 changes: 9 additions & 3 deletions src/commonTest/kotlin/tool/UsableToolTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ class UsableToolTest {
class TestTool(
val message: String
) : UsableTool {
override fun use(toolUseId: String) = ToolResult(toolUseId, message)
override suspend fun use(
toolUseId: String
) = ToolResult(toolUseId, message)
}

@Test
Expand Down Expand Up @@ -58,7 +60,9 @@ class UsableToolTest {
}

class NoAnnotationTool : UsableTool {
override fun use(toolUseId: String) = ToolResult(toolUseId, "nothing")
override suspend fun use(
toolUseId: String
) = ToolResult(toolUseId, "nothing")
}

@Test
Expand All @@ -72,7 +76,9 @@ class UsableToolTest {

@Serializable
class OnlySerializableAnnotationTool : UsableTool {
override fun use(toolUseId: String) = ToolResult(toolUseId, "nothing")
override suspend fun use(
toolUseId: String
) = ToolResult(toolUseId, "nothing")
}

@Test
Expand Down
96 changes: 96 additions & 0 deletions src/jvmTest/kotlin/StructuredOutputTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package com.xemantic.anthropic

import com.xemantic.anthropic.message.*
import com.xemantic.anthropic.tool.AnthropicTool
import com.xemantic.anthropic.tool.UsableTool
import io.kotest.assertions.assertSoftly
import io.kotest.matchers.collections.shouldHaveSize
import io.kotest.matchers.shouldBe
import kotlinx.coroutines.test.runTest
import kotlinx.serialization.Serializable
import kotlin.test.Test

/**
* This test tool is based on the
* [article by Dan Nguyen](https://gist.github.com/dannguyen/faaa56cebf30ad51108a9fe4f8db36d8),
* who showed how to extract financial disclosure reports as structured data by using OpenAI API.
* I wanted to try out the same approach with Anthropic API, and it seems like a great test case of this library.
*/
@AnthropicTool(
name = "DisclosureReport",
description = "Extract the text from this image"
)
class DisclosureReport(
val assets: List<Asset>
) : UsableTool {
override suspend fun use(toolUseId: String) = ToolResult(
toolUseId, "Data provided to client"
)
}

@Serializable
data class Asset(
val assetName: String,
val owner: String,
val location: String?,
val assetValueLow: Int?,
val assetValueHigh: Int?,
val incomeType: String,
val incomeLow: Int?,
val incomeHigh: Int?,
val txGt1000: Boolean
)

/**
* This test is located in the jvmTest folder, so it can use File API to read image files.
* In the future it can probably be moved to jvmAndPosix to support all the Kotlin platforms
* having access to the filesystem.
*/
class StructuredOutputTest {

@Test
fun shouldDecodeStructuredOutputFromReportImage() = runTest {
val client = Anthropic {
tool<DisclosureReport>()
}

val response = client.messages.create {
+Message {
+"Decode structured output from supplied image"
+Image(
path = "test-data/financial-disclosure-report.png",
mediaType = Image.MediaType.IMAGE_PNG
)
}
useTool<DisclosureReport>()
}

val tool = response.content.filterIsInstance<ToolUse>().first()
val report = tool.input<DisclosureReport>()

report.assets shouldHaveSize 6
assertSoftly(report.assets[0]) {
assetName shouldBe "11 Zinfandel Lane - Home & Vineyard [RP]"
owner shouldBe "JT"
location shouldBe "St. Helena/Napa, CA, US"
assetValueLow shouldBe 5000001
assetValueHigh shouldBe 25000000
incomeType shouldBe "Grape Sales"
incomeLow shouldBe 100001
incomeHigh shouldBe 1000000
txGt1000 shouldBe false
}
assertSoftly(report.assets[1]) {
assetName shouldBe "25 Point Lobos - Commercial Property [RP]"
owner shouldBe "SP"
location shouldBe "San Francisco/San Francisco, CA, US"
assetValueLow shouldBe 5000001
assetValueHigh shouldBe 25000000
incomeType shouldBe "Rent"
incomeLow shouldBe 100001
incomeHigh shouldBe 1000000
txGt1000 shouldBe false
}
}

}
Binary file added test-data/financial-disclosure-report.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit d0d48a1

Please sign in to comment.