Merge pull request #50 from Herb-AI/dev

Dev
Herb-AI · Jul 4, 2024 · c37099d · c37099d
2 parents df40145 + d4d3573
commit c37099d
Show file tree

Hide file tree

Showing 53,297 changed files with 234,679 additions and 263,043 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -0,0 +1,43 @@
+name: CI
+on:
+  push:
+    branches:
+      - master
+    tags: ['*']
+  pull_request:
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1.8'
+          - '1'
+          - 'nightly'
+        os:
+          - ubuntu-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/checkout@v3
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: julia-actions/cache@v1
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: Herb-AI/HerbBenchmarks.jl
+          files: lcov.info
+          fail_ci_if_error: false
diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml
@@ -0,0 +1,16 @@
+name: CompatHelper
+on:
+  schedule:
+    - cron: 0 0 * * *
+  workflow_dispatch:
+jobs:
+  CompatHelper:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pkg.add("CompatHelper")
+        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+      - name: CompatHelper.main()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
+        run: julia -e 'using CompatHelper; CompatHelper.main()'
diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml
@@ -0,0 +1,31 @@
+name: TagBot
+on:
+  issue_comment:
+    types:
+      - created
+  workflow_dispatch:
+    inputs:
+      lookback:
+        default: 3
+permissions:
+  actions: read
+  checks: read
+  contents: write
+  deployments: read
+  issues: read
+  discussions: read
+  packages: read
+  pages: read
+  pull-requests: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+jobs:
+  TagBot:
+    if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ssh: ${{ secrets.DOCUMENTER_KEY }}
diff --git a/.gitignore b/.gitignore
@@ -22,3 +22,6 @@ docs/site/
 # committed for packages, but should be committed for applications that require a static
 # environment.
 Manifest.toml
+
+# Ignore large data files
+*_large.jl
diff --git a/Project.toml b/Project.toml
@@ -1,4 +1,27 @@
 name = "HerbBenchmarks"
 uuid = "eadf8b74-d38a-4b1a-a063-8d36e493d376"
-authors = ["jaapjong <[email protected]>"]
-version = "0.1.0"
+authors = ["jaapjong <[email protected]>", "Tilman Hinnerichs <[email protected]>", "Sebastijan Dumancic <[email protected]>"]
+version = "0.2.1"
+
+[deps]
+FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
+HerbCore = "2b23ba43-8213-43cb-b5ea-38c12b45bd45"
+HerbGrammar = "4ef9e186-2fe5-4b24-8de7-9f7291f24af7"
+HerbSpecification = "6d54aada-062f-46d8-85cf-a1ceaf058a06"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+MLStyle = "d8e11817-5142-5d16-987a-aa16d5891078"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
+SExpressions = "eaa8e424-c5f6-11e8-1b3d-d576ba0eee97"
+
+[compat]
+HerbCore = "^0.3.0"
+HerbGrammar = "^0.3.0"
+HerbSpecification = "^0.1.0"
+julia = "^1.8"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]
diff --git a/README.md b/README.md
@@ -1,3 +1,41 @@
-# Benchmarks.jl
+[![codecov](https://codecov.io/gh/Herb-AI/HerbBenchmarks.jl/graph/badge.svg?token=VUK6MXLCU4)](https://codecov.io/gh/Herb-AI/HerbBenchmarks.jl)
+[![Build Status](https://github.com/Herb-AI/HerbBenchmarks.jl/actions/workflows/CI.yml/badge.svg?branch=master)](https://github.com/Herb-AI/HerbBenchmarks.jl/actions/workflows/CI.yml?query=branch%3Amaster)
+[![Dev-Docs](https://img.shields.io/badge/docs-latest-blue.svg)](https://Herb-AI.github.io/Herb.jl/dev)
+
+
+# HerbBenchmarks.jl
 
 A collection of useful program synthesis benchmarks. Each folder contains a different benchmark and a README.
+
+A benchmark has:
+- A readme, including description of data fields and dimensionality
+- (multiple) sets of input output examples
+- file that references where we found that
+- evaluation function/interpretation function
+- program grammar
+
+Optional:
+- a data grammar or data generation scripts, ideally with a notion of increasing complexity
+- download script to download that specific dataset, if too big for Github
+- test-/train-split getter functions if specified in the original data
+- landmarks for planning problems
+
+## How to use:
+HerbBenchmarks is still not yet complete and is lacking crucial benchmarking functionality. However, if you want to test on a single problem and grammar, you can do the following
+
+Select your favourite benchmark, we use the string transformation benchmark from the SyGuS challenge:
+```Julia
+using HerbSpecification, HerbGrammar
+
+using HerbBenchmarks.PBE_SLIA_Track_2019
+
+# The id has to be matching
+grammar = PBE_SLIA_Track_2019.grammar_11604909
+problem = PBE_SLIA_Track_2019.problem_11604909
+
+# Print out the grammar and problem in readable format
+println("grammar:", grammar)
+println("problem:", problem.examples)
+```
+
+For some benchmarks there is only a single grammar for all problems. 
diff --git a/e1-robots/Data_generation.jl b/e1-robots/Data_generation.jl
diff --git a/e1-robots/README.md b/e1-robots/README.md
diff --git a/e1-robots/Robotparser.jl b/e1-robots/Robotparser.jl
diff --git a/e1-robots/data/10-0-0.pl b/e1-robots/data/10-0-0.pl
diff --git a/e1-robots/data/10-0-1.pl b/e1-robots/data/10-0-1.pl
diff --git a/e1-robots/data/10-0-10.pl b/e1-robots/data/10-0-10.pl
diff --git a/e1-robots/data/10-0-2.pl b/e1-robots/data/10-0-2.pl
diff --git a/e1-robots/data/10-0-3.pl b/e1-robots/data/10-0-3.pl
diff --git a/e1-robots/data/10-0-4.pl b/e1-robots/data/10-0-4.pl
diff --git a/e1-robots/data/10-0-5.pl b/e1-robots/data/10-0-5.pl
diff --git a/e1-robots/data/10-0-6.pl b/e1-robots/data/10-0-6.pl
diff --git a/e1-robots/data/10-0-7.pl b/e1-robots/data/10-0-7.pl
diff --git a/e1-robots/data/10-0-8.pl b/e1-robots/data/10-0-8.pl
diff --git a/e1-robots/data/10-0-9.pl b/e1-robots/data/10-0-9.pl
diff --git a/e1-robots/data/10-1-0.pl b/e1-robots/data/10-1-0.pl
diff --git a/e1-robots/data/10-1-1.pl b/e1-robots/data/10-1-1.pl
diff --git a/e1-robots/data/10-1-10.pl b/e1-robots/data/10-1-10.pl
diff --git a/e1-robots/data/10-1-2.pl b/e1-robots/data/10-1-2.pl
diff --git a/e1-robots/data/10-1-3.pl b/e1-robots/data/10-1-3.pl
diff --git a/e1-robots/data/10-1-4.pl b/e1-robots/data/10-1-4.pl
diff --git a/e1-robots/data/10-1-5.pl b/e1-robots/data/10-1-5.pl
diff --git a/e1-robots/data/10-1-6.pl b/e1-robots/data/10-1-6.pl
diff --git a/e1-robots/data/10-1-7.pl b/e1-robots/data/10-1-7.pl
diff --git a/e1-robots/data/10-1-8.pl b/e1-robots/data/10-1-8.pl
diff --git a/e1-robots/data/10-1-9.pl b/e1-robots/data/10-1-9.pl
diff --git a/e1-robots/data/10-2-0.pl b/e1-robots/data/10-2-0.pl
diff --git a/e1-robots/data/10-2-1.pl b/e1-robots/data/10-2-1.pl
diff --git a/e1-robots/data/10-2-10.pl b/e1-robots/data/10-2-10.pl
diff --git a/e1-robots/data/10-2-2.pl b/e1-robots/data/10-2-2.pl
diff --git a/e1-robots/data/10-2-3.pl b/e1-robots/data/10-2-3.pl
diff --git a/e1-robots/data/10-2-4.pl b/e1-robots/data/10-2-4.pl
diff --git a/e1-robots/data/10-2-5.pl b/e1-robots/data/10-2-5.pl
diff --git a/e1-robots/data/10-2-6.pl b/e1-robots/data/10-2-6.pl
diff --git a/e1-robots/data/10-2-7.pl b/e1-robots/data/10-2-7.pl
diff --git a/e1-robots/data/10-2-8.pl b/e1-robots/data/10-2-8.pl
diff --git a/e1-robots/data/10-2-9.pl b/e1-robots/data/10-2-9.pl
diff --git a/e1-robots/data/10-3-0.pl b/e1-robots/data/10-3-0.pl
diff --git a/e1-robots/data/10-3-1.pl b/e1-robots/data/10-3-1.pl
diff --git a/e1-robots/data/10-3-10.pl b/e1-robots/data/10-3-10.pl
diff --git a/e1-robots/data/10-3-2.pl b/e1-robots/data/10-3-2.pl
diff --git a/e1-robots/data/10-3-3.pl b/e1-robots/data/10-3-3.pl
diff --git a/e1-robots/data/10-3-4.pl b/e1-robots/data/10-3-4.pl
diff --git a/e1-robots/data/10-3-5.pl b/e1-robots/data/10-3-5.pl
diff --git a/e1-robots/data/10-3-6.pl b/e1-robots/data/10-3-6.pl
diff --git a/e1-robots/data/10-3-7.pl b/e1-robots/data/10-3-7.pl
diff --git a/e1-robots/data/10-3-8.pl b/e1-robots/data/10-3-8.pl
diff --git a/e1-robots/data/10-3-9.pl b/e1-robots/data/10-3-9.pl
diff --git a/e1-robots/data/10-4-0.pl b/e1-robots/data/10-4-0.pl
diff --git a/e1-robots/data/10-4-1.pl b/e1-robots/data/10-4-1.pl
diff --git a/e1-robots/data/10-4-10.pl b/e1-robots/data/10-4-10.pl
diff --git a/e1-robots/data/10-4-2.pl b/e1-robots/data/10-4-2.pl
diff --git a/e1-robots/data/10-4-3.pl b/e1-robots/data/10-4-3.pl
diff --git a/e1-robots/data/10-4-4.pl b/e1-robots/data/10-4-4.pl
diff --git a/e1-robots/data/10-4-5.pl b/e1-robots/data/10-4-5.pl
diff --git a/e1-robots/data/10-4-6.pl b/e1-robots/data/10-4-6.pl
diff --git a/e1-robots/data/10-4-7.pl b/e1-robots/data/10-4-7.pl
diff --git a/e1-robots/data/10-4-8.pl b/e1-robots/data/10-4-8.pl
diff --git a/e1-robots/data/10-4-9.pl b/e1-robots/data/10-4-9.pl
diff --git a/e1-robots/data/10-5-0.pl b/e1-robots/data/10-5-0.pl
diff --git a/e1-robots/data/10-5-1.pl b/e1-robots/data/10-5-1.pl
diff --git a/e1-robots/data/10-5-10.pl b/e1-robots/data/10-5-10.pl
diff --git a/e1-robots/data/10-5-2.pl b/e1-robots/data/10-5-2.pl
diff --git a/e1-robots/data/10-5-3.pl b/e1-robots/data/10-5-3.pl
diff --git a/e1-robots/data/10-5-4.pl b/e1-robots/data/10-5-4.pl
diff --git a/e1-robots/data/10-5-5.pl b/e1-robots/data/10-5-5.pl
diff --git a/e1-robots/data/10-5-6.pl b/e1-robots/data/10-5-6.pl
diff --git a/e1-robots/data/10-5-7.pl b/e1-robots/data/10-5-7.pl
diff --git a/e1-robots/data/10-5-8.pl b/e1-robots/data/10-5-8.pl
diff --git a/e1-robots/data/10-5-9.pl b/e1-robots/data/10-5-9.pl
diff --git a/e1-robots/data/10-6-0.pl b/e1-robots/data/10-6-0.pl