From a2026d926b40a24e115351ebc36c07c69db56422 Mon Sep 17 00:00:00 2001 From: Nicolae Filat Date: Fri, 30 Jun 2023 10:10:17 +0200 Subject: [PATCH 1/2] Add package files --- .github/workflows/.github/workflows/CI.yml | 35 ++++ .../.github/workflows/CompatHelper.yml | 16 ++ .../workflows/.github/workflows/TagBot.yml | 31 +++ .github/workflows/CI.yml | 35 ++++ .github/workflows/CompatHelper.yml | 16 ++ .github/workflows/TagBot.yml | 31 +++ Project.toml | 12 +- README.md | 5 + src/sampling.jl | 181 ++++++++++++++++++ test/runtests.jl | 7 + 10 files changed, 367 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/.github/workflows/CI.yml create mode 100644 .github/workflows/.github/workflows/CompatHelper.yml create mode 100644 .github/workflows/.github/workflows/TagBot.yml create mode 100644 .github/workflows/CI.yml create mode 100644 .github/workflows/CompatHelper.yml create mode 100644 .github/workflows/TagBot.yml create mode 100644 src/sampling.jl create mode 100644 test/runtests.jl diff --git a/.github/workflows/.github/workflows/CI.yml b/.github/workflows/.github/workflows/CI.yml new file mode 100644 index 0000000..53f53a8 --- /dev/null +++ b/.github/workflows/.github/workflows/CI.yml @@ -0,0 +1,35 @@ +name: CI +on: + push: + branches: + - master + tags: ['*'] + pull_request: +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.8' + - 'nightly' + os: + - ubuntu-latest + arch: + - x64 + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v1 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 diff --git a/.github/workflows/.github/workflows/CompatHelper.yml b/.github/workflows/.github/workflows/CompatHelper.yml new file mode 100644 index 0000000..cba9134 --- /dev/null +++ b/.github/workflows/.github/workflows/CompatHelper.yml @@ -0,0 +1,16 @@ +name: CompatHelper +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} + run: julia -e 'using CompatHelper; CompatHelper.main()' diff --git a/.github/workflows/.github/workflows/TagBot.yml b/.github/workflows/.github/workflows/TagBot.yml new file mode 100644 index 0000000..2bacdb8 --- /dev/null +++ b/.github/workflows/.github/workflows/TagBot.yml @@ -0,0 +1,31 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: + inputs: + lookback: + default: 3 +permissions: + actions: read + checks: read + contents: write + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..53f53a8 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,35 @@ +name: CI +on: + push: + branches: + - master + tags: ['*'] + pull_request: +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.8' + - 'nightly' + os: + - ubuntu-latest + arch: + - x64 + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v1 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 0000000..cba9134 --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,16 @@ +name: CompatHelper +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} + run: julia -e 'using CompatHelper; CompatHelper.main()' diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml new file mode 100644 index 0000000..2bacdb8 --- /dev/null +++ b/.github/workflows/TagBot.yml @@ -0,0 +1,31 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: + inputs: + lookback: + default: 3 +permissions: + actions: read + checks: read + contents: write + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/Project.toml b/Project.toml index 61bd1c5..7b7dd74 100644 --- a/Project.toml +++ b/Project.toml @@ -1,15 +1,23 @@ name = "HerbGrammar" uuid = "4ef9e186-2fe5-4b24-8de7-9f7291f24af7" -authors = ["Sebastijan Dumancic ", "Jaap de Jong "] +authors = ["Sebastijan Dumancic ", "Jaap de Jong ", "Nicolae Filat ", "Piotr Cichoń "] version = "0.1.0" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" TreeView = "39424ebd-4cf3-5550-a685-96706a953f40" +HerbCore = "2b23ba43-8213-43cb-b5ea-38c12b45bd45" [compat] AbstractTrees = "0.4" DataStructures = "0.17,0.18" TreeView = "0.5" -julia = "1" +HerbCore = "0.1.0" +julia = "1.8" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/README.md b/README.md index 483f346..6d29e91 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Grammar.jl +[![Build Status](https://github.com/Herb-AI/HerbGrammar.jl/actions/workflows/CI.yml/badge.svg?branch=master)](https://github.com/Herb-AI/HerbGrammar.jl/actions/workflows/CI.yml?query=branch%3Amaster) This package contains functionality for declaring grammars for the Herb Program Synthesis framework. @@ -8,3 +9,7 @@ To use this project, initialize the project with ```shell julia --project=. -e 'using Pkg; Pkg.instantiate()' ``` + + + + diff --git a/src/sampling.jl b/src/sampling.jl new file mode 100644 index 0000000..c141ca3 --- /dev/null +++ b/src/sampling.jl @@ -0,0 +1,181 @@ +using StatsBase +""" + Contains all function for sampling expressions and from expressions +""" + +""" + rand(::Type{RuleNode}, grammar::Grammar, typ::Symbol, max_depth::Int=10) + +Generates a random RuleNode of return type typ and maximum depth max_depth. +""" +function Base.rand(::Type{RuleNode}, grammar::Grammar, typ::Symbol, max_depth::Int=10, + bin::Union{NodeRecycler,Nothing}=nothing) + dmap = mindepth_map(grammar) + return rand(RuleNode, grammar, typ, dmap, max_depth) +end + +""" + rand(::Type{RuleNode}, grammar::Grammar, max_depth::Int=10) + +Generates a random RuleNode with any type and maximum depth max_depth. +""" +function Base.rand(::Type{RuleNode}, grammar::Grammar, max_depth::Int=10, + bin::Union{NodeRecycler,Nothing}=nothing) + dmap = mindepth_map(grammar) + random_start_symbol = StatsBase.sample(grammar.types) + return rand(RuleNode, grammar, random_start_symbol, dmap, max_depth) +end + + +""" + rand(::Type{RuleNode}, grammar::Grammar, typ::Symbol, dmap::AbstractVector{Int}, max_depth::Int=10) + +Generates a random RuleNode of return type typ and maximum depth max_depth guided by a minimum depth map dmap. +""" +function Base.rand(::Type{RuleNode}, grammar::Grammar, typ::Symbol, dmap::AbstractVector{Int}, + max_depth::Int=10, bin::Union{NodeRecycler,Nothing}=nothing) + rules = grammar[typ] + filtered = filter(r->dmap[r] ≤ max_depth, rules) + if isempty(filtered) + error("The random function could not find an expression of the given $max_depth depth") + return + end + + rule_index = StatsBase.sample(filtered) + @assert max_depth >= 0 + + rulenode = iseval(grammar, rule_index) ? + RuleNode(bin, rule_index, Core.eval(grammar, rule_index)) : + RuleNode(bin, rule_index) + + if !grammar.isterminal[rule_index] + for ch in child_types(grammar, rule_index) + push!(rulenode.children, rand(RuleNode, grammar, ch, dmap, max_depth-1, bin)) + end + end + return rulenode +end + +mutable struct RuleNodeAndCount + node::RuleNode + cnt::Int +end +""" + sample(root::RuleNode, typ::Symbol, grammar::Grammar, maxdepth::Int=typemax(Int)) + +Selects a uniformly random node from the tree, limited to maxdepth. +""" +function StatsBase.sample(root::RuleNode, maxdepth::Int=typemax(Int)) + x = RuleNodeAndCount(root, 1) + for child in root.children + _sample(child, x, maxdepth-1) + end + x.node +end +function _sample(node::RuleNode, x::RuleNodeAndCount, maxdepth::Int) + maxdepth < 1 && return + x.cnt += 1 + if rand() <= 1/x.cnt + x.node = node + end + for child in node.children + _sample(child, x, maxdepth-1) + end +end + +""" + sample(root::RuleNode, typ::Symbol, grammar::Grammar, + maxdepth::Int=typemax(Int)) + +Selects a uniformly random node of the given return type, typ, limited to maxdepth. +""" +function StatsBase.sample(root::RuleNode, typ::Symbol, grammar::Grammar, + maxdepth::Int=typemax(Int)) + x = RuleNodeAndCount(root, 0) + if grammar.types[root.ind] == typ + x.cnt += 1 + end + for child in root.children + _sample(child, typ, grammar, x, maxdepth-1) + end + grammar.types[x.node.ind] == typ || error("type $typ not found in RuleNode") + x.node +end +function _sample(node::RuleNode, typ::Symbol, grammar::Grammar, x::RuleNodeAndCount, + maxdepth::Int) + maxdepth < 1 && return + if grammar.types[node.ind] == typ + x.cnt += 1 + if rand() <= 1/x.cnt + x.node = node + end + end + for child in node.children + _sample(child, typ, grammar, x, maxdepth-1) + end +end + + + + +mutable struct NodeLocAndCount + loc::NodeLoc + cnt::Int +end + + +""" + sample(::Type{NodeLoc}, root::RuleNode, maxdepth::Int=typemax(Int)) + +Selects a uniformly random node in the tree no deeper than maxdepth using reservoir sampling. +Returns a NodeLoc that specifies the location using its parent so that the subtree can be replaced. +""" + +function StatsBase.sample(::Type{NodeLoc}, root::RuleNode, maxdepth::Int=typemax(Int)) + x = NodeLocAndCount(root_node_loc(root), 1) + _sample(NodeLoc, root, x, maxdepth-1) + x.loc +end + + +function _sample(::Type{NodeLoc}, node::RuleNode, x::NodeLocAndCount, maxdepth::Int) + maxdepth < 1 && return + for (j,child) in enumerate(node.children) + x.cnt += 1 + if rand() <= 1/x.cnt + x.loc = NodeLoc(node, j) + end + _sample(NodeLoc, child, x, maxdepth-1) + end +end + +""" + sample(::Type{NodeLoc}, root::RuleNode, typ::Symbol, grammar::Grammar) + +Selects a uniformly random node in the tree of a given type, specified using its parent such that the subtree can be replaced. +Returns a NodeLoc. +""" +function StatsBase.sample(::Type{NodeLoc}, root::RuleNode, typ::Symbol, grammar::Grammar, + maxdepth::Int=typemax(Int)) + x = NodeLocAndCount(root_node_loc(root), 0) + if grammar.types[root.ind] == typ + x.cnt += 1 + end + _sample(NodeLoc, root, typ, grammar, x, maxdepth-1) + grammar.types[get(root,x.loc).ind] == typ || error("type $typ not found in RuleNode") + x.loc +end + +function _sample(::Type{NodeLoc}, node::RuleNode, typ::Symbol, grammar::Grammar, + x::NodeLocAndCount, maxdepth::Int) + maxdepth < 1 && return + for (j,child) in enumerate(node.children) + if grammar.types[child.ind] == typ + x.cnt += 1 + if rand() <= 1/x.cnt + x.loc = NodeLoc(node, j) + end + end + _sample(NodeLoc, child, typ, grammar, x, maxdepth-1) + end +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl new file mode 100644 index 0000000..24932a5 --- /dev/null +++ b/test/runtests.jl @@ -0,0 +1,7 @@ +using HerbGrammar +using Test + +@testset "HerbGrammar.jl" verbose=true begin + include("test_cfg.jl") + include("test_csg.jl") +end From 950fb7b2115e248cd36b7c2084486289be8f059c Mon Sep 17 00:00:00 2001 From: Nicolae Filat Date: Fri, 30 Jun 2023 11:51:13 +0200 Subject: [PATCH 2/2] Added github actions --- workflows/CI.yml | 35 +++++++++++++++++++++++++++++++++++ workflows/CompatHelper.yml | 16 ++++++++++++++++ workflows/TagBot.yml | 31 +++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 workflows/CI.yml create mode 100644 workflows/CompatHelper.yml create mode 100644 workflows/TagBot.yml diff --git a/workflows/CI.yml b/workflows/CI.yml new file mode 100644 index 0000000..53f53a8 --- /dev/null +++ b/workflows/CI.yml @@ -0,0 +1,35 @@ +name: CI +on: + push: + branches: + - master + tags: ['*'] + pull_request: +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.8' + - 'nightly' + os: + - ubuntu-latest + arch: + - x64 + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v1 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 diff --git a/workflows/CompatHelper.yml b/workflows/CompatHelper.yml new file mode 100644 index 0000000..cba9134 --- /dev/null +++ b/workflows/CompatHelper.yml @@ -0,0 +1,16 @@ +name: CompatHelper +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} + run: julia -e 'using CompatHelper; CompatHelper.main()' diff --git a/workflows/TagBot.yml b/workflows/TagBot.yml new file mode 100644 index 0000000..2bacdb8 --- /dev/null +++ b/workflows/TagBot.yml @@ -0,0 +1,31 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: + inputs: + lookback: + default: 3 +permissions: + actions: read + checks: read + contents: write + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }}