Feature/nix

Dockerfile

@@ -1,32 +1,67 @@
-FROM ocaml/opam:debian-10-ocaml-4.13
-RUN sudo apt install -y libcurl4-gnutls-dev libgmp-dev
+# syntax = docker/dockerfile:1.2
 
-WORKDIR /lang_agent/
-RUN opam install dune
+# Nix builder
+FROM nixos/nix:latest AS builder
 
+# Copy our source and setup our working dir.
+COPY . /tmp/gitsub
+# nasty hack to work around git submodules
+COPY gitconfig.txt /tmp/gitsub/.git/config 
+RUN git clone --mirror /tmp/gitsub /tmp/build
+WORKDIR /tmp/build
 
-# sudo docker run -it lang_agent bash 
-ENV PATH /home/opam/.opam/4.13/bin/:$PATH
-RUN /home/opam/.opam/4.13/bin/dune --version
-#RUN apt-get -y install opam
+# Build our Nix environment
+RUN nix \
+    --extra-experimental-features "nix-command flakes" \
+    --option filter-syscalls false \
+    build
 
+# Copy the Nix store closure into a directory. The Nix store closure is the
+# entire set of Nix store values that we need for our build.
+RUN mkdir /tmp/nix-store-closure
+RUN cp -R $(nix-store -qR result/) /tmp/nix-store-closure
 
-RUN opam repo add coq-released https://coq.inria.fr/opam/released
-RUN opam repo add coq-extra-dev https://coq.inria.fr/opam/extra-dev
-RUN opam install --yes ppx_yojson_conv_lib coq
-RUN opam install --yes ppx_yojson lwt_ppx containers lambdasoup uri ezcurl ppx_yojson_conv ppx_yojson_conv_lib ezcurl-lwt yojson
-RUN opam install --yes coq-metacoq-template
-RUN opam pin add --yes openai https://github.com/meta-introspector/openai-ocaml.git#main
+# Final image is based on scratch. We copy a bunch of Nix dependencies
+# but they're fully self-contained so we don't need Nix anymore.
+FROM scratch
 
-COPY lib /lang_agent/lib
-COPY bin /lang_agent/bin
-COPY dune-project /lang_agent/
-RUN opam install --yes --deps-only .
-RUN opam install --yes stdio
-RUN dune build .
+WORKDIR /app
 
-# sudo docker build . -t lang_agent
-# sudo docker run -it lang_agent bash
+# Copy /nix/store
+COPY --from=builder /tmp/nix-store-closure /nix/store
+COPY --from=builder /tmp/build/result /app
+CMD ["/app/bin/app"]
 
-# dune exec bin/worker.exe
-#  sudo docker  build . -t h4ckermike/lang_agent:dev
+# FROM ocaml/opam:debian-10-ocaml-4.13
+# RUN sudo apt install -y libcurl4-gnutls-dev libgmp-dev
+
+# WORKDIR /lang_agent/
+# RUN opam install dune
+
+
+# # sudo docker run -it lang_agent bash 
+# ENV PATH /home/opam/.opam/4.13/bin/:$PATH
+# RUN /home/opam/.opam/4.13/bin/dune --version
+# #RUN apt-get -y install opam
+
+
+# RUN opam repo add coq-released https://coq.inria.fr/opam/released
+# RUN opam repo add coq-extra-dev https://coq.inria.fr/opam/extra-dev
+# RUN opam install --yes ppx_yojson_conv_lib coq
+# RUN opam install --yes ppx_yojson lwt_ppx containers lambdasoup uri ezcurl ppx_yojson_conv ppx_yojson_conv_lib ezcurl-lwt yojson
+# RUN opam install --yes coq-metacoq-template
+# RUN opam pin add --yes openai https://github.com/meta-introspector/openai-ocaml.git#main
+
+# COPY lib /lang_agent/lib
+# COPY bin /lang_agent/bin
+# COPY dune-project /lang_agent/
+# RUN opam install --yes --deps-only .
+# RUN opam install --yes stdio
+# RUN dune build .
+
+
+# # sudo docker build . -t lang_agent
+# # sudo docker run -it lang_agent bash
+
+# # dune exec bin/worker.exe
+# #  sudo docker  build . -t h4ckermike/lang_agent:dev

Makefile

@@ -1,3 +1,5 @@
+nix:
+	nix shell --extra-experimental-features nix-command  --extra-experimental-features flakes
 tst:
 	dune build 
 	dune exec -- ./bin/argiope.exe http://mistral.ai

README.org

@@ -1,5 +1,21 @@
+
 * idea : create an ocaml functional agent for talking to llms.
 
+* nix
+
+#+begin_src shell
+nix shell --extra-experimental-features nix-command  --extra-experimental-features flakes
+#+end_src
+
+#+begin_src shell
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes
+#+end_src
+
+#+begin_src shell
+nix develop --extra-experimental-features nix-command --extra-experimental-features flakes
+#+end_src
+
+* normal
 #+begin_src shell
   opam install --yes ppx_yojson_conv_lib ppx_yojson_conv
   opam install --yes ppx_yojson lwt_ppx containers lambdasoup uri ezcurl ppx_yojson_conv ppx_yojson_conv_lib ezcurl-lwt yojson

bin/dune

@@ -2,7 +2,6 @@
  (_
   (flags
    (:standard -warn-error -32
-     ;;-8-9-26-27-11-39
      -dtypedtree -dparsetree -dump-into-file -dlambda -annot -dlocations -dsource  -drawlambda  ))))
 
 (executable

bin/simple.ml

@@ -10,7 +10,7 @@ type backend2 =
 
   | B2None  
   | B2Ollama of Ollama.client_t Lang_model.client_t
-  | B2OpenAi of Openai.Client.t Lang_model.client_t
+  | B2OpenAi of Client.t Lang_model.client_t
   | B2LlamaCpp of Llama_cpp.client_t Lang_model.client_t
 
 let lc_lang_prompt lang_client  param_record prompt =

flake.nix

@@ -1,18 +1,21 @@
 {
   inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable";	
     opam-nix.url = "github:tweag/opam-nix";
     flake-utils.url = "github:numtide/flake-utils";
     nixpkgs.follows = "opam-nix/nixpkgs";
+
   };
   outputs = { self, flake-utils, opam-nix, nixpkgs }@inputs:
     # Don't forget to put the package name instead of `throw':
-    let package = "lang-agent";
+    let package = "lang_agent";
     in flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
+
         on = opam-nix.lib.${system};
         scope =
-          on.buildOpamProject { } package ./. { ocaml-base-compiler = "*"; };
+          on.buildDuneProject { } package  ./. { ocaml-base-compiler = "*"; };
         overlay = final: prev:
           {
             # Your overrides go here

gen_fold_simple.sh

@@ -30,9 +30,9 @@ cat $PROMPT_NAME >> $PROMPT_NAME2
 echo " ENDSRC . Please imagine the simulation its execution. Imagine running the code and all the output it would generate. Generate ocaml code to do the merge of large language model prompt workflows." >> $PROMPT_NAME2
 
 mkdir -p data/folder
-
-dune exec bin/simple.exe -- \
-     --openai -m "mixtral" -u "https://dev-hub.agentartificial.com" \
+#      #     --openai -m "mixtral" -u "https://dev-hub.agentartificial.com" \
+./_build/install/default/bin/simple.exe \
+       --ollama -m "mixtral" -u "https://mixtral-agentartificial.ngrok.app" \
      -f $PROMPT_NAME2 \
     -s "data/folder/sim_folder_1_${DS}"   \
     -x ".txt" \

grammars/ebnf.ebnf.prompt

@@ -1,75 +1,110 @@
 We are bootstrapping a new system using EBNF grammars.
 We want to make an ebnf grammar that is super detailed and self expressive.
 Here is the code we wrote so far 
-# GBNF (GGML BNF) is a format for defining formal grammars to constrain model outputs in llama.cpp. 
-# Backus-Naur Form (BNF) is a notation for describing the syntax of formal languages like programming languages, file formats, and protocols. GBNF is an extension of BNF that primarily adds a few modern regex-like features.
-# In GBNF, we define production rules that specify how a non-terminal (rule name) can be replaced with sequences of terminals (characters, specifically Unicode code points) and other non-terminals. The basic format of a production rule is nonterminal ::= sequence....
-
-production_rule ::= alternation
-lhs ::= identifier
-rule ::= lhs   S   "="   S   production_rule   S    | comment
-root ::= ( S   rule   S ) *
-
 # Terminals support the full range of Unicode. Unicode characters can be specified directly in the grammar, for example hiragana ::= [ぁ-ゟ], or with escapes: 8-bit (\xXX), 16-bit (\uXXXX) or 32-bit (\UXXXXXXXX).
+
 range ::=  "-"
-factor_range ::= term   S   range   S   term
 
 # Character ranges can be negated with ^:
-negate ::=  "^"
 
-#Sequences and Alternatives
-#The order of symbols in a sequence matter. For example, in "1. " move " " move "\n", the "1. " must come before the first move, etc.
-concatenation ::= ( S   factor   S   ? ) +
+negate ::= "^"
 
 # Alternatives, denoted by |, give different sequences that are acceptable. 
-alternation ::=  "|"
-alternation ::= ( S   concatenation   S   alternation ? ) +
+
+alternationsymbol ::= "|"
+
+
+#Sequences and Alternatives
 
 # Parentheses () can be used to group sequences, which allows for embedding alternatives in a larger rule or applying repetition and optional symbols (below) to a sequence.
-parens_open ::= "("
-parens_close ::=  ")"
-parens ::= parens_open | parens_close
+
+parensopen ::= "("
+
+parensclose ::=  ")"
+
+parens ::= parensopen | parensclose
 
 #Repetition and Optional Symbols
-repetition_symbols ::= repetition_plus | repetition_star | repetition_optional
+
+repetitionsymbols ::= repetitionplus | repetitionstar | repetitionoptional
 
 #* after a symbol or sequence means that it can be repeated zero or more times.
-repetition_star ::=  "*" 
+
+repetitionstar ::=  "*" 
 
 #+ denotes that the symbol or sequence should appear one or more times.
-repetition_plus ::=  "+" 
+
+repetitionplus ::=  "+" 
 
 #? makes the preceding symbol or sequence optional.
-repetition_optional ::=  "?" 
+
+repetitionoptional ::=  "?" 
 
 
 #Comments and newlines
 #Comments can be specified with #:
+
 comment ::= "#" [a-zA-Z0-9 \t]*
 
 # Newlines are allowed between rules and between symbols or sequences nested inside parentheses. Additionally, a newline after an alternate marker | will continue the current rule, even outside of parentheses.
 
-
 letter ::= [a-zA-Z]
+
 digit ::= [0-9]
-S ::= ( " " | "\n" | "\t" | "\r"  )
-braces_open ::= "[" 
-braces_close ::= "]"
-braces_symbol ::= braces_open | braces_close
+
+spacechar ::= " "
+S ::= ( spacechar | "\n" | "\t" | "\r"  )
+
+bracesopen ::= "[" 
+
+bracesclose ::= "]"
+
+bracessymbol ::= bracesopen | bracesclose
 
 quote ::=  "\""
+
 assignment ::= "::="
-
-symbol ::= braces_symbol | parens | quotes |assignment | alternation | range | repetition_symbols  | negate
-
-character ::= letter | digit | symbol | "_" | " "
-identifier ::= letter ( letter | digit | "_" )*
+
+underscorecharacter ::=  "_"
+
+character ::= letter | digit | symbol | underscorecharacter | spacechar
+
+symbol ::= bracessymbol | parens | quote | assignment | alternationsymbol | range | repetitionsymbols  | negate
+
+
+
+# no underscores in these name
+
+identifier ::= letter ( letter | digit  )*
+
 terminal ::= quote character+  quote 
-group ::= parens_open   S   production_rule   S   parens_close
-range_term ::=  braces_open   S   production_rule   S   braces_close
-term ::= group |range_term | terminal      | identifier
 
-repetition ::= term   S   occurence  
-factor_negate ::= negate S factor
-factor ::= repetition |factor_range | term   S
+group ::= parensopen S productionrule S parensclose
+
+rangeterm ::=  bracesopen   S   productionrule   S   bracesclose
+
+term ::= group |rangeterm | terminal      | identifier
+
+repetition ::= term   S repetitionsymbols
+
+factornegate ::= negate S factor
+
+factorrange ::= term   S   range   S   term
+
+factor ::= repetition |factorrange | term   S
+
+#The order of symbols in a sequence matter. 
+
+concatenation ::= ( S   factor   S   ? ) +
+
+alternation ::= ( S   concatenation   S   alternationsymbol ? ) +
+
+productionrule ::= alternation
+
+
+lhs ::= identifier
+
+rule ::= lhs   S   assignment   S   productionrule   S    | comment
+
+root ::= ( S   rule   S ) *
  ENDSRC . Lets rewrite this  EBNF to be more expressive and explicit in its naming but keep the syntax the same for compatibility with GBNF and llama.cpp.