Refactor Configs and generalize language service evaluation

bin/langeval

@@ -0,0 +1,21 @@
+#!/bin/sh
+#set -x
+VONDA_VERSIONS="3.0\|2\|1"
+. `dirname $0`/utils.sh
+
+if test -z "$JARIMAGES"; then
+    if test -z "$jardir"; then
+        cd `getscriptdir`/..
+        jardir=`pwd`
+        cd $here
+    fi
+
+    JARIMAGES=`find $jardir -name vonda-\*.jar | tr '\012' ':'`
+fi
+
+if test -z "$JARIMAGES"; then
+    echo "No executable image found"
+    exit 1
+fi
+MAINCLASS=de.dfki.mlt.rudimant.agent.nlp.EvalLangServices
+java $silent -cp "$JARIMAGES" $MAINCLASS "$@"

bin/srgseval → bin/utils.sh

@@ -1,10 +1,7 @@
-#!/bin/sh
-#set -x
-VONDA_VERSIONS="3.0\|2\|1"
 script_dir=`dirname $0`
 here=`pwd`
-if test -f "$script_dir/logback.xml"; then
-    silent="$silent -Dlogback.configurationFile=$script_dir/logback.xml "
+if test -f "logback.xml"; then
+    silent="$silent -Dlogback.configurationFile=logback.xml "
 else
     silent="-Dorg.slf4j.simpleLogger.defaultLogLevel=warn "
 fi
@@ -45,18 +42,7 @@ else
     }
 fi
 
-if test -z "$jardir"; then
+getscriptdir() {
     script_dir=`realpath "$0"`
-    script_dir=`dirname "$script_dir"`
-    cd $script_dir
-    cd ../compiler/target
-    jardir=`pwd`
-    cd $here
-fi
-
-JARIMAGE=`find $jardir -name vonda-\*-compiler.jar | grep "$VONDA_VERSIONS" | head -1`
-if test -z "$JARIMAGE"; then
-    echo "No executable image found"
-    exit 1
-fi
-java $silent -cp "$JARIMAGE" de.dfki.mlt.rudimant.agent.nlp.EvalSrgs "$@"
+    dirname "$script_dir"
+}

bin/vondac

@@ -1,55 +1,10 @@
 #!/bin/sh
 #set -x
 VONDA_VERSIONS="3.0\|2\|1"
-script_dir=`dirname $0`
-here=`pwd`
-if test -f "$script_dir/logback.xml"; then
-    silent="$silent -Dlogback.configurationFile=$script_dir/logback.xml "
-else
-    silent="-Dorg.slf4j.simpleLogger.defaultLogLevel=warn "
-fi
-
-java --version | grep -q ' 1[1-6]\.' && \
-    silent="${silent}--illegal-access=permit "
-
-silent="${silent}--add-modules=ALL-SYSTEM \
---add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
---add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED \
---add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
---add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
---add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
---add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED "
-
-if realpath . > /dev/null 2>&1 ; then
-    :
-else
-    # does not exist on Mac OSX
-    realpath() {
-        target=$1
-
-        cd `dirname $target`
-        target=`basename $target`
-
-        # Iterate down a (possible) chain of symlinks
-        while [ -L "$target" ]
-        do
-            target=`readlink $target`
-            cd `dirname $target`
-            target=`basename $target`
-        done
-
-        # Compute the canonicalized name by finding the physical path
-        # for the directory we're in and appending the target file.
-        phys_dir=`pwd -P`
-        echo "$phys_dir/$target"
-    }
-fi
+. `dirname $0`/utils.sh
 
 if test -z "$jardir"; then
-    script_dir=`realpath "$0"`
-    script_dir=`dirname "$script_dir"`
-    cd $script_dir
-    cd ../compiler/target
+    cd `getscriptdir`/../compiler/target
     jardir=`pwd`
     cd $here
 fi

compiler/pom.xml

@@ -11,15 +11,15 @@
   <parent>
     <groupId>de.dfki.mlt</groupId>
     <artifactId>vonda-top</artifactId>
-    <version>3.2.3</version>
+    <version>3.2.4</version>
     <relativePath>..</relativePath>
   </parent>
 
   <dependencies>
     <dependency>
       <groupId>de.dfki.mlt</groupId>
       <artifactId>vonda</artifactId>
-      <version>3.2.2</version>
+      <version>${project.version}</version>
     </dependency>
 
     <dependency>

compiler/src/main/java/de/dfki/mlt/rudimant/compiler/CompilerMain.java

@@ -19,21 +19,18 @@
 
 package de.dfki.mlt.rudimant.compiler;
 
-import static de.dfki.mlt.rudimant.common.Constants.*;
+import static de.dfki.mlt.rudimant.common.Configs.*;
+import static de.dfki.mlt.rudimant.common.Constants.RULE_LOCATION_FILE;
 import static de.dfki.mlt.rudimant.compiler.Constants.COMPILER_VERSION;
 import static java.nio.file.Files.createDirectories;
 
 import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.yaml.snakeyaml.Yaml;
-
 import de.dfki.lt.hfc.WrongFormatException;
 import de.dfki.mlt.rudimant.common.IncludeInfo;
 import de.dfki.mlt.rudimant.compiler.io.BisonParser;
@@ -98,23 +95,6 @@ static void setValue(String key, String option, Object def,
     }
   }
 
-  public static Map<String,Object> readConfig(File confFile)
-      throws FileNotFoundException {
-    Yaml yaml = new Yaml();
-    File confDir = confFile.getParentFile();
-    if (confDir == null) {
-      confDir = new File(".");
-    }
-    Map<String,Object> configs = yaml.load(new FileReader(confFile));
-    configs.put(CFG_CONFIG_DIRECTORY, confDir);
-    return configs;
-  }
-
-  public static Map<String,Object> readConfig(String confName)
-      throws FileNotFoundException {
-    return readConfig(new File(confName));
-  }
-
   /**
    * Saves generated rule structure and errors/warnings to a YAML file
    */

compiler/src/main/java/de/dfki/mlt/rudimant/compiler/RudimantCompiler.java

@@ -20,7 +20,8 @@
 package de.dfki.mlt.rudimant.compiler;
 
 import static com.google.common.io.Files.asCharSink;
-import static de.dfki.mlt.rudimant.common.Constants.*;
+import static de.dfki.mlt.rudimant.common.Configs.*;
+import static de.dfki.mlt.rudimant.common.Constants.RULE_FILE_EXTENSION;
 import static de.dfki.mlt.rudimant.compiler.Constants.AGENT_DEFS;
 import static de.dfki.mlt.rudimant.compiler.tree.GrammarFile.parseAndTypecheck;
 import static java.nio.file.Files.createDirectories;

compiler/src/test/java/de/dfki/mlt/rudimant/CoverageTest.java

@@ -18,9 +18,11 @@
  */
 
 package de.dfki.mlt.rudimant;
-import static de.dfki.mlt.rudimant.common.Constants.*;
+import static de.dfki.mlt.rudimant.common.Configs.CFG_CONFIG_DIRECTORY;
+import static de.dfki.mlt.rudimant.common.Configs.CFG_VISUALISE;
 import static de.dfki.mlt.rudimant.compiler.tree.TestUtilities.RESOURCE_DIR;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 import java.awt.GraphicsDevice;
 import java.awt.GraphicsEnvironment;
@@ -31,6 +33,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Map;
+
 import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.Test;

compiler/src/test/java/de/dfki/mlt/rudimant/compiler/tree/TestHelper.java

@@ -1,6 +1,6 @@
 package de.dfki.mlt.rudimant.compiler.tree;
 
-import static de.dfki.mlt.rudimant.common.Constants.CFG_CONFIG_DIRECTORY;
+import static de.dfki.mlt.rudimant.common.Configs.*;
 
 import java.io.ByteArrayInputStream;
 import java.io.File;
@@ -27,14 +27,14 @@ public class TestHelper {
   private static File confDir;
   public static Map<String, Object> configs;
 
-  public static void readConfig(String configFile) throws FileNotFoundException {
-    configs = CompilerMain.readConfig(configFile);
+  public static void getConfig(String configFile) throws FileNotFoundException {
+    configs = readConfig(configFile);
     confDir = (File)configs.get(CFG_CONFIG_DIRECTORY);
   }
 
   public static void setUp(String configFile, String h, String f)
       throws FileNotFoundException {
-    readConfig(configFile);
+    getConfig(configFile);
     header = h;
     footer = f;
   }

library/pom.xml

@@ -11,7 +11,7 @@
   <parent>
     <groupId>de.dfki.mlt</groupId>
     <artifactId>vonda-top</artifactId>
-    <version>3.2.3</version>
+    <version>3.2.4</version>
     <relativePath>..</relativePath>
   </parent>
 

library/src/main/java/de/dfki/mlt/rudimant/agent/Agent.java

@@ -19,7 +19,7 @@
 
 package de.dfki.mlt.rudimant.agent;
 
-import static de.dfki.mlt.rudimant.common.Constants.CFG_DEBUG_PORT;
+import static de.dfki.mlt.rudimant.common.Configs.CFG_DEBUG_PORT;
 import static de.dfki.mlt.rudimant.common.Constants.STATE_ALWAYS;
 import static de.dfki.mlt.rudimant.common.Constants.STATE_NEVER;
 
@@ -48,9 +48,9 @@
 import de.dfki.lt.hfc.db.rdfProxy.RdfClass;
 import de.dfki.lt.hfc.db.rdfProxy.RdfProxy;
 import de.dfki.lt.tr.dialogue.cplan.DagNode;
-import de.dfki.mlt.rudimant.agent.nlp.LanguageServices;
 import de.dfki.mlt.rudimant.agent.nlp.DiaHierarchy;
 import de.dfki.mlt.rudimant.agent.nlp.DialogueAct;
+import de.dfki.mlt.rudimant.agent.nlp.LanguageServices;
 import de.dfki.mlt.rudimant.agent.nlp.Pair;
 import de.dfki.mlt.rudimant.common.RuleLogger;
 import de.dfki.mlt.rudimant.common.SimpleServer;

library/src/main/java/de/dfki/mlt/rudimant/agent/nlp/EvalSrgs.java → library/src/main/java/de/dfki/mlt/rudimant/agent/nlp/EvalLangServices.java

@@ -1,5 +1,7 @@
 package de.dfki.mlt.rudimant.agent.nlp;
 
+import static de.dfki.mlt.rudimant.common.Configs.*;
+
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
@@ -15,13 +17,13 @@
 import org.slf4j.LoggerFactory;
 import org.yaml.snakeyaml.Yaml;
 
-public class EvalSrgs {
-  private static Logger logger = LoggerFactory.getLogger(EvalSrgs.class);
+public class EvalLangServices {
+  private static Logger logger = LoggerFactory.getLogger(EvalLangServices.class);
 
   PrintWriter hits;
   PrintWriter misses;
 
-  SrgsParser srgs = new SrgsParser();
+  LanguageServices ls;
 
   private static class Section {
     private Section(String i, String e) {
@@ -64,20 +66,28 @@ public static List<Section> readCorpus(File f) throws FileNotFoundException {
     }
   }
 
-  public void loadSrgs(File configDir, Map<String, Object> config) {
+  public void loadServices(Map<String, Object> config) {
     logger.info("Loading grammar from {}", config.get("grammar"));
+    File configDir = (File)config.get(CFG_CONFIG_DIRECTORY);
     String language = (String)config.get("language");
     if (language == null) {
       language = "en_EN";
     }
-    if (!srgs.init(configDir, language, config)) {
-      logger.error("SRGS grammar loading failed");
+    ls = new LanguageServices();
+    try {
+      ls.loadGrammar(configDir, language, config);
+    } catch (Exception ex) {
+      logger.error("grammar loading failed: {}", ex.getMessage());
       throw new RuntimeException();
     }
   }
 
-  private DialogueAct parse(String input) {
-    return srgs.analyse(input);
+  public DialogueAct parse(String input) {
+    return ls.interpret(input);
+  }
+
+  public Pair<String, String> generate(DialogueAct input) {
+    return ls.generate(input.getDag());
   }
 
   private static final Pattern ANNOT_REGEX = Pattern.compile(
@@ -91,11 +101,9 @@ private static String removeAnnotations(String input) {
 
   @SuppressWarnings("unchecked")
   void evaluate(File configFile) throws FileNotFoundException {
-    File configDir = configFile.getAbsoluteFile().getParentFile();
-    Yaml yaml = new Yaml();
-    Map<String, Object> configs =
-        (Map<String, Object>) yaml.load(new FileReader(configFile));
-    loadSrgs(configDir, configs);
+    Map<String, Object> configs = readConfig(configFile);
+    loadServices(configs);
+    File configDir = (File)configs.get(CFG_CONFIG_DIRECTORY);
     List<String> corpora = (List<String>)configs.get("corpora");
     for (String corpName : corpora) {
       File corpus = new File(corpName);
@@ -189,7 +197,7 @@ void parseCorpus(File rootDir, File corpusFile) throws FileNotFoundException {
   }
 
   public static void main(String[] args) throws FileNotFoundException {
-    EvalSrgs e = new EvalSrgs();
+    EvalLangServices e = new EvalLangServices();
     if (args.length == 0) {
       System.out.println("Usage: eval_nlu configfile.yml");
       System.out.println("       the configfile.yml needs a \"corpus\" property "

library/src/main/java/de/dfki/mlt/rudimant/agent/nlp/Interpreter.java

@@ -19,7 +19,7 @@
 
 package de.dfki.mlt.rudimant.agent.nlp;
 
-import static de.dfki.mlt.rudimant.common.Constants.CFG_NLU_CONVERTER;
+import static de.dfki.mlt.rudimant.common.Configs.CFG_NLU_CONVERTER;
 
 import java.io.File;
 import java.io.IOException;

library/src/main/java/de/dfki/mlt/rudimant/agent/nlp/LanguageGenerator.java

@@ -19,7 +19,9 @@
 
 package de.dfki.mlt.rudimant.agent.nlp;
 
-import static de.dfki.mlt.rudimant.common.Constants.*;
+import static de.dfki.mlt.rudimant.common.Configs.CFG_NLG_GENERATION_PROJECT;
+import static de.dfki.mlt.rudimant.common.Configs.CFG_NLG_MAPPER_PROJECT;
+import static de.dfki.mlt.rudimant.common.Configs.CFG_NLG_TRANSLATE_NUMBERS;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -87,6 +89,7 @@ protected void load() {
    *         of failure.
    * @throws FileNotFoundException
    */
+  @Override
   public boolean init(File configDir, String lang,
       Map<String, Object> cfgs) {
     configs = new HashMap<>(cfgs);
@@ -159,6 +162,7 @@ private DagNode toDag(String genericDialogueAct) {
     return generate(rawInput);
   }
 
+  @Override
   public Pair<String, String> generate(DagNode rawInput) {
     // obsolete
     // _ruleMapper.schemaLfStringToLfString(genericDialogueAct));