Merge pull request #1050 from numbbo/development

Merge development branch into master before release

README.md

@@ -53,7 +53,9 @@ _Additional_ requirements for running an algorithm in a specific language.
   containing VC9, available [here](https://www.microsoft.com/en-us/download/details.aspx?id=44266). 
   These are necessary to build the C extensions for the Python `cocoex` module for Windows. 
   The package contains 32-bit and 64-bit compilers and the Windows SDK headers.
+* **Pyhon on Linux**: `python-dev` must be installed to compile/install the `cocoex` module.
 * **Octave**: Octave 4.0.0 or later. On operating systems other than Windows, earlier versions might work.
+  Under Linux the package `liboctave-dev` might be necessary. 
 
 ### Guaranties (None)
 We tested the framework on Mac OSX, Ubuntu linux, Fedora linux, and Windows (XP,
@@ -342,7 +344,26 @@ processes if there is any) before to run the `do.py` command again.
 
 
 ### Octave
-none for the moment
+
+#### `octave-dev` under Linux
+When running 
+```
+  python do.py run-octave
+```
+or 
+```
+  python do.py build-octave
+```
+and seeing something like
+```
+   [...]
+   compiling cocoCall.c...error: mkoctfile: please install the Debian package "liboctave-dev" to get the mkoctfile command
+```
+then, unsurprisingly, installing `liboctave-dev` like
+```
+  sudo apt-get install liboctave-dev
+```
+should do the job. 
 
 
 ### Python
@@ -388,6 +409,17 @@ cython/interface.c:4:20: error: Python.h: file not found
 cython/interface.c:6:6: error: #error Python headers needed to compile C extensions, please install development version of Python.
 error: command 'gcc' failed with exit status 1
 ```
+or
+```
+$ python do.py run-python  # or build-python
+[...]
+cython/interface.c -o build/temp.linux-x86_64-2.7/cython/interface.o
+cython/interface.c:4:20: fatal error: Python.h: No such file or directory
+#include "Python.h"
+^
+compilation terminated.
+error: command 'x86_64-linux-gnu-gcc' failed with exit status 1
+```
 Under Linux
 ```
   sudo apt-get install python-dev

code-postprocessing/bbob_pproc/__init__.py

@@ -22,8 +22,6 @@
 
 from __future__ import absolute_import
 
-import sys
-
 import matplotlib  # just to make sure the following is actually done first
 matplotlib.use('Agg')  # To avoid window popup and use without X forwarding
 
@@ -35,9 +33,9 @@
 
 import pkg_resources
 
-__all__  = ['comp2', 'compall', 'main', 'ppfigdim', 'pplogloss', 'pprldistr',
-            'pproc', 'ppsingle', 'pptable', 'rungeneric', 'rungeneric1',
-            'rungeneric2', 'rungenericmany', 'algportfolio', 'pptex',
-            'toolsstats']
+__all__ = ['comp2', 'compall', 'main', 'ppfigdim', 'pplogloss', 'pprldistr',
+           'pproc', 'ppsingle', 'pptable', 'rungeneric', 'rungeneric1',
+           'rungeneric2', 'rungenericmany', 'algportfolio', 'pptex',
+           'toolsstats']
 
 __version__ = pkg_resources.require('bbob_pproc')[0].version

code-postprocessing/bbob_pproc/__main__.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python
-"""``python bbob_pproc`` tests the package bbob_pproc and should run through 
-smoothly from a system command shell. It however depends on data files that 
-might not be available (to be improved). 
+"""``python bbob_pproc`` tests the package bbob_pproc and should run through
+smoothly from a system command shell. It however depends on data files that
+might not be available (to be improved).
 
-This test can and should become much more sophisticated.  
+This test can and should become much more sophisticated.
 
 """
 
@@ -36,7 +36,8 @@
         import cocopp
     # run either this main here as cocopp._main or rungeneric.main
     if len(args) == 0:
-        print("WARNING: this tests the post-processing, this will change in future (use -h for help)")
+        print("WARNING: this tests the post-processing, this will change "
+              + "in future (use -h for help)")
         cocopp._main(args)
     elif args[0] == '-t' or args[0].startswith('--t'):
         args.pop(0)
@@ -63,15 +64,24 @@ def copy_latex_templates():
     shutil.copy(join_path(templateFolder, 'comment.sty'), currentFolder)
     shutil.copy(join_path(templateFolder, 'acmcopyright.sty'), currentFolder)
     shutil.copy(join_path(templateFolder, 'bbob.bib'), currentFolder)
-    
+
 
 def run_latex_template(filename):
     filePath = os.path.abspath(join_path(os.path.dirname(__file__), filename))
     args = ['pdflatex', filePath]
     DEVNULL = open(os.devnull, 'wb')
-    return subprocess.call(args, stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL)
+    result = subprocess.call(args, stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL)
+    assert not result, 'Test failed: error while generating pdf from %s.' % filename
+
+    # filePath = os.path.splitext(filePath)[0]
+    # args = ['bibtex', filePath]
+    # DEVNULL = open(os.devnull, 'wb')
+    # output_file = open("bibtex.log", "w")
+    # result = subprocess.call(args, stdin=DEVNULL, stdout=output_file, stderr=DEVNULL)
+    # assert not result, 'Test failed: error while running bibtex on %s.' % os.path.splitext(filename)[0]
+
 
-def retrieve_algorithm(dataPath, folderName, algorithmName, fileName = None):
+def retrieve_algorithm(dataPath, folderName, algorithmName, fileName=None):
     algorithmFile = join_path(dataPath, fileName if fileName else algorithmName)
     if not os.path.exists(algorithmFile):
         dataurl = 'http://coco.gforge.inria.fr/data-archive/%s/%s' % (folderName, algorithmName)
@@ -82,21 +92,21 @@ def prepare_data(run_all_tests):
 
     dataPath = os.path.abspath(join_path(os.path.dirname(__file__), 'data'))
 
-    # Retrieving the algorithms    
+    # Retrieving the algorithms
     # retrieve_algorithm(dataPath, '2010', 'IPOP-ACTCMA-ES_ros_noiseless.tar.gz')
-    # [outcommented and replaced by BIPOP until 2010 data is in new format] 
-    retrieve_algorithm(dataPath, '2009', 'BFGS_ros_noiseless.tgz')    
+    # [outcommented and replaced by BIPOP until 2010 data is in new format]
+    retrieve_algorithm(dataPath, '2009', 'BFGS_ros_noiseless.tgz')
     retrieve_algorithm(dataPath, 'biobj-test', 'RS_on_bbob-biobj-test.tgz', 'RS.tgz')
 
     if run_all_tests:
-        retrieve_algorithm(dataPath, '2009', 'BIPOP-CMA-ES_hansen_noiseless.tgz')   
-        retrieve_algorithm(dataPath, '2009', 'MCS_huyer_noiseless.tgz')    
-        retrieve_algorithm(dataPath, '2009', 'NEWUOA_ros_noiseless.tgz')    
-        retrieve_algorithm(dataPath, '2009', 'RANDOMSEARCH_auger_noiseless.tgz')    
-        retrieve_algorithm(dataPath, '2013', 'SMAC-BBOB_hutter_noiseless.tgz')    
-        retrieve_algorithm(dataPath, '2013', 'lmm-CMA-ES_auger_noiseless.tgz')    
-        retrieve_algorithm(dataPath, '2009', 'DE-PSO_garcia-nieto_noiseless.tgz')    
-        retrieve_algorithm(dataPath, '2009', 'VNS_garcia-martinez_noiseless.tgz')    
+        retrieve_algorithm(dataPath, '2009', 'BIPOP-CMA-ES_hansen_noiseless.tgz')
+        retrieve_algorithm(dataPath, '2009', 'MCS_huyer_noiseless.tgz')
+        retrieve_algorithm(dataPath, '2009', 'NEWUOA_ros_noiseless.tgz')
+        retrieve_algorithm(dataPath, '2009', 'RANDOMSEARCH_auger_noiseless.tgz')
+        retrieve_algorithm(dataPath, '2013', 'SMAC-BBOB_hutter_noiseless.tgz')
+        retrieve_algorithm(dataPath, '2013', 'lmm-CMA-ES_auger_noiseless.tgz')
+        retrieve_algorithm(dataPath, '2009', 'DE-PSO_garcia-nieto_noiseless.tgz')
+        retrieve_algorithm(dataPath, '2009', 'VNS_garcia-martinez_noiseless.tgz')
 
     return dataPath
 
@@ -109,18 +119,18 @@ def process_doctest_output(stream=None):
     state = 0
     for line in fileinput.input(stream):  # takes argv as file or stdin
         if 1 < 3:
-            
+
             s3 += line
             if state < -1 and line.startswith('***'):
                 print(s3)
             if line.startswith('***'):
                 s3 = ""
-            
+
         if state == -1:  # found a failed example line
             s1 += '\n\n*** Failed Example:' + line
             s2 += '\n\n\n'   # line
-            # state = 0  # wait for 'Expected:' line 
-            
+            # state = 0  # wait for 'Expected:' line
+
         if line.startswith('Expected:'):
             state = 1
             continue
@@ -141,46 +151,42 @@ def process_doctest_output(stream=None):
             s2 += line + ''
 
 def main(args):
-    """these tests are executed when ``python bbob_pproc`` is called.  
+    """these tests are executed when ``python bbob_pproc`` is called.
 
-    with ``wine`` as second argument ``C:\\Python26\\python.exe`` 
+    with ``wine`` as second argument ``C:\\Python26\\python.exe``
     instead of ``python`` is called
-    
+
     """
 
     run_all_tests = len(args) == 1 and args[0] == 'all'
 
-    python = 'python -m '  # how to call python 
+    python = 'python -m '  # how to call python
     if len(sys.argv) > 1 and sys.argv[1] == 'wine':
         python = 'C:\\Python26\\python.exe ' # works for wine
-    
+
     data_path = ' ' + prepare_data(run_all_tests)
-        
+
     command = ' bbob_pproc --no-svg --settings=grayscale '
-    
+
     copy_latex_templates()
     print('LaTeX templates copied.')
-    
+
     print('*** testing module bbob_pproc ***')
     t0 = time.time()
     print(python + command + '--conv' + join_path(data_path, 'BFGS_ros_noiseless.tgz'))
     result = os.system(python + command + '--conv' + join_path(data_path, 'BFGS_ros_noiseless.tgz'))
     print('**  subtest 1 finished in ', time.time() - t0, ' seconds')
     assert result == 0, 'Test failed: rungeneric on one algorithm with option --conv.'
-
-    result = run_latex_template("templateBBOBarticle.tex")
-    assert not result, 'Test failed: error while generating pdf from templateBBOBarticle.tex.'
+    run_latex_template("templateBBOBarticle.tex")
 
     t0 = time.time()
     print(python + command + join_path(data_path, 'RS.tgz'))
     result = os.system(python + command + join_path(data_path, 'RS.tgz'))
     print('**  subtest 1 finished in ', time.time() - t0, ' seconds')
     assert result == 0, 'Test failed: rungeneric on one bi-objective algorithm.'
+    run_latex_template("templateBIOBJarticle.tex")
 
-    result = run_latex_template("templateBIOBJarticle.tex")
-    assert not result, 'Test failed: error while generating pdf from templateBIOBJarticle.tex.'
-
-    if run_all_tests:    
+    if run_all_tests:
         t0 = time.time()
         print(time.asctime())
         result = os.system(python + command + # ' --omit-single ' +
@@ -191,38 +197,30 @@ def main(args):
                     join_path(data_path, 'BFGS_ros_noiseless.tgz'))
         print('**  subtest 2 finished in ', time.time() - t0, ' seconds')
         assert result == 0, 'Test failed: rungeneric on many algorithms.'
-
-        result = run_latex_template("templateBBOBmany.tex")
-        assert not result, 'Test failed: error while generating pdf from templateBBOBmany.tex.'
-
+        run_latex_template("templateBBOBmany.tex")
+
         t0 = time.time()
         result = os.system(python + command + '--conv' +
                     join_path(data_path, 'SMAC-BBOB_hutter_noiseless.tgz') +
                     join_path(data_path, 'lmm-CMA-ES_auger_noiseless.tgz'))
         print('**  subtest 3 finished in ', time.time() - t0, ' seconds')
         assert result == 0, 'Test failed: rungeneric on two algorithms with option --conv.'
-
-        result = run_latex_template("templateBBOBcmp.tex")
-        assert not result, 'Test failed: error while generating pdf from templateBBOBcmp.tex.'
-
+        run_latex_template("templateBBOBcmp.tex")
+
         t0 = time.time()
         result = os.system(python + command + ' --omit-single ' +
                     join_path(data_path, 'DE-PSO_garcia-nieto_noiseless.tgz') +
                     join_path(data_path, 'VNS_garcia-martinez_noiseless.tgz'))
         print('**  subtest 4 finished in ', time.time() - t0, ' seconds')
         assert result == 0, 'Test failed: rungeneric on two algorithms with option --omit-single.'
-
-        result = run_latex_template("templateBBOBcmp.tex")
-        assert not result, 'Test failed: error while generating pdf from templateBBOBcmp.tex.'
-
+        run_latex_template("templateBBOBcmp.tex")
+
         t0 = time.time()
         result = os.system(python + command + ' --expensive ' +
                     join_path(data_path, 'VNS_garcia-martinez_noiseless.tgz'))
         print('**  subtest 5 finished in ', time.time() - t0, ' seconds')
         assert result == 0, 'Test failed: rungeneric on one algorithm with option --expensive.'
-
-        result = run_latex_template("templateBBOBarticle.tex")
-        assert not result, 'Test failed: error while generating pdf from templateBBOBarticle.tex.'
+        run_latex_template("templateBBOBarticle.tex")
 
     print('launching doctest (it might be necessary to close a few pop up windows to finish)')
     t0 = time.time()
@@ -232,17 +230,17 @@ def main(args):
         test_count = 0
         #doctest.testmod(report=True, verbose=True)  # this is quite cool!
         # go through the py files in the bbob_pproc folder
-        currentPath = os.getcwd()        
+        currentPath = os.getcwd()
         newPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
-        os.chdir(newPath)        
+        os.chdir(newPath)
         for root, dirnames, filenames in os.walk(os.path.dirname(os.path.realpath(__file__))):
-          for filename in fnmatch.filter(filenames, '*.py'):
-            current_failure_count, current_test_count = doctest.testfile(
-                os.path.join(root, filename), report=True, module_relative=False)              
-            failure_count += current_failure_count
-            test_count += current_test_count
-            if current_failure_count:
-                print('doctest file "%s" failed' % os.path.join(root, filename))
+            for filename in fnmatch.filter(filenames, '*.py'):
+                current_failure_count, current_test_count = doctest.testfile(
+                    os.path.join(root, filename), report=True, module_relative=False)
+                failure_count += current_failure_count
+                test_count += current_test_count
+                if current_failure_count:
+                    print('doctest file "%s" failed' % os.path.join(root, filename))
         os.chdir(currentPath)
     else:
         stdout = sys.stdout
@@ -257,8 +255,8 @@ def main(args):
     print('** doctest finished in ', time.time() - t0, ' seconds')
     # print('    more info in file _bbob_pproc_doctest_.txt)')
     print('*** done testing module bbob_pproc ***')
-    
-    if (failure_count > 0):
+
+    if failure_count > 0:
         raise ValueError('%d of %d tests failed' % (failure_count, test_count))
 
 """
@@ -287,7 +285,8 @@ def main(args):
         if is_module:
             rungeneric.main(args)  # just prints help
         else:
-            print("WARNING: this tests the post-processing, this might change in future (use -h for help)")
+            print("WARNING: this tests the post-processing, this might change "
+                  + "in future (use -h for help)")
             main(args)
     elif args[0] == '-t' or args[0].startswith('--t'):
         args.pop(0)

code-postprocessing/bbob_pproc/algportfolio.py

@@ -49,11 +49,7 @@
 
 from __future__ import absolute_import
 
-import os
 import sys
-import glob
-import getopt
-import pickle
 from pdb import set_trace
 import warnings
 import numpy as np
@@ -78,10 +74,10 @@ class DataSet(pp.DataSet):
 
     def __init__(self, dslist):
         """Instantiate one algorithm portfolio data set.
-        
+
         :param dict dslist: list of :py:class:`pproc.DataSetList`
                             instances.
-        
+
         """
 
         def _conv_evals(evals, algnb, maxevals):