Fix Akumuli

Fix test mode
Add missing vagrant files to newest databases

TSDBBench.py

@@ -167,6 +167,7 @@ def check_result_file_extended(path, workload, logger):
         file = open(path, "r")
         resultDict={}
         error = False
+        atLeastOneReturnedZeroDict = {}
         for line in file:
             if re.match("\[(INSERT|READ|SCAN|AVG|COUNT|SUM)\],\s*(Return=|Operations).+$", line) != None:
                 splitters = line.split(",")
@@ -179,6 +180,14 @@ def check_result_file_extended(path, workload, logger):
                     else:
                         resultDict[queryType] = [amount,0]
                 elif "Return=" in lineType:
+                    # check if at least a few non-INSERT queries returned 0 (=succesful)
+                    # INSERT queries must return 0, -1 is not allowed
+                    if queryType not in atLeastOneReturnedZeroDict.keys():
+                        atLeastOneReturnedZeroDict[queryType] = False
+                    if "Return=0" in lineType and "INSERT" in queryType and amount == resultDict[queryType][0]:
+                        atLeastOneReturnedZeroDict[queryType] = True
+                    elif "Return=0" in lineType and amount > 0:
+                        atLeastOneReturnedZeroDict[queryType] = True
                     if queryType not in resultDict.keys():
                         error = True # should already be found in operations line
                     else:
@@ -189,6 +198,9 @@ def check_result_file_extended(path, workload, logger):
                 sum += resultDict[key][1]
             if resultDict[key][0] != resultDict[key][1]:
                 return True
+        for key in atLeastOneReturnedZeroDict:
+            if not atLeastOneReturnedZeroDict[key]:
+                return True
         if (workload == "testworkloada" and len(resultDict.keys()) != 2 and sum !=  resultDict["INSERT"][1]) or \
            (workload == "testworkloadb" and len(resultDict.keys()) != 5 and sum !=  resultDict["INSERT"][1])    :
             return True

Vm.py

@@ -11,6 +11,7 @@
 import Util
 import re
 from fabric.api import *
+import subprocess
 
 class Vm():
 
@@ -59,7 +60,16 @@ def run(self):
 
     def destroy(self):
         if self.vm != None:
-            return self.vm.destroy()
+            try:
+                if self.vm.status()[0].state == vagrant.Vagrant.RUNNING:
+                    self.vm.halt()
+                if self.vm.status()[0].state != vagrant.Vagrant.POWEROFF:
+                    self.vm.halt(force=True)
+                return self.vm.destroy()
+            except subprocess.CalledProcessError:
+                # ignore warnings that destroy did not work (occurs sometimes)
+                self.logger.warning("There might be some leftovers from vm '%s'." %(self.name))
+                return True
         return True
 
     def create_vm(self):
@@ -151,7 +161,7 @@ def create_vm(self):
             #if not self.vm.destroy():
                 # self.logger.error('Can not destroy %s.' %(self.name), exc_info=True)
             ## vm.destroy() seems to be always returning false at this stage (opentsack), but destroying works fine -> Ignore it.
-            self.vm.destroy()
+            self.destroy()
             return False
         self.logger.info("END Creating %s." %(self.name))
         self.logger.info("GET IP of %s." %(self.name))
@@ -160,7 +170,7 @@ def create_vm(self):
         if self.ip == None:
             self.logger.error('Failed getting IP while creating vm %s.' %(self.name), exc_info=True)
             self.logger.info('Since creation failed, trying to destroy vm %s.' %(self.name), exc_info=True)
-            self.vm.destroy()
+            self.destroy()
             return False
         self.created=True
         return True

vagrant_files/akumuli_cl1_rf1/akumuli_cl1_rf1_0.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/copy_files.rb'
 load 'basic/akumuli.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/akumuli_cl1_rf1/files/akumuli.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=akumuli
+
+[Service]
+User=root
+Group=root
+WorkingDirectory=/home/vagrant/Akumuli/akumulid
+ExecStart=/home/vagrant/Akumuli/akumulid/akumulid
+
+[Install]
+WantedBy=multi-user.target

vagrant_files/axibase1/axibase1_0.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/axibase.rb'
 load 'basic/copy_files.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/basic/akumuli.rb

@@ -1,8 +1,15 @@
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # Installing deps
-  config.vm.provision "shell", inline: "cp /vagrant/files/libakumuli_so /vagrant/files/libakumuli.so"
+  config.vm.provision "shell", inline: "apt-get --allow-unauthenticated -q -y  install git cmake libboost-dev libboost-system-dev libboost-thread-dev libboost-filesystem-dev libboost-test-dev libboost-coroutine-dev libboost-context-dev libboost-program-options-dev libboost-regex-dev libapr1-dev libaprutil1-dev libaprutil1-dbd-sqlite3 libmicrohttpd-dev liblog4cxx10-dev liblog4cxx10 libjemalloc-dev libsqlite3-dev"
+  config.vm.provision "shell", inline: "cd /home/vagrant/ && git clone \"#{$git_akumuli_link}\""
+  config.vm.provision "shell", inline: "cd /home/vagrant/Akumuli && git checkout \"#{$git_akumuli_hash}\""
+  config.vm.provision "shell", inline: "cd /home/vagrant/Akumuli && cmake ."
+  config.vm.provision "shell", inline: "cd /home/vagrant/Akumuli && make -j"
   config.vm.provision "shell", inline: "apt-get -q -y install sysstat libc6 net-tools netcat-openbsd libboost-dev libapr1 libaprutil1 libaprutil1-dbd-sqlite3  liblog4cxx10 libboost1.55-all-dev libmicrohttpd-dev libjemalloc-dev libsqlite3-dev"
-  config.vm.provision "shell", inline: "cd /vagrant/files/ ; ./akumulid --init"
-  config.vm.provision "shell", inline: "cd /vagrant/files/ ; ./akumulid --create"
-  config.vm.provision "shell", inline: '/vagrant/files/daemonize -c /vagrant/files/ -e /var/log/akumulid.err -o /var/log/akumulid.log -l /var/run/akumulid.lock -v /vagrant/files/akumulid'
+  config.vm.provision "shell", inline: "cd /home/vagrant/Akumuli/akumulid && ./akumulid --init"
+  config.vm.provision "shell", inline: "cd /home/vagrant/Akumuli/akumulid && ./akumulid --create"
+  config.vm.provision "shell", inline: "cp /vagrant/files/akumuli.service /etc/systemd/system"
+  config.vm.provision "shell", inline: "chown root:root /etc/systemd/system/akumuli.service"
+  config.vm.provision "shell", inline: "systemctl daemon-reload"
+  config.vm.provision "shell", inline: "systemctl start akumuli.service"  
 end

vagrant_files/databus_rtable_cl1_rf1/databus_rtable_cl1_rf1_0.vagrant

@@ -10,8 +10,10 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/copy_files.rb'
 load 'basic/cassandra.rb'
 load 'basic/databus.rb'
+load 'basic/provider_specific_end.rb'

vagrant_files/databus_rtstable_cl1_rf1/databus_rtstable_cl1_rf1_0.vagrant

@@ -10,8 +10,10 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/copy_files.rb'
 load 'basic/cassandra.rb'
 load 'basic/databus.rb'
+load 'basic/provider_specific_end.rb'

vagrant_files/databus_tstable_cl1_rf1/databus_tstable_cl1_rf1_0.vagrant

@@ -10,8 +10,10 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/copy_files.rb'
 load 'basic/cassandra.rb'
 load 'basic/databus.rb'
+load 'basic/provider_specific_end.rb'

vagrant_files/elasticsearch_cl1_rf1/elasticsearch_cl1_rf1_0.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/elasticsearch.rb'
 load 'basic/copy_files.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/elasticsearch_cl5_rf1/elasticsearch_cl5_rf1_0.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/elasticsearch.rb'
 load 'basic/copy_files.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/elasticsearch_cl5_rf1/elasticsearch_cl5_rf1_1.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm1"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/elasticsearch.rb'
 load 'basic/copy_files.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/elasticsearch_cl5_rf1/elasticsearch_cl5_rf1_2.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm2"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/elasticsearch.rb'
 load 'basic/copy_files.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/elasticsearch_cl5_rf1/elasticsearch_cl5_rf1_3.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm3"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/elasticsearch.rb'
 load 'basic/copy_files.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/elasticsearch_cl5_rf1/elasticsearch_cl5_rf1_4.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm4"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/elasticsearch.rb'
 load 'basic/copy_files.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/gnocchi_cl1/gnocchi_cl1_0.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/gnocchi.rb'
 load 'basic/copy_files.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/graphite_cl1_rf1/graphite_cl1_rf1_0.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/copy_files.rb'
 load 'basic/graphite.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/kdbplus_cl1_rf1/kdbplus_cl1_rf1_0.vagrant

@@ -7,17 +7,19 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
+load 'basic/provider_specific.rb'
+load 'basic/change_hostname.rb'
+load 'basic/copy_files.rb'
+load 'basic/update.rb'
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  config.vm.provision "shell", inline: "/root/change_hostname.sh vm0"
-  config.vm.provision "shell", privileged: false,  inline: "if [ -d /vagrant/files ]; then cp -r /vagrant/files ~/; fi"
-  config.vm.provision "shell", privileged: false,  inline: "if [ -d ~/files ]; then chmod +x -f ~/files/*.sh; exit 0; fi"   
-  config.vm.provision "shell", inline: "apt-get --allow-unauthenticated -q -y update"
   config.vm.provision "shell", inline: "apt-get --allow-unauthenticated -q -y install unzip libc6-i386"
   config.vm.provision "shell", privileged: false, inline: "unzip ~/files/linux.zip -d ~/"
   config.vm.provision "shell", privileged: false, inline: "gcc ~/files/daemonize.c -o ~/daemonize"
   config.vm.provision "shell", privileged: false, inline: "~/daemonize -e ~/q/l32/stderr -o ~/q/l32/stdout -p ~/q/l32/pidfile ~/q/l32/q -p 5001"
   # config.vm.provision "shell", privileged: false, inline: "~/daemonize -e ~/q/l32/stderr -o ~/q/l32/stdout -p ~/q/l32/pidfile ~/q/l32/q ~/q/l32/qlog -l -p 5001"
 
 end
+
+load 'basic/provider_specific_end.rb'

vagrant_files/seriesly_cl1_rf1/seriesly_cl1_rf1_0.vagrant

@@ -7,9 +7,9 @@ HOSTNAME = "vm0"
 
 load 'vagrantconf.rb'
 load 'vagrantconf_db.rb'
-
+load 'basic/provider_specific.rb'
 load 'basic/change_hostname.rb'
 load 'basic/update.rb'
 load 'basic/copy_files.rb'
 load 'basic/seriesly.rb'
-
+load 'basic/provider_specific_end.rb'

vagrant_files/vagrantconf.rb

@@ -67,8 +67,12 @@
 $links_ycsbts = 'http://nemarcontrolvm.iaas.uni-stuttgart.de/bin/ycsb-0.4.0.tar.gz'
 $links_blueflood = 'http://nemarcontrolvm.iaas.uni-stuttgart.de/bin/blueflood-all-2.0.0-SNAPSHOT-jar-with-dependencies.jar'
 
+# Akumuli git link and commit id
+$git_akumuli_link = 'https://github.com/akumuli/Akumuli.git'
+$git_akumuli_hash = 'fa85dd90a2e09bc28630bbe3fdd28fe155566360'
+
 # h5serv (hdf5) git commit ids
-$git_h5json = 'c31bfb7ff2da864a737ce9464ba1895b0d5af8f4'
+$git_h5json = 'a85c6b195dc63687391f8e69be3f6d595679c0f5'
 $git_h5serv = '18709351fb173697e323eb62828cb8868f2e1db3'
 
 # hbase