Cores and memory (#25)

* Cores and memory

1. Allow users to set the number of cores and memory to be used by the HTCondor job.
2. Display the number of cores and memory for each job, if this attribute is not available, the placeholder ‘()’ is going to be set.
3. Include the constraint described in the issue into the frontend and backend sides.
4. Set the default value for old Gridpack requests if they are reset and the number of cores and memory is not available.

* Include more core options

Allow users to set 1 and 2 cores for the HTCondor jobs

controller.py

@@ -34,6 +34,8 @@ def __init__(self):
         self.gridpacks_to_create_requests = []
         self.repository_tick_pause = 60
         self.tick_lock = Lock()
+        self.job_cores = [1, 2, 4, 8, 16, 32, 64]
+        self.job_memory = [cores * 1000 for cores in self.job_cores]
 
     def update_repository_tree(self):
         now = int(time.time())

frontend/index.html

@@ -163,6 +163,8 @@
         <th>
           <span class="sort" @click="onSort('condor_status')">Job</span>
         </th>
+        <th>Cores</th>
+        <th>Memory</th>
         <th>Gridpack</th>
         <th>PrepID</th>
         <th>
@@ -214,6 +216,16 @@
         <td>
           <pre style="margin: 0">{{gridpack.condor_status}} ({{gridpack.condor_id}})</pre>
         </td>
+        <!-- Number of cores-->
+        <td>
+          <pre v-if="gridpack.job_cores" style="margin: 0">{{gridpack.job_cores}}</pre>
+          <pre v-else style="margin: 0">()</pre>
+        </td>
+        <!-- Memory -->
+        <td>
+          <pre v-if="gridpack.job_memory" style="margin: 0">{{gridpack.job_memory}} MB</pre>
+          <pre v-else style="margin: 0">()</pre>
+        </td>
         <td>
           <a v-if="dev" :href="'https://pdmv-gridpacks.web.cern.ch/?q=' + gridpack.archive" target="_blank" style="letter-spacing: -0.4px">{{gridpack.archive}}</a>
           <span v-if="!dev" style="letter-spacing: -0.4px">{{gridpack.archive}}</span>
@@ -253,6 +265,8 @@ <h5 class="modal-title" id="createGridpackModalLabel">
                 <th>Tune</th>
                 <th>Events</th>
                 <th>GEN productions</th>
+                <th>Job cores</th>
+                <th>Job memory</th>
                 <th>Actions</th>
               </tr>
               <tr v-for="(wizardObject, index) in wizardObjects">
@@ -292,6 +306,16 @@ <h5 class="modal-title" id="createGridpackModalLabel">
                     <option v-for="branch in systemInfo.options.branches" :key="branch">{{branch}}</option>
                   </select>
                 </td>
+                <td>
+                  <select v-model="wizardObject.job_cores">
+                    <option v-for="cores_option in systemInfo.job_cores" :key="cores_option">{{cores_option}}</option>
+                  </select>
+                </td>
+                <td>
+                  <select v-model="wizardObject.job_memory">
+                    <option v-for="memory_option in systemInfo.job_memory" :key="memory_option">{{memory_option}}</option>
+                  </select>
+                </td>
                 <td style="text-align: left;">
                   <span style="cursor: pointer;" title="Delete row" @click="wizardDeleteRow(wizardObject)">&#10007;</span>
                   <span style="cursor: pointer;" title="Clone row" @click="wizardCloneRow(wizardObject)">&#10697;</span>
@@ -576,6 +600,20 @@ <h5 class="modal-title" id="createGridpackModalLabel">
               alert('Please enter a positive number of events in row ' + index);
               return;
             }
+            gridpack.job_cores = parseInt(gridpack.job_cores);
+            if (gridpack.job_cores == undefined || isNaN(gridpack.job_cores)) {
+              alert('Please set the number of cores in row ' + index);
+              return;
+            }
+            gridpack.job_memory = parseInt(gridpack.job_memory);
+            if (gridpack.job_memory == undefined || isNaN(gridpack.job_memory)) {
+              alert('Please set the memory for the job in row ' + index);
+              return;
+            }
+            if (gridpack.job_memory < gridpack.job_cores * 1000) {
+              alert(`Please avoid to set the memory less than ${gridpack.job_cores * 1000} MB in row ${index}`);
+              return;
+            }
             index++;
           }
           this.createGridpacks(this.wizardObjects);
@@ -596,6 +634,8 @@ <h5 class="modal-title" id="createGridpackModalLabel">
             events: wizardObject.events,
             tune: wizardObject.tune,
             genproductions: wizardObject.genproductions,
+            job_cores: wizardObject.job_cores,
+            job_memory: wizardObject.job_memory
           });
         },
         wizardMoveRowUp: function(wizardObject) {

gridpack.py

@@ -10,8 +10,7 @@
 from user import User
 
 
-CORES = 16
-MEMORY = CORES * 2000
+MEMORY_FACTOR_MB = 1000
 
 
 class Gridpack():
@@ -33,7 +32,9 @@ class Gridpack():
         'dataset_name': '',
         'history': [],
         'prepid': '',
-        'store_into_subfolders': False
+        'store_into_subfolders': False,
+        'job_cores': 16,
+        'job_memory': 32000
     }
 
     def __init__(self, data):
@@ -100,6 +101,12 @@ def validate(self):
         dataset = self.data['dataset']
         if dataset not in cards[generator][process]:
             return f'Bad dataset "{dataset}"'
+
+        memory = self.data['job_memory']
+        cores = self.data['job_cores']
+        minimum_memory = cores * MEMORY_FACTOR_MB
+        if memory < minimum_memory:
+            return f'Memory set for Gridpack should be equal or greater than {minimum_memory} MB'
 
         return None
 
@@ -133,6 +140,18 @@ def set_condor_status(self, condor_status):
 
     def get_condor_id(self):
         return self.data['condor_id']
+
+    def get_cores(self):
+        return self.data.get(
+            'job_cores',
+            Gridpack.schema['job_cores']
+        )
+
+    def get_memory(self):
+        return self.data.get(
+            'job_memory',
+            Gridpack.schema['job_memory']
+        )
 
     def set_condor_id(self, condor_id):
         """
@@ -341,7 +360,7 @@ def prepare_script(self):
         command = ['#!/bin/sh',
                    'export HOME=$(pwd)',
                    'export ORG_PWD=$(pwd)',
-                   f'export NB_CORE={CORES}',
+                   f'export NB_CORE={self.get_cores()}',
                    f'wget https://github.com/{repository}/tarball/{genproductions} -O genproductions.tar.gz',
                    'tar -xzf genproductions.tar.gz',
                    f'GEN_FOLDER=$(ls -1 | grep {repository.replace("/", "-")}- | head -n 1)',
@@ -384,8 +403,8 @@ def prepare_jds_file(self):
             "output                  = output.log",
             "error                   = error.log",
             "log                     = job.log",
-            f"RequestCpus            = {CORES}",
-            f"RequestMemory          = {MEMORY}",
+            f"RequestCpus            = {self.get_cores()}",
+            f"RequestMemory          = {self.get_memory()}",
             '+REQUIRED_OS            = "rhel7"',
             '+AccountingGroup        = "group_u_CMS.u_zh.priority"',
             "leave_in_queue          = JobStatus == 4 && (CompletionDate =?= UNDEFINED || ((CurrentTime - CompletionDate) < 7200))",

main.py

@@ -86,7 +86,9 @@ def system_info():
     return output_text({'last_tick': controller.last_tick,
                         'last_repository_tick': controller.last_repository_tick,
                         'options': controller.repository_tree,
-                        'gen_repository': Config.get('gen_repository')})
+                        'gen_repository': Config.get('gen_repository'),
+                        'job_cores': controller.job_cores,
+                        'job_memory': controller.job_memory})
 
 
 @app.route('/api/mcm', methods=['POST'])