Merge pull request apel#354 from Sae126V/apel-codebase-migration-to-p…

…ython3

Apel codebase migration to python3

.github/workflows/unit-test.yml

@@ -49,7 +49,7 @@ jobs:
     - name: Set up dependencies for mysqlclient
       run: sudo apt-get install default-libmysqlclient-dev pkg-config
 
-    - name: Base requirements for SSM
+    - name: Base requirements for APEL
       run: pip install -r requirements.txt
 
     - name: Additional requirements for the unit and coverage tests

.travis.yml

@@ -19,9 +19,9 @@ before_cache: rm -fv ~/.cache/pip/log/debug.log
 
 install:
   # Base requirements for apel
-  - pip install -r requirements.txt
+  - pip install -r requirements-py2.txt
   # Additional requirements for the unit and coverage tests
-  - pip install -r requirements-test.txt
+  - pip install -r requirements-test-py2.txt
 
 # Commands to prepare environment for the test
 before_script:

apel.spec

@@ -25,7 +25,7 @@ The project is written in Python.
 %package lib
 Summary:        Libraries required for Apel Client, Server and Parsers
 Group:          Development/Languages
-Requires:       MySQL-python, python-ldap < 3.4.0 , python-iso8601, python-dirq
+Requires:       MySQL-python, python-ldap < 3.4.0 , python-iso8601, python-dirq, python-future
 Requires(pre):  shadow-utils
 
 %description lib

apel/common/__init__.py

@@ -16,10 +16,12 @@
    @author: Konrad Jopek
 '''
 
-from datetime_utils import valid_from,valid_until, parse_timestamp, parse_time, iso2seconds
-from exceptions import install_exc_handler, default_handler
-from parsing_utils import parse_fqan
-from hashing import calculate_hash
+from __future__ import absolute_import
+
+from .datetime_utils import valid_from,valid_until, parse_timestamp, parse_time, iso2seconds
+from .exceptions import install_exc_handler, default_handler
+from .parsing_utils import parse_fqan
+from .hashing import calculate_hash
 
 import logging
 import sys

apel/common/hashing.py

@@ -29,25 +29,26 @@ def calculate_hash(fname):
     For sample usage please go to: apel2/bin/client.py
     '''
 
-    data = 'initial'
+    data = b'initial'
 
     md = md5()
 
     # try opening as a gzip file, and if it fails
     # try as a regular file
     try:
-        fp = gzip.open(fname, 'r')
-        while data != '':
+        fp = gzip.open(fname, 'rb')
+        while data != b'':
             # 128kiB buffer
             data = fp.read(131072)
             md.update(data)
     except IOError: # not a gzipped file
-        fp = open(fname, 'r')
-        while data != '':
+        fp = open(fname, 'rb')
+        while data != b'':
             # 128kiB buffer
             data = fp.read(131072)
             md.update(data)
 
+    finally:
+        fp.close()
 
-    fp.close()
     return md.hexdigest()

apel/db/apeldb.py

@@ -16,9 +16,13 @@
    @author: Konrad Jopek
 '''
 
+from future.builtins import object, str
+
 import logging
+
 from apel.db import LOGGER_ID
 
+
 logger = logging.getLogger(LOGGER_ID)
 
 class ApelDbException(Exception):
@@ -44,7 +48,7 @@ def __new__(cls, backend, host, port, username, pwd, db):
         except ImportError:
             logger.info('Cannot import mysql backend')
 
-        if backend not in BACKENDS.keys():
+        if backend not in list(BACKENDS.keys()):
             raise ApelDbException('Unknown backend: %s' % (backend))
 
         backend = BACKENDS[backend]

apel/db/backends/mysql.py

@@ -18,6 +18,7 @@
     @author: Will Rogers, Konrad Jopek
 '''
 
+from future.builtins import object, str
 
 from apel.db import ApelDbException
 from apel.db.records import (BlahdRecord,

apel/db/loader/__init__.py

@@ -18,9 +18,11 @@
 Module containing the Loader object.
 '''
 
+from __future__ import absolute_import
+
 # public interface for apel.dbloader
 
-from loader import Loader, LoaderException
-from car_parser import CarParser
-from aur_parser import AurParser
-from star_parser import StarParser
+from .loader import Loader, LoaderException
+from .car_parser import CarParser
+from .aur_parser import AurParser
+from .star_parser import StarParser

apel/db/loader/aur_parser.py

@@ -16,8 +16,10 @@
    @author: Will Rogers, Konrad Jopek, Stuart Pullinger
 '''
 
+from __future__ import absolute_import
+
 from apel.common import iso2seconds, parse_timestamp
-from xml_parser import XMLParser, XMLParserException
+from .xml_parser import XMLParser, XMLParserException
 from apel.db.records.normalised_summary import NormalisedSummaryRecord
 from apel.db.loader.car_parser import CarParser
 import logging
@@ -128,9 +130,9 @@ def parseAurRecord(self, xml_record):
         for field in functions:
             try:
                 data[field] = functions[field](nodes)
-            except IndexError, e:
+            except IndexError as e:
                 log.debug('Failed to parse field %s: %s', field, e)
-            except KeyError, e:
+            except KeyError as e:
                 log.debug('Failed to parse field %s: %s', field, e)
 
         nsr = NormalisedSummaryRecord()

apel/db/loader/car_parser.py

@@ -16,8 +16,10 @@
    @author: Konrad Jopek, Will Rogers
 '''
 
+from __future__ import absolute_import
+
 from apel.common import iso2seconds, parse_timestamp
-from xml_parser import XMLParser, XMLParserException
+from .xml_parser import XMLParser, XMLParserException
 from apel.db.records.job import JobRecord
 import logging
 
@@ -157,7 +159,7 @@ def parse_car(self, xml_record):
         for field in functions:
             try:
                 data[field] = functions[field](nodes)
-            except (IndexError, KeyError, AttributeError), e:
+            except (IndexError, KeyError, AttributeError) as e:
                 log.debug('Failed to parse field %s: %s', field, e)
 
         jr = JobRecord()

apel/db/loader/loader.py

@@ -18,16 +18,20 @@
 Module containing the Loader class.
 '''
 
+from __future__ import absolute_import
+from future.builtins import next, object, str
+
 import logging
 import os
-from xml.parsers.expat import ExpatError
+import sys
+from xml.parsers.expat import ExpatError, errors
 
 from dirq.queue import Queue
 
 import apel.db
 from apel.db.loader.xml_parser import XMLParserException
 from apel.db.records import InvalidRecordException
-from record_factory import RecordFactory, RecordFactoryException
+from .record_factory import RecordFactory, RecordFactoryException
 
 
 # set up the logger
@@ -136,7 +140,7 @@ def load_all_msgs(self):
         while self.current_msg:
             if not self._inq.lock(self.current_msg):
                 log.warning("Skipping locked message %s", self.current_msg)
-                self.current_msg = self._inq.next()
+                self.current_msg = next(self._inq, None)
                 continue
             log.debug("Reading message %s", self.current_msg)
             data = self._inq.get(self.current_msg)
@@ -157,7 +161,11 @@ def load_all_msgs(self):
             except (RecordFactoryException, LoaderException,
                     InvalidRecordException, apel.db.ApelDbException,
                     XMLParserException, ExpatError) as err:
-                errmsg = "Parsing unsuccessful: %s" % str(err)
+                if sys.version_info >= (3,) and isinstance(err, ExpatError):
+                    errmsg = "Parsing unsuccessful: %s" % str(errors.messages[err.code])
+                else:
+                    errmsg = "Parsing unsuccessful: %s" % str(err)
+
                 log.warning('Message rejected. %s', errmsg)
                 name = self._rejectq.add({"body": msg_text,
                                           "signer": signer,
@@ -167,7 +175,7 @@ def load_all_msgs(self):
 
             log.info("Removing message %s. ID = %s", self.current_msg, msg_id)
             self._inq.remove(self.current_msg)
-            self.current_msg = self._inq.next()
+            self.current_msg = next(self._inq, None)
 
         if num_msgs:  # Only tidy up if messages found
             log.info('Tidying message directories')

apel/db/loader/record_factory.py

@@ -18,6 +18,8 @@
 Module containing the RecordFactory class.
 '''
 
+from future.builtins import object
+
 from apel.db.records.job import JobRecord
 from apel.db.records.summary import SummaryRecord
 from apel.db.records.normalised_summary import NormalisedSummaryRecord

apel/db/loader/star_parser.py

@@ -15,13 +15,14 @@
 
 '''
 
+from __future__ import absolute_import
 
 import logging
 
 from apel.db.records.storage import StorageRecord
 from apel.db.records.group_attribute import GroupAttributeRecord
 from apel.common.datetime_utils import parse_timestamp
-from xml_parser import XMLParser, XMLParserException
+from .xml_parser import XMLParser, XMLParserException
 
 
 log = logging.getLogger(__name__)
@@ -117,8 +118,8 @@ def parseStarRecord(self, xml_storage_record):
 
         # Here we copy keys from functions.
         # We only want to change 'RecordId' to 'RecordIdentity'.
-        nodes = {}.fromkeys(map(lambda f: f == 'RecordId' and
-                                'RecordIdentity' or f, [S for S in functions]))
+        nodes = {}.fromkeys([f == 'RecordId' and
+                                'RecordIdentity' or f for f in [S for S in functions]])
         # nodes = {}.fromkeys(functions.keys())
         data = {}
 
@@ -136,7 +137,7 @@ def parseStarRecord(self, xml_storage_record):
         for field in functions:
             try:
                 data[field] = functions[field](nodes)
-            except (IndexError, KeyError), e:
+            except (IndexError, KeyError) as e:
                 log.debug("Failed to get field %s: %s", field, e)
 
         sr = StorageRecord()

apel/db/loader/xml_parser.py

@@ -15,6 +15,9 @@
 
    @author: Konrad Jopek
 '''
+
+from future.builtins import object
+
 import xml.dom.minidom
 
 class XMLParserException(Exception):

apel/db/records/__init__.py

@@ -19,19 +19,22 @@
 
 Interface for apel.db.records.
 '''
+
+from __future__ import absolute_import
+
 import time
 import datetime
 
-from record import Record, InvalidRecordException
-from blahd import BlahdRecord
-from cloud import CloudRecord
-from cloud_summary import CloudSummaryRecord
-from event import EventRecord
-from group_attribute import GroupAttributeRecord
-from job import JobRecord
-from processed import ProcessedRecord
-
-from storage import StorageRecord
-from summary import SummaryRecord
-from normalised_summary import NormalisedSummaryRecord
-from sync import SyncRecord
+from .record import Record, InvalidRecordException
+from .blahd import BlahdRecord
+from .cloud import CloudRecord
+from .cloud_summary import CloudSummaryRecord
+from .event import EventRecord
+from .group_attribute import GroupAttributeRecord
+from .job import JobRecord
+from .processed import ProcessedRecord
+
+from .storage import StorageRecord
+from .summary import SummaryRecord
+from .normalised_summary import NormalisedSummaryRecord
+from .sync import SyncRecord

apel/db/records/cloud.py

@@ -16,6 +16,8 @@
 @author Will Rogers
 '''
 
+from future.builtins import str
+
 from apel.db.records import Record, InvalidRecordException
 from apel.common import parse_fqan
 from datetime import datetime, timedelta
@@ -127,5 +129,5 @@ def _check_start_end_times(self):
             if datetime.fromtimestamp(end) > tomorrow:
                 raise InvalidRecordException("Epoch time " + str(end) + " is in the future.")
 
-        except ValueError:
+        except (ValueError, OverflowError, OSError):
             raise InvalidRecordException("Cannot parse an integer from StartTime or EndTime.")

apel/db/records/job.py

@@ -16,6 +16,8 @@
     @author Will Rogers, Konrad Jopek
 '''
 
+from future.builtins import str
+
 from apel.db.records import Record, InvalidRecordException
 from datetime import datetime, timedelta
 from xml.dom.minidom import Document
@@ -257,26 +259,26 @@ def get_ur(self, withhold_dns=False):
         service_level.appendChild(doc.createTextNode(str(self.get_field('ServiceLevel'))))
         ur.appendChild(service_level)
 
-        if self.get_field('MemoryReal') > 0:
+        if self.get_field('MemoryReal') is not None and int(self.get_field('MemoryReal')) > 0:
             pmem = doc.createElement('urf:Memory')
             pmem.setAttribute('urf:type', 'Physical')
             pmem.setAttribute('urf:storageUnit', 'KB')
             pmem.appendChild(doc.createTextNode(str(self.get_field('MemoryReal'))))
             ur.appendChild(pmem)
 
-        if self.get_field('MemoryVirtual') > 0:
+        if self.get_field('MemoryVirtual') is not None and int(self.get_field('MemoryVirtual')) > 0:
             vmem = doc.createElement('urf:Memory')
             vmem.setAttribute('urf:type', 'Shared')
             vmem.setAttribute('urf:storageUnit', 'KB')
             vmem.appendChild(doc.createTextNode(str(self.get_field('MemoryVirtual'))))
             ur.appendChild(vmem)
 
-        if self.get_field('NodeCount') > 0:
+        if self.get_field('NodeCount') is not None and int(self.get_field('NodeCount')) > 0:
             ncount = doc.createElement('urf:NodeCount')
             ncount.appendChild(doc.createTextNode(str(self.get_field('NodeCount'))))
             ur.appendChild(ncount)
 
-        if self.get_field('Processors') > 0:
+        if self.get_field('Processors') is not None and int(self.get_field('Processors')) > 0:
             procs = doc.createElement('urf:Processors')
             procs.appendChild(doc.createTextNode(str(self.get_field('Processors'))))
             ur.appendChild(procs)