feat: added gruop_by on hosts and schedules

removed ansible facts gathering
test fixes

Closes: #464

Change-Id: I01871d3d209d25877f57fac262bb7b185dd6028b

conf/quads.yml

@@ -123,15 +123,6 @@ json_web_path: /var/www/html/cloud
 # number of days of retaining old .json files
 json_retention_days: 0
 
-# Whether or not you want the QUADS host to gather and display ansible facts in
-# an HTMl page, you need ansible-cmdb rpm for this functionality which can be
-# got from https://github.com/fboender/ansible-cmdb/releases
-gather_ansible_facts: false
-
-# this is where we place the generated ansible configuration management database
-# html
-ansible_facts_web_path: /var/www/html/ansible_facts
-
 # untouchable_hosts are hosts that should be avoided by QUADS in any way.
 # use this to define hosts QUADS should never move.
 untouchable_hosts: foreman.example.com c08-h30-r630.example.com

quads/quads_api.py

@@ -37,15 +37,11 @@ def __init__(self, config: Config):
         self.config = config
         self.base_url = config.API_URL
         self.session = requests.Session()
-        self.auth = HTTPBasicAuth(
-            self.config.get("quads_api_username"), self.config.get("quads_api_password")
-        )
+        self.auth = HTTPBasicAuth(self.config.get("quads_api_username"), self.config.get("quads_api_password"))
 
     # Base functions
     def get(self, endpoint: str) -> Response:
-        _response = self.session.get(
-            os.path.join(self.base_url, endpoint), verify=False, auth=self.auth
-        )
+        _response = self.session.get(os.path.join(self.base_url, endpoint), verify=False, auth=self.auth)
         if _response.status_code == 500:
             raise APIServerException("Check the flask server logs")
         if _response.status_code == 400:
@@ -85,9 +81,7 @@ def patch(self, endpoint, data) -> Response:
         return _response
 
     def delete(self, endpoint) -> Response:
-        _response = self.session.delete(
-            os.path.join(self.base_url, endpoint), verify=False, auth=self.auth
-        )
+        _response = self.session.delete(os.path.join(self.base_url, endpoint), verify=False, auth=self.auth)
         if _response.status_code == 500:
             raise APIServerException("Check the flask server logs")
         if _response.status_code == 400:
@@ -105,6 +99,10 @@ def get_hosts(self) -> List[Host]:
             hosts.append(host_obj)
         return hosts
 
+    def get_host_models(self):
+        response = self.get("hosts?group_by=model")
+        return response.json()
+
     def filter_hosts(self, data) -> List[Host]:
         url_params = url_parse.urlencode(data)
         response = self.get(f"hosts?{url_params}")

quads/server/blueprints/hosts.py

@@ -26,7 +26,14 @@ def get_hosts() -> Response:
 
     else:
         _hosts = HostDao.get_hosts()
-    return jsonify([_host.as_dict() for _host in _hosts])
+
+    if _hosts and type(_hosts[0]) is Host:
+        return jsonify([_host.as_dict() for _host in _hosts])
+    else:
+        for _host in _hosts:
+            return jsonify([tuple(_host) for _host in _hosts])
+
+    return jsonify(_hosts)
 
 
 @host_bp.route("/<hostname>")

quads/server/blueprints/schedules.py

@@ -16,7 +16,7 @@
 def get_schedules() -> Response:
     if request.args:
         try:
-            _schedules = ScheduleDao.filter_schedules(**request.args)
+            _schedules = ScheduleDao.filter_schedule_dict(request.args)
         except (EntryNotFound, InvalidArgument) as ex:
             response = {
                 "status_code": 400,

quads/server/dao/baseDao.py

@@ -1,6 +1,7 @@
 from quads.server.models import Interface, Disk, Memory, Processor, Host, db
 from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy import func
 
 FILTERING_OPERATORS = {
     "==": "eq",
@@ -67,9 +68,21 @@ def safe_commit() -> bool:
             return False
 
     @classmethod
-    def create_query_select(cls, model, filters=None, columns=None):
-        query_columns = cls.create_query_columns(model=model, columns=columns)
-        query = db.session.query(*query_columns).distinct(model.id)
+    def create_query_select(cls, model, filters=None, columns=None, group_by=None):
+        """
+        Create a query to select data from a model with filters and columns.
+        :param model: The model to query.
+        :param filters: A list of filter expressions.
+        :param columns: A list of columns to select.
+        :param group_by: A column to group by.
+        :return: The query result.
+        """
+        if group_by:
+            group_by_column = cls.get_group_by_column(model=model, group_by=group_by)
+            query_columns = [group_by_column, func.count(group_by_column)]
+        else:
+            query_columns = cls.create_query_columns(model=model, columns=columns)
+        query = db.session.query(*query_columns)
         for expression in filters:
             try:
                 column_name, op, value = expression
@@ -99,12 +112,12 @@ def create_query_select(cls, model, filters=None, columns=None):
                         % FILTERING_OPERATORS[op]
                     )
                 except IndexError:  # pragma: no cover
-                    raise Exception(
-                        "Invalid filter operator: %s" % FILTERING_OPERATORS[op]
-                    )
+                    raise Exception("Invalid filter operator: %s" % FILTERING_OPERATORS[op])
                 if value == "null":
                     value = None
                 query = query.filter(getattr(column, attr)(value))
+        if group_by:
+            query = query.group_by(group_by_column)
         return query.all()
 
     @classmethod
@@ -114,8 +127,15 @@ def create_query_columns(cls, model, columns):
 
         cols = []
         for column in columns:
-            attr = getattr(model, column, None)
-            if not attr:
+            _attr = getattr(model, column, None)
+            if not _attr:
                 raise Exception("Invalid column name %s" % column)
-            cols.append(attr)
+            cols.append(_attr)
         return cols
+
+    @classmethod
+    def get_group_by_column(cls, model, group_by):
+        _attr = getattr(model, group_by)
+        if not _attr:
+            raise Exception("Invalid column name %s" % group_by)
+        return _attr

quads/server/dao/host.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from sqlalchemy import Boolean
+from sqlalchemy import Boolean, func
 from sqlalchemy.orm import RelationshipProperty
 from sqlalchemy.orm.relationships import Relationship
 
@@ -19,9 +19,7 @@
 
 class HostDao(BaseDao):
     @classmethod
-    def create_host(
-        cls, name: str, model: str, host_type: str, default_cloud: str
-    ) -> Host:
+    def create_host(cls, name: str, model: str, host_type: str, default_cloud: str) -> Host:
         _host_obj = cls.get_host(name)
         if _host_obj:
             raise EntryExisting
@@ -95,10 +93,16 @@ def get_hosts() -> List[Host]:
         hosts = db.session.query(Host).all()
         return hosts
 
+    @staticmethod
+    def get_host_models():
+        host_models = db.session.query(Host.model, func.count(Host.model)).group_by(Host.model).all()
+        return host_models
+
     @staticmethod
     def filter_hosts_dict(data: dict) -> List[Host]:
         filter_tuples = []
         operator = "=="
+        group_by = None
         for k, value in data.items():
             fields = k.split(".")
             if len(fields) > 2:
@@ -115,6 +119,10 @@ def filter_hosts_dict(data: dict) -> List[Host]:
                         operator = OPERATORS[op]
                         break
 
+            if fields[0].lower() == "group_by":
+                first_field = value
+                group_by = value
+                k = value
             field = Host.__mapper__.attrs.get(first_field)
             if not field:
                 raise InvalidArgument(f"{k} is not a valid field.")
@@ -133,17 +141,16 @@ def filter_hosts_dict(data: dict) -> List[Host]:
                 if first_field.lower() in MAP_HOST_META.keys():
                     if len(fields) > 1:
                         field_name = f"{first_field.lower()}.{field_name.lower()}"
-            filter_tuples.append(
-                (
-                    field_name,
-                    operator,
-                    value,
+
+            if fields[0].lower() != "group_by":
+                filter_tuples.append(
+                    (
+                        field_name,
+                        operator,
+                        value,
+                    )
                 )
-            )
-        if filter_tuples:
-            _hosts = HostDao.create_query_select(Host, filters=filter_tuples)
-        else:
-            _hosts = HostDao.get_hosts()
+        _hosts = HostDao.create_query_select(Host, filters=filter_tuples, group_by=group_by)
         return _hosts
 
     @staticmethod

quads/server/dao/schedule.py

@@ -1,19 +1,19 @@
 from datetime import datetime
 from typing import List, Type
-from sqlalchemy import and_
+from sqlalchemy import and_, Boolean, func
+from sqlalchemy.orm import RelationshipProperty
+from sqlalchemy.orm.relationships import Relationship
 
 from quads.server.dao.assignment import AssignmentDao
-from quads.server.dao.baseDao import BaseDao, EntryNotFound, InvalidArgument, SQLError
+from quads.server.dao.baseDao import BaseDao, EntryNotFound, InvalidArgument, SQLError, OPERATORS, MAP_HOST_META
 from quads.server.dao.cloud import CloudDao
 from quads.server.dao.host import HostDao
 from quads.server.models import db, Host, Schedule, Cloud, Assignment
 
 
 class ScheduleDao(BaseDao):
     @classmethod
-    def create_schedule(
-        cls, start: datetime, end: datetime, assignment: Assignment, host: Host
-    ) -> Schedule:
+    def create_schedule(cls, start: datetime, end: datetime, assignment: Assignment, host: Host) -> Schedule:
         _schedule_obj = Schedule(start=start, end=end, assignment=assignment, host=host)
         db.session.add(_schedule_obj)
         cls.safe_commit()
@@ -90,6 +90,73 @@ def get_future_schedules(host: Host = None, cloud: Cloud = None) -> List[Schedul
         future_schedules = query.all()
         return future_schedules
 
+    @staticmethod
+    def filter_schedule_dict(data: dict) -> List[Schedule]:
+        filter_tuples = []
+        date_fields = ["start", "end", "build_start", "build_end"]
+        operator = "=="
+        group_by = None
+        for k, value in data.items():
+            fields = k.split(".")
+            if len(fields) > 2:
+                raise InvalidArgument(f"Too many arguments: {fields}")
+
+            first_field = fields[0]
+            field_name = fields[-1]
+            if "__" in k:
+                for op in OPERATORS.keys():
+                    if op in field_name:
+                        if first_field == field_name:
+                            first_field = field_name[: field_name.index(op)]
+                        field_name = field_name[: field_name.index(op)]
+                        operator = OPERATORS[op]
+                        break
+
+            if value.lower() == "none":
+                value = None
+
+            if fields[0].lower() == "group_by":
+                first_field = value
+                group_by = value
+                k = value
+            field = Schedule.__mapper__.attrs.get(first_field)
+            if not field:
+                raise InvalidArgument(f"{k} is not a valid field.")
+            if (
+                type(field) != RelationshipProperty
+                and type(field) != Relationship
+                and type(field.columns[0].type) == Boolean
+            ):
+                value = value.lower() in ["true", "y", 1, "yes"]
+            else:
+                if first_field.lower() == "host":
+                    host = HostDao.get_host(value)
+                    if not host:
+                        raise EntryNotFound(f"Host not found: {value}")
+                    value = host
+                    field_name = first_field
+
+            if first_field in date_fields:
+                try:
+                    if value:
+                        value = datetime.strptime(value, "%Y-%m-%dT%H:%M")
+                except ValueError:
+                    raise InvalidArgument(f"Invalid date format for {first_field}: {value}")
+
+            if fields[0].lower() != "group_by":
+                filter_tuples.append(
+                    (
+                        field_name,
+                        operator,
+                        value,
+                    )
+                )
+        try:
+            _schedules = ScheduleDao.create_query_select(Schedule, filters=filter_tuples, group_by=group_by)
+        except Exception as e:
+            raise InvalidArgument(str(e))
+        return _schedules
+
     @staticmethod
     def filter_schedules(
         start: datetime = None,
@@ -116,9 +183,7 @@ def filter_schedules(
                     end_date = datetime.strptime(end, "%Y-%m-%dT%H:%M")
                     end = end_date
                 except ValueError:
-                    raise InvalidArgument(
-                        "end argument must be a datetime object or a correct datetime format string"
-                    )
+                    raise InvalidArgument("end argument must be a datetime object or a correct datetime format string")
             elif not isinstance(end, datetime):
                 raise InvalidArgument("end argument must be a datetime object")
             query = query.filter(Schedule.end <= end)
@@ -135,9 +200,7 @@ def filter_schedules(
         return filter_schedules
 
     @staticmethod
-    def get_current_schedule(
-        date: datetime = None, host: Host = None, cloud: Cloud = None
-    ) -> List[Type[Schedule]]:
+    def get_current_schedule(date: datetime = None, host: Host = None, cloud: Cloud = None) -> List[Type[Schedule]]:
         query = db.session.query(Schedule)
         if cloud:
             query = query.join(Assignment).filter(Assignment.cloud == cloud)