Skip to content

Commit

Permalink
Merge branch 'main' into fix/delete-many-fail-silently
Browse files Browse the repository at this point in the history
  • Loading branch information
@dekkers
dekkers authored Nov 7, 2024
2 parents 800dce1 + d5b3dfa commit 54a0072
Show file tree
Hide file tree
Showing 10 changed files with 181 additions and 54 deletions.
2 changes: 1 addition & 1 deletion docs/source/manual/reports.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ The table below gives an overview of the elements that can be found in each repo

Report flow
===========
On the Reports page you can generate new reports and get an overview of all generated reports. With the button 'Generate report' you get into the Report flow wizard, which can be used to choose your report, objects and plugins that are required for the report. Please note that enabling plugins during the report flow wizard will result in inaccurate data, as the plugins will take some time before they have gathered and analyzed all data. Check the Tasks page to verify that all tasks have completed.
On the Reports page you can generate new reports and get an overview of all generated reports. With the button 'Generate report' you get into the Report flow wizard, which can be used to choose your report, objects and plugins that are required for the report. There are two ways to select objects. You can manually select objects, which will be static. Or you can select a live set of objects by continuing with the selected filters. The selected objects will then always be based on the selected filters at the time of generating the report. and Please note that enabling plugins during the report flow wizard will result in inaccurate data, as the plugins will take some time before they have gathered and analyzed all data. Check the Tasks page to verify that all tasks have completed.


Plugins
Expand Down
51 changes: 33 additions & 18 deletions rocky/reports/runner/report_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
from tools.models import Organization

from octopoes.connector.octopoes import OctopoesAPIConnector
from octopoes.models import Reference
from octopoes.models import Reference, ScanLevel, ScanProfileType
from octopoes.models.types import type_by_name
from reports.report_types.aggregate_organisation_report.report import AggregateOrganisationReport, aggregate_reports
from reports.report_types.definitions import report_plugins_union
from reports.report_types.helpers import get_report_by_id
Expand All @@ -26,16 +27,34 @@ def run(self, report_task: ReportTask) -> None:
connector = OctopoesAPIConnector(
settings.OCTOPOES_API, report_task.organisation_id, timeout=settings.ROCKY_OUTGOING_REQUEST_TIMEOUT
)
recipe = connector.get(Reference.from_str(f"ReportRecipe|{report_task.report_recipe_id}"), valid_time)
recipe_ref = Reference.from_str(f"ReportRecipe|{report_task.report_recipe_id}")
recipe = connector.get(recipe_ref, valid_time)

report_types = [get_report_by_id(report_type_id) for report_type_id in recipe.report_types]
oois_count = len(recipe.input_recipe["input_oois"])
oois = []
now = datetime.now(timezone.utc)

for ooi_id in recipe.input_recipe["input_oois"]:
ooi = connector.get(Reference.from_str(ooi_id), valid_time)
oois.append(ooi)
if input_oois := recipe.input_recipe.get("input_oois"):
for ooi_id in input_oois:
ooi = connector.get(Reference.from_str(ooi_id), valid_time)
oois.append(ooi)
elif query := recipe.input_recipe.get("query"):
types = {type_by_name(t) for t in query["ooi_types"]}
scan_level = {ScanLevel(cl) for cl in query["scan_level"]}
scan_type = {ScanProfileType(t) for t in query["scan_type"]}

oois = connector.list_objects(
types=types,
valid_time=datetime.now(tz=timezone.utc),
scan_level=scan_level,
scan_profile_type=scan_type,
search_string=query["search_string"],
order_by=query["order_by"],
asc_desc=query["asc_desc"],
).items

oois_count = len(oois)
ooi_pks = [ooi.primary_key for ooi in oois]

self.bytes_client.organization = report_task.organisation_id

Expand All @@ -48,9 +67,7 @@ def run(self, report_task: ReportTask) -> None:
report_type_ids = [report.id for report in report_types]

if "${ooi}" in parent_report_name and oois_count == 1:
ooi = recipe.input_recipe["input_oois"][0]
ooi_human_readable = Reference.from_str(ooi).human_readable
parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi_human_readable)
parent_report_name = Template(parent_report_name).safe_substitute(ooi=oois[0].human_readable)

aggregate_report, post_processed_data, report_data, report_errors = aggregate_reports(
connector, oois, report_type_ids, valid_time, report_task.organisation_id
Expand All @@ -60,10 +77,10 @@ def run(self, report_task: ReportTask) -> None:
connector,
Organization.objects.get(code=report_task.organisation_id),
valid_time,
recipe.input_recipe["input_oois"],
ooi_pks,
{
"input_data": {
"input_oois": recipe.input_recipe["input_oois"],
"input_oois": ooi_pks,
"report_types": recipe.report_types,
"plugins": report_plugins_union(report_types),
}
Expand All @@ -72,12 +89,11 @@ def run(self, report_task: ReportTask) -> None:
report_data,
post_processed_data,
aggregate_report,
recipe_ref,
)
else:
subreport_names = []
error_reports, report_data = collect_reports(
valid_time, connector, recipe.input_recipe["input_oois"], report_types
)
error_reports, report_data = collect_reports(valid_time, connector, ooi_pks, report_types)

for report_type_id, data in report_data.items():
report_type = get_report_by_id(report_type_id)
Expand All @@ -96,9 +112,7 @@ def run(self, report_task: ReportTask) -> None:
)

if "${ooi}" in parent_report_name and oois_count == 1:
ooi = recipe.input_recipe["input_oois"][0]
ooi_human_readable = Reference.from_str(ooi).human_readable
parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi_human_readable)
parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi[0].human_readable)

save_report_data(
self.bytes_client,
Expand All @@ -107,14 +121,15 @@ def run(self, report_task: ReportTask) -> None:
Organization.objects.get(code=report_task.organisation_id),
{
"input_data": {
"input_oois": recipe.input_recipe["input_oois"],
"input_oois": ooi_pks,
"report_types": recipe.report_types,
"plugins": report_plugins_union(report_types),
}
},
report_data,
subreport_names,
parent_report_name,
recipe_ref,
)

self.bytes_client.organization = None
1 change: 1 addition & 0 deletions rocky/reports/templates/forms/report_form_fields.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,4 @@
name="choose_recurrence"
value="{{ request.POST.choose_recurrence }}">
{% endif %}
<input type="hidden" name="object_selection" value="{{ object_selection }}">
45 changes: 40 additions & 5 deletions rocky/reports/templates/partials/report_ooi_list.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,40 @@ <h2>
Select objects ({{ total_oois }})
{% endblocktranslate %}
</h2>
<p>{% translate "Select which objects you want to include in your report." %}</p>
<p>
{% blocktranslate %}
Select which objects you want to include in your report. You can either continue
with a live set or you can select the objects manually from the table below.
{% endblocktranslate %}
</p>
<p>
{% blocktranslate %}
A live set is a set of objects based on the applied filters.
Any object that matches this applied filter (now or in the future) will be used as
input for the scheduled report. If your live set filter (e.g. 'hostnames' with
'L2 clearance' that are 'declared') shows 2 hostnames that match the filter today,
the scheduled report will run for those 2 hostnames. If you add 3 more hostnames
tomorrow (with the same filter criteria), your next scheduled report will contain
5 hostnames. Your live set will update as you go.
{% endblocktranslate %}
</p>
{% include "partials/ooi_list_filters.html" %}

<form novalidate
method="post"
action="{{ next }}"
class="inline layout-wide checkboxes_required">
{% csrf_token %}
{% include "forms/report_form_fields.html" %}

<input type="hidden" name="ooi" value="all">
<button type="submit"
class="button select_all_objects_element"
name="object_selection"
value="query">
{% translate "Continue with live set" %}<span class="icon ti-chevron-right"></span>
</button>
</form>
{% if not ooi_list %}
<p>{% translate "No objects found." %}</p>
<div class="button-container">
Expand Down Expand Up @@ -51,7 +82,8 @@ <h2>
<form novalidate
method="post"
action="{{ next }}"
class="inline layout-wide checkboxes_required">
class="inline layout-wide checkboxes_required"
id="object_table_form">
{% csrf_token %}
{% if all_oois_selected %}
{% include "forms/report_form_fields.html" %}
Expand Down Expand Up @@ -95,10 +127,13 @@ <h2>
{% endfor %}
</tbody>
</table>
<button type="submit"
class="button ghost"
name="object_selection"
value="selection">
{% translate "Continue with selection" %}<span class="icon ti-chevron-right"></span>
</button>
</div>
<button type="submit" class="button">
{% translate "Continue with selection" %}<span class="icon ti-chevron-right"></span>
</button>
</form>
{% include "partials/list_paginator.html" %}

Expand Down
6 changes: 3 additions & 3 deletions rocky/reports/templates/report_overview/report_history_table.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
<tr>
<th scope="col">{% translate "Name" %}</th>
<th scope="col">{% translate "Report type" %}</th>
<th scope="col">{% translate "Input Objects" %}</th>
<th scope="col">{% translate "Reference date" %}</th>
<th scope="col">{% translate "Creation date" %}</th>
<th scope="col" class="nowrap">{% translate "Input objects" %}</th>
<th scope="col" class="nowrap">{% translate "Reference date" %}</th>
<th scope="col" class="nowrap">{% translate "Creation date" %}</th>
{% if report.children_reports %}<th scope="col"></th>{% endif %}
</tr>
</thead>
Expand Down
16 changes: 11 additions & 5 deletions rocky/reports/templates/report_overview/scheduled_reports_table.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,11 @@
<caption class="visually-hidden">{% translate "Scheduled Reports:" %}</caption>
<thead>
<tr>
<th scope="col">{% translate "Name" %}</th>
<th scope="col" class="nowrap">{% translate "Name" %}</th>
<th scope="col">{% translate "Report type" %}</th>
<th scope="col">{% translate "Input Object" %}</th>
<th scope="col">{% translate "Reference date" %}</th>
<th scope="col">{% translate "Creation date" %}</th>
<th scope="col">{% translate "Input object" %}</th>
<th scope="col" class="nowrap">{% translate "Reference date" %}</th>
<th scope="col" class="nowrap">{% translate "Creation date" %}</th>
</tr>
</thead>
<tbody>
Expand All @@ -77,7 +77,13 @@
<span class="label tags-color-{{ report.report_type|get_report_type_label_style }}">{{ report.report_type|get_report_type_name }}</span>
</td>
<td>
{% for ooi in report.input_oois %}<span>{{ ooi|human_readable }}</span>{% endfor %}
{% if report.input_oois|length == 1 %}
{% with ooi=report.input_oois.0 %}
<a href="{% ooi_url 'ooi_detail' ooi organization.code query=ooi.mandatory_fields %}">{{ ooi|human_readable }}</a>
{% endwith %}
{% else %}
{{ report.input_oois|length }}
{% endif %}
</td>
<td class="nowrap">{{ report.observed_at|date }}</td>
<td class="nowrap">{{ report.date_generated }}</td>
Expand Down
54 changes: 45 additions & 9 deletions rocky/reports/views/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ def format_plugin_data(report_type_plugins: dict[str, list[Plugin]]):
]


class BaseReportView(OOIFilterView):
class BaseReportView(OOIFilterView, ReportBreadcrumbs):
"""
This view is the base for the report creation wizard.
All the necessary functions and variables needed.
Expand Down Expand Up @@ -268,13 +268,25 @@ def is_scheduled_report(self) -> bool:
return recurrence_choice == "repeat"

def create_report_recipe(
self, report_name_format: str, subreport_name_format: str, parent_report_type: str | None, schedule: str
self,
report_name_format: str,
subreport_name_format: str,
parent_report_type: str | None,
schedule: str,
query: dict[str, Any] | None,
) -> ReportRecipe:
input_recipe: dict[str, Any] = {}

if query:
input_recipe = {"query": query}
else:
input_recipe = {"input_oois": self.get_ooi_pks()}

report_recipe = ReportRecipe(
recipe_id=uuid4(),
report_name_format=report_name_format,
subreport_name_format=subreport_name_format,
input_recipe={"input_oois": self.get_ooi_pks()},
input_recipe=input_recipe,
parent_report_type=parent_report_type,
report_types=self.get_report_type_ids(),
cron_expression=schedule,
Expand Down Expand Up @@ -302,6 +314,7 @@ def get_context_data(self, **kwargs):
context["all_oois_selected"] = self.all_oois_selected()
context["selected_oois"] = self.selected_oois
context["selected_report_types"] = self.selected_report_types
context["object_selection"] = self.request.POST.get("object_selection", "")

return context

Expand All @@ -311,6 +324,13 @@ class OOISelectionView(BaseReportView, BaseOOIListView):
Shows a list of OOIs to select from and handles OOIs selection requests.
"""

def setup(self, request, *args, **kwargs):
super().setup(request, *args, **kwargs)
object_selection = request.GET.get("object_selection", "")

if object_selection == "query":
return PostRedirect(self.get_next())

def post(self, request, *args, **kwargs):
if not (self.get_ooi_selection() or self.all_oois_selected()):
messages.error(request, self.NONE_OOI_SELECTION_MESSAGE)
Expand All @@ -322,7 +342,7 @@ def get_context_data(self, **kwargs):
return context


class ReportTypeSelectionView(BaseReportView, ReportBreadcrumbs):
class ReportTypeSelectionView(BaseReportView, TemplateView):
"""
Shows report types and handles selections and requests.
"""
Expand All @@ -332,7 +352,8 @@ def setup(self, request, *args, **kwargs):
self.available_report_types, self.counted_report_types = self.get_available_report_types()

def post(self, request, *args, **kwargs):
if not (self.get_ooi_selection() or self.all_oois_selected()):
object_selection = request.GET.get("object_selection", "")
if not (self.get_ooi_selection() or self.all_oois_selected()) and object_selection != "query":
return PostRedirect(self.get_previous())
return self.get(request, *args, **kwargs)

Expand All @@ -349,8 +370,11 @@ def get_context_data(self, **kwargs):

return context

def all_oois_selected(self) -> bool:
return "all" in self.request.POST.getlist("ooi", [])


class ReportPluginView(BaseReportView, ReportBreadcrumbs, TemplateView):
class ReportPluginView(BaseReportView, TemplateView):
"""
This view shows the required and optional plugins together with the summary per report type.
"""
Expand Down Expand Up @@ -446,7 +470,7 @@ def get_context_data(self, **kwargs):
return context


class ReportFinalSettingsView(BaseReportView, ReportBreadcrumbs, SchedulerView, TemplateView):
class ReportFinalSettingsView(BaseReportView, SchedulerView, TemplateView):
report_type: type[BaseReport] | None = None
task_type = "report"
is_a_scheduled_report = False
Expand Down Expand Up @@ -515,7 +539,7 @@ def get_context_data(self, **kwargs):
return context


class SaveReportView(BaseReportView, ReportBreadcrumbs, SchedulerView):
class SaveReportView(BaseReportView, SchedulerView):
task_type = "report"

def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
Expand All @@ -537,6 +561,18 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
subreport_name_format = request.POST.get("child_report_name", "")
recurrence = request.POST.get("recurrence", "")
deadline_at = request.POST.get("start_date", datetime.now(timezone.utc).date())
object_selection = request.POST.get("object_selection", "")

query = {}
if object_selection == "query":
query = {
"ooi_types": [t.__name__ for t in self.get_ooi_types()],
"scan_level": self.get_ooi_scan_levels(),
"scan_type": self.get_ooi_profile_types(),
"search_string": self.search_string,
"order_by": self.order_by,
"asc_desc": self.sorting_order,
}

parent_report_type = None
if self.report_type is not None:
Expand All @@ -547,7 +583,7 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
schedule = self.convert_recurrence_to_cron_expressions(recurrence)

report_recipe = self.create_report_recipe(
report_name_format, subreport_name_format, parent_report_type, schedule
report_name_format, subreport_name_format, parent_report_type, schedule, query
)

self.create_report_schedule(report_recipe, deadline_at)
Expand Down
Loading

0 comments on commit 54a0072

Please sign in to comment.