Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Limit requesting prior tasks for ranking in scheduler #3836

Merged
merged 3 commits into from
Nov 21, 2024
Merged

Limit requesting prior tasks for ranking in scheduler #3836

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e064fe9
Limit requesting prior tasks for ranking
jpbruinsslot Nov 13, 2024
2671d49
Merge branch 'main' into fix/mula/limit-prior-tasks
underdarknl Nov 20, 2024
e916dab
Merge branch 'main' into fix/mula/limit-prior-tasks
underdarknl Nov 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions mula/scheduler/rankers/boefje.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ def rank(self, obj: Any) -> int:
grace_period = timedelta(seconds=self.ctx.config.pq_grace_period)

# New tasks that have not yet run before
if obj.prior_tasks is None or not obj.prior_tasks:
if obj.latest_task is None or not obj.latest_task:
jpbruinsslot marked this conversation as resolved.
Show resolved Hide resolved
jpbruinsslot marked this conversation as resolved.
Show resolved Hide resolved
return 2

# Make sure that we don't have tasks that are still in the grace period
time_since_grace_period = ((datetime.now(timezone.utc) - obj.prior_tasks[0].modified_at) - grace_period).seconds
time_since_grace_period = ((datetime.now(timezone.utc) - obj.latest_task.modified_at) - grace_period).seconds
if time_since_grace_period < 0:
return -1

Expand Down
4 changes: 2 additions & 2 deletions mula/scheduler/schedulers/boefje.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -562,8 +562,8 @@ def push_boefje_task(self, boefje_task: BoefjeTask, caller: str = "") -> None:
)
return

prior_tasks = self.ctx.datastores.task_store.get_tasks_by_hash(boefje_task.hash)
score = self.priority_ranker.rank(SimpleNamespace(prior_tasks=prior_tasks, task=boefje_task))
latest_task = self.ctx.datastores.task_store.get_latest_task_by_hash(boefje_task.hash)
score = self.priority_ranker.rank(SimpleNamespace(latest_task=latest_task, task=boefje_task))

task = Task(
id=boefje_task.id,
Expand Down
6 changes: 3 additions & 3 deletions mula/tests/integration/test_boefje_scheduler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -577,10 +577,10 @@ def test_push_task_no_ooi(self):
@mock.patch("scheduler.schedulers.BoefjeScheduler.has_boefje_permission_to_run")
@mock.patch("scheduler.schedulers.BoefjeScheduler.has_boefje_task_grace_period_passed")
@mock.patch("scheduler.schedulers.BoefjeScheduler.is_item_on_queue_by_hash")
@mock.patch("scheduler.context.AppContext.datastores.task_store.get_tasks_by_hash")
@mock.patch("scheduler.context.AppContext.datastores.task_store.get_latest_task_by_hash")
def test_push_task_queue_full(
self,
mock_get_tasks_by_hash,
mock_get_latest_task_by_hash,
mock_is_item_on_queue_by_hash,
mock_has_boefje_task_grace_period_passed,
mock_has_boefje_permission_to_run,
Expand All @@ -606,7 +606,7 @@ def test_push_task_queue_full(
mock_has_boefje_task_started_running.return_value = False
mock_has_boefje_task_grace_period_passed.return_value = True
mock_is_item_on_queue_by_hash.return_value = False
mock_get_tasks_by_hash.return_value = None
mock_get_latest_task_by_hash.return_value = None
self.mock_get_plugin.return_value = PluginFactory(scan_level=0, consumes=[ooi.object_type])

# Act
Expand Down