diff --git a/backend/howtheyvote/alembic/versions/1f516b18c4f6_rename_result_column_to_status.py b/backend/howtheyvote/alembic/versions/1f516b18c4f6_rename_result_column_to_status.py new file mode 100644 index 00000000..b4cbe85a --- /dev/null +++ b/backend/howtheyvote/alembic/versions/1f516b18c4f6_rename_result_column_to_status.py @@ -0,0 +1,23 @@ +"""Rename result column to status + +Revision ID: 1f516b18c4f6 +Revises: 9b35d19b64c4 +Create Date: 2024-12-08 11:25:26.051408 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "1f516b18c4f6" +down_revision = "9b35d19b64c4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.alter_column("pipeline_runs", column_name="result", new_column_name="status") + + +def downgrade() -> None: + op.alter_column("pipeline_runs", column_name="status", new_column_name="result") diff --git a/backend/howtheyvote/models/__init__.py b/backend/howtheyvote/models/__init__.py index 0aa3a2d8..75ece9a2 100644 --- a/backend/howtheyvote/models/__init__.py +++ b/backend/howtheyvote/models/__init__.py @@ -1,4 +1,4 @@ -from .common import Base, BaseWithId, DataIssue, Fragment, PipelineRun, PipelineRunResult +from .common import Base, BaseWithId, DataIssue, Fragment, PipelineRun, PipelineStatus from .country import Country, CountryType from .eurovoc import EurovocConcept, EurovocConceptType from .group import Group @@ -24,7 +24,7 @@ "BaseWithId", "Fragment", "PipelineRun", - "PipelineRunResult", + "PipelineStatus", "DataIssue", "Country", "CountryType", diff --git a/backend/howtheyvote/models/common.py b/backend/howtheyvote/models/common.py index ef084442..ecc40c93 100644 --- a/backend/howtheyvote/models/common.py +++ b/backend/howtheyvote/models/common.py @@ -36,10 +36,11 @@ class DataIssue(Enum): VOTE_GROUP_NO_MAIN_VOTE = "VOTE_GROUP_NO_MAIN_VOTE" -class PipelineRunResult(Enum): +class PipelineStatus(Enum): SUCCESS = "SUCCESS" FAILURE = "FAILURE" DATA_UNAVAILABLE = "DATA_UNAVAILABLE" + DATA_UNCHANGED = "DATA_UNCHANGED" class PipelineRun(Base): @@ -49,4 +50,4 @@ class PipelineRun(Base): started_at: Mapped[sa.DateTime] = mapped_column(sa.DateTime) finished_at: Mapped[sa.DateTime] = mapped_column(sa.DateTime) pipeline: Mapped[str] = mapped_column(sa.Unicode) - result: Mapped[PipelineRunResult] = mapped_column(sa.Enum(PipelineRunResult)) + status: Mapped[PipelineStatus] = mapped_column(sa.Enum(PipelineStatus)) diff --git a/backend/howtheyvote/worker/__init__.py b/backend/howtheyvote/worker/__init__.py index 36ac5f28..e88e3041 100644 --- a/backend/howtheyvote/worker/__init__.py +++ b/backend/howtheyvote/worker/__init__.py @@ -9,7 +9,7 @@ from ..db import Session from ..export import generate_export from ..files import file_path -from ..models import PipelineRun, PipelineRunResult, PlenarySession +from ..models import PipelineRun, PlenarySession from ..pipelines import MembersPipeline, PressPipeline, RCVListPipeline, SessionsPipeline from ..query import session_is_current_at from .worker import SkipPipelineError, Weekday, Worker, pipeline_ran_successfully diff --git a/backend/howtheyvote/worker/worker.py b/backend/howtheyvote/worker/worker.py index 91667c61..e093dba1 100644 --- a/backend/howtheyvote/worker/worker.py +++ b/backend/howtheyvote/worker/worker.py @@ -14,7 +14,7 @@ from .. import config from ..db import Session -from ..models import PipelineRun, PipelineRunResult +from ..models import PipelineRun, PipelineStatus from ..pipelines import DataUnavailableError log = get_logger(__name__) @@ -26,13 +26,13 @@ PIPELINE_RUN_DURATION = Histogram( "htv_worker_pipeline_run_duration_seconds", "Duration of pipeline runs executed by the worker", - ["pipeline", "result"], + ["pipeline", "status"], ) PIPELINE_RUNS = Counter( "htv_worker_pipeline_runs_total", "Total number of pipeline runs executed by the worker", - ["pipeline", "result"], + ["pipeline", "status"], ) PIPELINE_NEXT_RUN = Gauge( @@ -70,7 +70,7 @@ def pipeline_ran_successfully( .select_from(PipelineRun) .where(PipelineRun.pipeline == pipeline.__name__) .where(func.date(PipelineRun.started_at) == func.date(date)) - .where(PipelineRun.result == PipelineRunResult.SUCCESS) + .where(PipelineRun.status == PipelineStatus.SUCCESS) ) result = Session.execute(query).scalar() or 0 @@ -79,7 +79,7 @@ def pipeline_ran_successfully( class Worker: """Running a worker starts a long-running process that executes data pipelines in regular - intervals and stores the result of the pipeline runs in the database.""" + intervals and stores the status of the pipeline runs in the database.""" def __init__(self) -> None: self._scheduler = Scheduler() @@ -165,19 +165,19 @@ def wrapped_handler() -> None: try: handler() - result = PipelineRunResult.SUCCESS + status = PipelineStatus.SUCCESS except SkipPipelineError: # Do not log skipped pipeline runs return except DataUnavailableError: - result = PipelineRunResult.DATA_UNAVAILABLE + status = PipelineStatus.DATA_UNAVAILABLE except Exception: - result = PipelineRunResult.FAILURE + status = PipelineStatus.FAILURE duration = time.time() - start_time finished_at = datetime.datetime.now(datetime.UTC) - labels = {"pipeline": name, "result": result.value} + labels = {"pipeline": name, "status": status.value} PIPELINE_RUNS.labels(**labels).inc() PIPELINE_RUN_DURATION.labels(**labels).observe(duration) @@ -185,7 +185,7 @@ def wrapped_handler() -> None: pipeline=name, started_at=started_at, finished_at=finished_at, - result=result.value, + status=status, ) Session.add(run) @@ -198,7 +198,7 @@ def wrapped_handler() -> None: started_at=started_at.isoformat(), finished_at=finished_at.isoformat(), duration=duration, - result=result.value, + status=status.value, ) Session.remove() diff --git a/backend/tests/worker/test_worker.py b/backend/tests/worker/test_worker.py index 61b54319..cd994760 100644 --- a/backend/tests/worker/test_worker.py +++ b/backend/tests/worker/test_worker.py @@ -3,7 +3,7 @@ import time_machine from sqlalchemy import select -from howtheyvote.models import PipelineRun, PipelineRunResult +from howtheyvote.models import PipelineRun, PipelineStatus from howtheyvote.pipelines import DataUnavailableError, PipelineError from howtheyvote.worker.worker import Weekday, Worker, pipeline_ran_successfully @@ -133,7 +133,7 @@ def test_worker_schedule_pipeline_log_runs(db_session): run = runs[0] assert run.pipeline == "test" - assert run.result == PipelineRunResult.SUCCESS + assert run.status == PipelineStatus.SUCCESS assert run.started_at.date() == datetime.date(2024, 1, 1) assert run.finished_at.date() == datetime.date(2024, 1, 1) @@ -166,10 +166,10 @@ def pipeline_error(): assert len(runs) == 2 assert runs[0].pipeline == "data_unavailable_error" - assert runs[0].result == PipelineRunResult.DATA_UNAVAILABLE + assert runs[0].status == PipelineStatus.DATA_UNAVAILABLE assert runs[1].pipeline == "pipeline_error" - assert runs[1].result == PipelineRunResult.FAILURE + assert runs[1].status == PipelineStatus.FAILURE def test_pipeline_ran_successfully(db_session): @@ -183,7 +183,7 @@ class TestPipeline: started_at=now, finished_at=now, pipeline=TestPipeline.__name__, - result=PipelineRunResult.FAILURE, + status=PipelineStatus.FAILURE, ) db_session.add(run) db_session.commit() @@ -194,7 +194,7 @@ class TestPipeline: started_at=now, finished_at=now, pipeline=TestPipeline.__name__, - result=PipelineRunResult.SUCCESS, + status=PipelineStatus.SUCCESS, ) db_session.add(run) db_session.commit() @@ -206,7 +206,7 @@ class TestPipeline: started_at=now, finished_at=now, pipeline=TestPipeline.__name__, - result=PipelineRunResult.SUCCESS, + status=PipelineStatus.SUCCESS, ) db_session.add(run) db_session.commit()