diff --git a/README.md b/README.md index 47b621e..91023df 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ names of people involved in contest preparation such that filenames and special strings in judge submissions can be used to associate submissions with those people. -## Counting indepedent submissions +## Counting independent submissions ### How crifx decides the author of a submission In order of precedence, crifx determines the author of a file by: @@ -40,3 +40,78 @@ a git username. More details to come, including an example. If you encounter a scenario where crifx is incorrectly attributing a submission to the wrong person and it cannot easily be fixed by adding a user and alias to the `crifx.toml` file. Crifx does not yet support disambiguating git users with the same git name. + +## Configuration + +### Top-level +The top-level configuration of `crifx` is defined in a TOML text file named +`crifx.toml` placed in the directory containing the directories for each problem. + + +- `github_repo_url`. Optional. String. Default: `null`. The GitHub repository url for +the problem set. This should be the root url for the repository and not the url to +some directory in the repository tree. + +#### `[review_requirements]` + +- `independent_ac`. Optional. Integer. Default: `3`. +- `language_groups`. Optional. Integer. Default: `2`. +- `submissions_wa`. Optional. Integer. Default: `1`. +- `submissions_tle`. Optional. Integer. Default: `1`. +- `statement_reviewers`. Optional. Integer. Default: `3`. +- `validator_reviewers`. Optional. Integer. Default: `2`. +- `data_reviewers`. Optional. Integer. Default: `2`. + +#### `[[judge]]` +The `judge` array of tables is used to associate judge names and aliases. The +judge name can also optionally be associated with a git name. +- `primary_name`. Required. String. A name to identify the judge with. This name +will be used in the `crifx` report when listing who wrote each submission and who +is ineligible to provide further contributions for a problem. +- `git_name`. Optional. String. This is the git name for the judge if they +have one. Note that this is _not_ the GitHub username. From the command line, you +can see your git name with the command `git config user.name`. By +specifying a `git_name`, submissions from a judge that are uploaded or edited by +someone else can be associated with submissions that the judge themself uploads. +For the submission uploaded by someone else, the submission will need to be +identified by file name or a `crifx!(author=name)` string in the submission file. +- `aliases`. Optional. Array of Strings. List of aliases that can be used to identify +the judge. Submission file names are split into parts by underscores. If any part +matches an alias of a judge, then the submission is assumed to belong to the judge +with the alias and associated primary name/git name. Aliases can also be used in +`crifx!(author=name)` strings inside submission files to identify a judge. + +#### `[[language_group]]` +- `name`. Required. String. A name to use for the language group. E.g., `"C/C++"` +- `languages`. Required. Array of Strings. A list of languages to include in the +group. The languages must be known to `crifx`. +- `required_ac_count`. Optional. Integer. Default: `0`. The number of Accepted +submissions that are required for this language group. E.g., a value of `2` means +that each problem must have at least `2` accepted submissions from languages in +this language group. + +### Per problem configuration +For each problem, a TOML text file placed in the root directory for the problem +(i.e., at the same level as the `problem.yaml` file and the `submissions` directory), +called `crifx-problem-status.toml`. This file can be used to track who has reviewed +different parts of a problem. It can also be used to define some per-problem configuration. + +- `github_issue_id`. Optional. Integer. Default: `null`. The GitHub Issue id number +for a GitHub Issue that may have been created to discuss the problem. This is used +to generate links to the GitHub Issue url from within the `crifx` report. + +#### `[review_status]` +- `statement_reviewed_by`. Optional. Array of Strings. A list of judge aliases for +judges that have reviewed the problem statement and are happy with the state of the +problem statement for use in a contest. The judge's name should only be added to this +list if they think that the statement is clear and unambiguous. +- `validators_reviewed_by`. Optional. Array of Strings. A list of judge aliases for +judges that have reviewed the input (and output) validators. The judge's name should +only be added to this list if they have verified that the input validator checks all +guarantees made in the problem statement about the input data. +- `data_reviewed_by`. Optional. Array of Strings. A list of judge aliases for judges +that have reviewed the test data (secret and sample). The judge's name should only +be added to this list if they have inspected the generated test data and after doing +so they cannot think of another test case that should be added to ensure that correct +submissions will be judged as correct and incorrect submissions will be judged as +incorrect. diff --git a/crifx/cli.py b/crifx/cli.py index 27b5c73..904b753 100644 --- a/crifx/cli.py +++ b/crifx/cli.py @@ -77,8 +77,12 @@ def main(): crifx_dir_path = make_crifx_dir(problemset_root_path) config = parse_config(problemset_root_path) git_manager = GitManager(problemset_root_path) + track_review_status = config.track_review_status() problemset_parser = ProblemSetParser( - problemset_root_path, git_manager, config.alias_groups + problemset_root_path, + git_manager, + config.alias_groups, + track_review_status, ) problemset = problemset_parser.parse_problemset() writer = ReportWriter(problemset, config, git_manager) diff --git a/crifx/config_parser.py b/crifx/config_parser.py index f255a91..9f6d85b 100644 --- a/crifx/config_parser.py +++ b/crifx/config_parser.py @@ -164,6 +164,15 @@ def __init__(self, toml_dict): alias_group_2.identifier, ) + @property + def track_review_status(self) -> bool: + """Return True iff manual reviews are required to be tracked in files per problem.""" + return ( + self.review_requirements.statement_reviewers > 0 + or self.review_requirements.data_reviewers > 0 + or self.review_requirements.validator_reviewers > 0 + ) + def parse_config(problemset_root_path: str) -> Config: """Parse a configuration file into a Config object.""" diff --git a/crifx/problemset_parser.py b/crifx/problemset_parser.py index 389e7ee..be4c10f 100644 --- a/crifx/problemset_parser.py +++ b/crifx/problemset_parser.py @@ -4,7 +4,7 @@ import os import re import tomllib -from typing import Any +from typing import Any, Optional from crifx.config_parser import AliasGroup from crifx.contest_objects import ( @@ -38,6 +38,7 @@ def __init__( problemset_root_path: str, git_manager: GitManager, alias_groups: list[AliasGroup], + track_review_status: bool, ): if not is_contest_problems_root(problemset_root_path): raise ValueError( @@ -45,6 +46,7 @@ def __init__( ) self.problemset_root_path = problemset_root_path self.git_manager = git_manager + self.track_review_status = track_review_status self.judges_by_name: dict[str, Judge] = {} self._set_judges_by_name(alias_groups) @@ -264,8 +266,13 @@ def _parse_submissions_dir( submissions.append(submission) return submissions - def _parse_review_status(self, problem_root_dir: str) -> ReviewStatus: + def _parse_review_status( + self, + problem_root_dir: str, + ) -> ReviewStatus: """Parse the problem review status.""" + if not self.track_review_status: + return DEFAULT_REVIEW_STATUS review_status_path = os.path.join( problem_root_dir, PROBLEM_REVIEW_STATUS_FILENAME, @@ -285,7 +292,6 @@ def _parse_review_status(self, problem_root_dir: str) -> ReviewStatus: github_issue_id = toml_dict.get("github_issue_id") if not isinstance(github_issue_id, int): github_issue_id = None - run_problemtools = bool(toml_dict.get("run_problemtools", False)) review_status_dict = toml_dict.get("review_status", {}) statement_reviewed_by = _read_reviewers( review_status_dict, "statement_reviewed_by", review_status_path @@ -298,7 +304,6 @@ def _parse_review_status(self, problem_root_dir: str) -> ReviewStatus: ) return ReviewStatus( github_issue_id, - run_problemtools, statement_reviewed_by, validators_reviewed_by, data_reviewed_by, diff --git a/crifx/report_objects/review_status.py b/crifx/report_objects/review_status.py index e5dce2f..8f4a663 100644 --- a/crifx/report_objects/review_status.py +++ b/crifx/report_objects/review_status.py @@ -8,23 +8,18 @@ class ReviewStatus: """Per-problem status tracking data.""" github_issue_id: int | None - run_problemtools: bool statement_reviewed_by: list[str] validators_reviewed_by: list[str] data_reviewed_by: list[str] -DEFAULT_REVIEW_STATUS = ReviewStatus(None, False, [], [], []) +DEFAULT_REVIEW_STATUS = ReviewStatus(None, [], [], []) DEFAULT_REVIEW_STATUS_TOML = """ # The GitHub Issue id for the problem, if there is one. # github_issue_id = -# Set `run_problemtools` to "true" when the problem is sufficiently developed -# to run the problemtools verifyproblem routine. -run_problemtools = false - [review_status] # Add your name in double quotes on a new line in this list if you have read # the problem statement and believe that no further changes are necessary to diff --git a/crifx/report_writer.py b/crifx/report_writer.py index 41f624c..1ccff2c 100644 --- a/crifx/report_writer.py +++ b/crifx/report_writer.py @@ -407,15 +407,6 @@ def _write_problem_details(self, problem: Problem): f"{submission.filename} by {submission.author}. " f"{submission.lines_of_code} lines of code." ) - with self.doc.create(Subsection("Problemtools verifyproblem output")): - if problem.review_status.run_problemtools: - self.doc.append("null") - else: - self.doc.append( - "Including problemtools verifyproblem output is disabled " - "for this problem. It can be enabled in the " - "crifx-problem-status.toml file for this problem." - ) with self.doc.create(Subsection("Test Cases")): self.doc.append( "Test case descriptions are rendered below if .desc files exist." diff --git a/examples/example_problemset/addtwonumbers/crifx-problem-status.toml b/examples/example_problemset/addtwonumbers/crifx-problem-status.toml index 1769e83..8a398d1 100644 --- a/examples/example_problemset/addtwonumbers/crifx-problem-status.toml +++ b/examples/example_problemset/addtwonumbers/crifx-problem-status.toml @@ -1,7 +1,4 @@ -# This problem is not yet ready for problemtools verifyproblem runs. -run_problemtools = false - # github_issue_id is not required. # github_issue_id = diff --git a/examples/example_problemset/helloworld/crifx-problem-status.toml b/examples/example_problemset/helloworld/crifx-problem-status.toml index 4155aa1..1e50e34 100644 --- a/examples/example_problemset/helloworld/crifx-problem-status.toml +++ b/examples/example_problemset/helloworld/crifx-problem-status.toml @@ -1,9 +1,6 @@ # GitHub Issue id number associated with this problem. github_issue_id = 1 -# Whether or not to run problemtools verifyproblem. -run_problemtools = true - [review_status] # List of names of judges who have reviewed the problem statement. statement_reviewed_by = [