[WIP] Adding the support of python packages to streamline the commit & PR flow

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,49 @@
+## Summary
+
+Include a summary of major changes in bullet points:
+
+- Feature 1
+- Feature 2
+- Fix 1
+- Fix 2
+
+## Additional dependencies introduced (if any)
+
+- List all new dependencies needed and justify why. While adding dependencies that bring
+  significantly useful functionality is perfectly fine, adding ones that add trivial
+  functionality, e.g., to use one single easily implementable function, is frowned upon.
+  Provide a justification why that dependency is needed. Especially frowned upon are
+  circular dependencies.
+
+## TODO (if any)
+
+If this is a work-in-progress, write something about what else needs to be done.
+
+- Feature 1 supports A, but not B.
+
+## Checklist
+
+Work-in-progress pull requests are encouraged, but please put [WIP] in the pull request
+title.
+
+Before a pull request can be merged, the following items must be checked:
+
+- [ ] Code is in the [standard Python style](https://www.python.org/dev/peps/pep-0008/).
+  The easiest way to handle this is to run the following in the **correct sequence** on
+  your local machine. Start with running [black](
+  https://black.readthedocs.io/en/stable/index.html) on your new code. This will
+  automatically reformat your code to PEP8 conventions and removes most issues. Then run
+  [pycodestyle](https://pycodestyle.readthedocs.io/en/latest/), followed by [flake8](
+  http://flake8.pycqa.org/en/latest/).
+- [ ] Docstrings have been added in the[Numpy docstring format](
+  https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_numpy.html).
+  Run [pydocstyle](http://www.pydocstyle.org/en/2.1.1/index.html) on your code.
+- [ ] Type annotations are **highly** encouraged. Run [mypy](http://mypy-lang.org/) to
+  type check your code.
+- [ ] Tests have been added for any new functionality or bug fixes.
+- [ ] All linting and tests pass.
+
+Note that the CI system will run all the above checks. But it will be much more
+efficient if you already fix most errors prior to submitting the PR. It is highly
+recommended that you use the pre-commit hook provided in the repository. Simply
+`cp pre-commit .git/hooks` and a check will be run prior to allowing commits.

.pre-commit-config.yaml

@@ -0,0 +1,64 @@
+default_language_version:
+  python: python3
+exclude: "^src/atomate2/vasp/schemas/calc_types/"
+repos:
+  # - repo: https://github.com/charliermarsh/ruff-pre-commit
+  #   rev: v0.0.282
+  #   hooks:
+  #     - id: ruff
+  #       args: [--fix]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+      - id: fix-encoding-pragma
+        args: [--remove]
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+  - repo: https://github.com/asottile/blacken-docs
+    rev: v1.12.1
+    hooks:
+      - id: blacken-docs
+        additional_dependencies: [black]
+        exclude: README.md
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        entry: pflake8
+        files: ^src/
+        additional_dependencies:
+          - pyproject-flake8==6.0.0
+          - flake8-bugbear==22.12.6
+          - flake8-typing-imports==1.14.0
+          - flake8-docstrings==1.6.0
+          - flake8-rst-docstrings==0.3.0
+          - flake8-rst==0.8.0
+        args: ['--ignore']  # Add the error codes you want to ignore here
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0
+    hooks:
+      - id: python-use-type-annotations
+      - id: rst-backticks
+      - id: rst-directive-colons
+      - id: rst-inline-touching-normal
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.991
+    hooks:
+      - id: mypy
+        files: ^src/
+        additional_dependencies:
+          - tokenize-rt==4.1.0
+          - types-pkg_resources==0.1.2
+          - types-paramiko
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.2
+    hooks:
+      - id: codespell
+        stages: [commit, commit-msg]
+        args: [--ignore-words-list, "titel,statics,ba,nd,te"]
+        types_or: [python, rst, markdown]

alab_management/alarm.py

@@ -3,14 +3,15 @@
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
 
+
 def format_message_to_codeblock(message: str) -> str:
     """
     This function takes a message and formats it as a code block.
     It is used to format tracebacks as code blocks in Slack.
     Args:
         message: The message to format (String). This is usually a traceback.
     Returns:
-        formatted_message: The formatted message. This will be formatted into code block in slack if the message contains traceback, 
+        formatted_message: The formatted message. This will be formatted into code block in slack if the message contains traceback,
           otherwise it will be the original message.
     """
     # Check if "Traceback (most recent call last):" is in the message
@@ -47,10 +48,12 @@ def format_message_to_codeblock(message: str) -> str:
         formatted_message = message
     return formatted_message
 
+
 class Alarm:
     """
     A class to send alerts to the user via email or slack.
     """
+
     def __init__(
         self,
         email_receivers: list = None,
@@ -83,7 +86,9 @@ def __init__(
                 print("Slackbot setup failed, please recheck config file")
         self.platforms = {"email": self.email_alert, "slack": self.slack_alert}
 
-    def setup_email(self, email_receivers: list, email_sender: str, email_password: str):
+    def setup_email(
+        self, email_receivers: list, email_sender: str, email_password: str
+    ):
         """
         Try to setup email notification (called in __init__)
         Args:
@@ -134,7 +139,7 @@ def alert(self, message: str, category: str):
     def send_email(self, message, category):
         """
         Sends an email to the receiver email address with the exception and category.
-        Category is the type of exception that occured.
+        Category is the type of exception that occurred.
         Automatically use "Error" as category if the message contains traceback.
         Args:
             message: The message to print in the email
@@ -155,8 +160,8 @@ def send_email(self, message, category):
 
     def send_slack_notification(self, message, category):
         """
-        Sends a slack messsage to the receiver email address with the exception and category.
-        Category is the type of exception that occured.
+        Sends a slack message to the receiver email address with the exception and category.
+        Category is the type of exception that occurred.
         Automatically use "Error" as category if the message contains traceback.
         Args:
             message: The message to print in the email

alab_management/builders/experimentbuilder.py

@@ -7,7 +7,7 @@
 class ExperimentBuilder:
     """
     It takes a list of samples and a list of tasks, and returns a dictionary
-    that can be used to generate an input file for the `experiment` command
+    that can be used to generate an input file for the `experiment` to run
 
     Args:
       name (str): The name of the experiment.
@@ -17,6 +17,7 @@ def __init__(self, name: str, tags: Optional[List[str]] = None, **metadata):
         """
         Args:
           name (str): The name of the experiment.
+          tags (List[str]): A list of tags to attach to the experiment.
         """
         self.name = name
         self._samples: List[SampleBuilder] = []
@@ -28,10 +29,12 @@ def add_sample(
         self, name: str, tags: Optional[List[str]] = None, **metadata
     ) -> SampleBuilder:
         """
-        This function adds a sample to the experiment
+        This function adds a sample to the batch. Each sample already has multiple tasks binded to it. Each
+        batch is a directed graph of tasks.
 
         Args:
           name (str): The name of the sample. This must be unique within this ExperimentBuilder.
+          tags (List[str]): A list of tags to attach to the sample.
           **metadata: Any additional keyword arguments will be attached to this sample as metadata.
         Returns:
           A SampleBuilder object. This can be used to add tasks to the sample.
@@ -51,6 +54,18 @@ def add_task(
         task_kwargs: Dict[str, Any],
         samples: List[SampleBuilder],
     ) -> None:
+        """
+        This function adds a task to the sample. You should use this function only for special cases which
+        are not handled by the `add_sample` function.
+
+        Args:
+            task_id (str): The object id of the task in mongodb
+            task_name (str): The name of the task.
+            task_kwargs (Dict[str, Any]): Any additional keyword arguments will be attached to this sample as metadata.
+            samples (List[SampleBuilder]): A list of samples to which this task is binded to.
+        Returns:
+            None
+        """
         if task_id in self._tasks:
             return
         self._tasks[task_id] = {
@@ -59,7 +74,16 @@ def add_task(
             "samples": [sample.name for sample in samples],
         }
 
-    def to_dict(self):
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        This function returns a dictionary that can be used to generate an input file for the `experiment`
+        to run.
+        Args:
+            None
+        Returns:
+            A dictionary that can be used to generate an input file for the `experiment` to run.
+
+        """
         samples = []
         tasks = []
         task_ids = {}
@@ -95,6 +119,14 @@ def to_dict(self):
     def generate_input_file(
         self, filename: str, fmt: Literal["json", "yaml"] = "json"
     ) -> None:
+        """
+        This function generates an input file for the `experiment` command.
+        Args:
+            filename (str): The name of the file to be generated.
+            fmt (Literal["json", "yaml"]): The format of the file to be generated.
+        Returns:
+            None
+        """
         with Path(filename).open("w", encoding="utf-8") as f:
             if fmt == "json":
                 import json
@@ -105,7 +137,14 @@ def generate_input_file(
 
                 yaml.dump(self.to_dict(), f, default_flow_style=False, indent=2)
 
-    def plot(self, ax=None):
+    def plot(self, ax=None) -> None:
+        """
+        This function plots the directed graph of tasks.
+        Args:
+            ax (matplotlib.axes.Axes): The axes on which to plot the graph.
+        Returns:
+            None
+        """
         import networkx as nx
         import matplotlib.pyplot as plt
 

alab_management/builders/samplebuilder.py

@@ -13,6 +13,14 @@
 
 
 class SampleBuilder:
+    """
+    This class is used to build a sample. Each sample has a name, tags, and metadata. Each sample also
+    has a list of tasks which are binded to it. Each sample is a node in a directed graph of tasks.
+    Each task has a list of samples which are binded to it. Each sample also has a list of tags and
+    metadata. The tags and metadata are used to filter the samples and tasks. The tags and metadata are
+    also used to group the samples and tasks.
+    """
+
     def __init__(
         self,
         name: str,
@@ -31,6 +39,14 @@ def add_task(
         self,
         task_id: str,
     ) -> None:
+        """
+        This function adds a task to the sample. You should use this function only for special cases which
+        are not handled by the `add_sample` function.
+        Args:
+            task_id (str): The object id of the task in mongodb
+        Returns:
+            None
+        """
         if task_id not in self._tasks:
             self._tasks.append(task_id)
 
@@ -65,7 +81,7 @@ def __repr__(self):
         return f"<Sample: {self.name}>"
 
 
-### Format checking for API inputs
+# ## Format checking for API inputs
 
 
 # class TaskInputFormat(BaseModel):

alab_management/builders/utils.py

@@ -14,7 +14,8 @@ def append_task(
     """Used to add basetask to a SampleBuilder's tasklist during Experiment construction.
 
     Args:
-        samples (Union[SampleBuilder, List[SampleBuilder]]): One or more SampleBuilder's which will have this task appended to their tasklists.
+        samples (Union[SampleBuilder, List[SampleBuilder]]): One or more SampleBuilder's which will
+        have this task appended to their tasklists.
     """
     if not task.is_simulation:
         raise RuntimeError(

alab_management/device_view/device.py

@@ -153,13 +153,13 @@ def sample_positions(self):
                   SamplePosition(
                       "inside",
                       description="The position inside the furnace, where the samples are heated",
-                      number=8
+                      number=8,
                   ),
                   SamplePosition(
                       "furnace_table",
                       description="Temporary position to transfer samples",
-                      number=16
-                  )
+                      number=16,
+                  ),
               ]
 
         """
@@ -286,6 +286,7 @@ class DeviceSignalEmitter:
     This class is responsible for periodically logging device signals to the database.
     It is intended to be used as a singleton, and should be instantiated once per device.
     """
+
     def __init__(self, device: BaseDevice):
         from alab_management.device_view import DeviceView
 
@@ -323,6 +324,7 @@ def _worker(self):
         """
         This is the worker thread that will periodically log device signals to the database.
         """
+
         def wait_with_option_to_kill(time_to_wait: float):
             """
             Waits until the next log is due, or until the logging is stopped. When stopping
@@ -390,9 +392,11 @@ def log_method_to_db(self, method_name: str, signal_name: str):
         try:
             value = method()
         except Exception:
-            value = f"Error reading {method_name} from device {self.device.name}." \
-                    f"The error message is: " \
-                    f"{format_exc()}"
+            value = (
+                f"Error reading {method_name} from device {self.device.name}."
+                f"The error message is: "
+                f"{format_exc()}"
+            )
 
         self.dblogger.log_device_signal(
             device_name=self.device.name,