expands pipeline documentation

docs/requirements.txt

@@ -1,4 +1,5 @@
 sphinx
 pydata-sphinx-theme
 sphinx-autoapi
-sphinx-favicon
+sphinx-favicon
+sphinxcontrib-mermaid

docs/source/conf.py

@@ -34,7 +34,8 @@
 extensions = ['autoapi.extension',
               'sphinx.ext.autodoc',
               'sphinx.ext.napoleon',
-              'sphinx_favicon']
+              'sphinx_favicon',
+              'sphinxcontrib.mermaid']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -71,15 +72,21 @@
         "image_dark": "_static/logo.png",
     }
 }
+
+mermaid_params = ['--backgroundColor', 'red']
+mermaid_verbose = True
+
 html_context = {
     # "github_url": "https://github.com", # or your GitHub Enterprise site
     "github_user": "punch-mission",
     "github_repo": "punchpipe",
     "github_version": "main",
     "doc_path": "docs/source/",
+
 }
 
 
+
 autoapi_dirs = ['../../punchpipe']
 
 favicons = ["favicon.ico"]

docs/source/control/configuration.rst

@@ -0,0 +1,4 @@
+Configuration
+==============
+
+How to set up a configuration file.

docs/source/control/index.rst

@@ -0,0 +1,9 @@
+Control Segment Design
+=========================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   states
+   configuration

docs/source/control/states.rst

@@ -0,0 +1,42 @@
+Flow and File States
+=====================
+The flow and file tables in the Reduction Database have a state property.
+This is expected to change according to the following state diagrams.
+
+File State Progression
+---------------------------
+
+.. mermaid::
+
+    graph LR;
+    planned --> creating;
+    creating --> failed;
+    creating --> created;
+    creating --> unreported;
+    created --> progressed;
+
+For files, they always begin in the ``planned`` state.
+Once the flow to create them is kicked off, they enter the ``creating`` state.
+From ``creating`` they can either become ``failed`` (when the flow creating them
+encounters an exception in execution),
+``created`` (the flow succeeded in creating the file),
+or ``unreported`` (when the flow that was supposed to update their state doesn't respond).
+Once files are ``created`` they wait in that state until a later flow picks them up as ``progressed``
+to the next level. Since level 3 is the last level, they never will become ``progressed``.
+
+
+
+Flow State Progression
+---------------------------
+
+.. mermaid::
+
+    graph LR;
+    planned --> running;
+    running --> failed;
+    running --> completed;
+
+Flow behavior is a bit simpler. Flows are ``planned`` by the scheduler.
+They then start ``running`` when the launcher decides its' time.
+From ``running`` they can either enter the ``failed`` or ``completed`` states depending
+on the success of their execution.

docs/source/index.rst

@@ -1,15 +1,12 @@
-.. punchpipe documentation master file, created by
-   sphinx-quickstart on Fri Nov 10 00:19:10 2023.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
 Welcome to punchpipe's documentation!
 =====================================
 
 .. toctree::
    :maxdepth: 2
    :caption: Contents:
 
+   control/index
+   levels/index
 
 
 Indices and tables

docs/source/levels/index.rst

@@ -0,0 +1,7 @@
+Levels of Processing
+=====================
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:

punchpipe/controlsegment/scheduler.py

@@ -20,7 +20,7 @@ def generic_scheduler_flow_logic(query_ready_files_func,
         # mark the file as progressed so that there aren't duplicate processing flows
         update_file_state(session, file_id, "progressed")
 
-        # get the level0 file's information
+        # get the prior level file's information
         parent_file = session.query(File).where(File.file_id == file_id).one()
 
         # prepare the new level flow and file
@@ -30,10 +30,10 @@ def generic_scheduler_flow_logic(query_ready_files_func,
         session.add(database_flow_info)
         session.commit()
 
-        # set the processing flow now that we know the flow_id after committing thea flow info
+        # set the processing flow now that we know the flow_id after committing the flow info
         child_file.processing_flow = database_flow_info.flow_id
         session.commit()
 
-        # create a file relationship between the level 0 and level 1
+        # create a file relationship between the prior and next levels
         session.add(FileRelationship(parent=parent_file.file_id, child=child_file.file_id))
         session.commit()