From 7c8a6a55e30454723c9abc9e9525209eb9d950e0 Mon Sep 17 00:00:00 2001 From: Mojmir Fendek Date: Tue, 9 Jun 2020 09:35:52 +1200 Subject: [PATCH 1/3] MISC: Improved documentation on special job variables. --- README.md | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/README.md b/README.md index 046ffb19..a7ee6636 100644 --- a/README.md +++ b/README.md @@ -487,6 +487,119 @@ $this->assertNotEquals(QueuedJob::STATUS_BROKEN, $descriptor->JobStatus); For example, this code snippet runs the job and checks if the job ended up in a non-broken state. +## Special job variables + +It's good to be aware of special variables which should be used in your job implementation. + +* `totalSteps` (integer) - defaults to `0`, maps to `TotalSteps` DB column +* `currentStep` (integer) - defaults to `0`, maps to `StepsProcessed` DB column +* `isComplete` (boolean) - defaults to `false`, related to `JobStatus` DB column + +See `copyJobToDescriptor` for more details on the mapping between `Job` and `JobDescriptor`. + +### Total steps + +* represents total number of steps needed to complete the job +* this variable should be set to proper value in the `setup()` function of your job +* the value should not be changed after that +* this variable is not used by the Queue runner and is only meant to indicate how many steps are needed (information only) +* it is recommended to avoid using this variable inside the `process()` function of your job +* instead, determine if your job is complete based on the job data (if there are any more items left to process) + +### Current step + +* represents number of steps processed +* your job should increment this variable each time a job step was successfully completed +* Queue runner will read this variable to determine if your job is stalled or not +* it is recommended to return out of the `process()` function each time you increment this variable +* this allows the queue runner to create a checkpoint by saving your job progress into the job descriptor which is stored in the DB + +### Is complete + +* represents the job state (complete or not) +* setting this variable to `true` will give a signal to the queue runner to mark the job as successfully completed +* your job should set this variable to `true` only once + +### Summary + +* `totalSteps` - information only +* `currentStep` - Queue runner uses this to determine if job is stalled or not +* `isComplete` - Queue runner uses this to determine if job is completed or not + +### Example + +```php + +items = $items; + } + + /** + * @return string + */ + public function getTitle(): string + { + return 'My awesome job'; + } + + public function setup(): void + { + $this->remaining = $this->items; + + // Set the total steps to the number of items we want to process + $this->totalSteps = count($this->items); + } + + public function process(): void + { + $remaining = $this->remaining; + + // check for trivial case + if (count($remaining) === 0) { + $this->isComplete = true; + + return; + } + + $item = array_shift($remaining); + + // code that will process your item goes here + $this->doSomethingWithTheItem($item); + + $this->remaining = $remaining; + + // Updating current step tells the Queue runner that the job is progressing + $this->currentStep += 1; + + // check for job completion + if (count($remaining) > 0) { + // Note that we do not process more than one item at a time + // this makes the Queue runner save the job progress into DB + // in case something goes wrong the job will be resumed from the last checkpoint + return; + } + + // Queue runner will mark this job as finished + $this->isComplete = true; + } +} +``` + ## Advanced job setup This section is recommended for developers who are already familiar with basic concepts and want to take full advantage of the features in this module. From 7172da70d93a9980ae48463ab9bf0830b6fad25e Mon Sep 17 00:00:00 2001 From: Mojmir Fendek Date: Fri, 26 Jun 2020 15:13:44 +1200 Subject: [PATCH 2/3] PR fixes --- README.md | 225 +++++++++++++++++++++++++++--------------------------- 1 file changed, 112 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index a7ee6636..04dede27 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ In your YML set the below: ```yaml + --- Name: localproject After: '#queuedjobsettings' @@ -109,6 +110,7 @@ SilverStripe\Core\Injector\Injector: * Create a `_config/queuedjobs.yml` file in your project with the following declaration ```yaml + --- Name: localproject After: '#queuedjobsettings' @@ -289,6 +291,116 @@ detected: ``` */5 * * * * /path/to/silverstripe/vendor/bin/sake dev/tasks/CheckJobHealthTask ``` +## Special job variables + +It's good to be aware of special variables which should be used in your job implementation. + +* `totalSteps` (integer) - defaults to `0`, maps to `TotalSteps` DB column, information only +* `currentStep` (integer) - defaults to `0`, maps to `StepsProcessed` DB column, Queue runner uses this to determine if job is stalled or not +* `isComplete` (boolean) - defaults to `false`, related to `JobStatus` DB column, Queue runner uses this to determine if job is completed or not + +See `copyJobToDescriptor` for more details on the mapping between `Job` and `JobDescriptor`. + +### Total steps + +Represents total number of steps needed to complete the job. + +* this variable should be set to the number of steps you expect your job to go though during its execution +* this needs to be done in the `setup()` function of your job, the value should not be changed after that +* this variable is not used by the Queue runner and is only meant to indicate how many steps are needed (information only) +* it is recommended to avoid using this variable inside the `process()` function of your job instead, determine if your job is complete based on the job data (if there are any more items left to process) + +### Current step + +Represents number of steps processed. + +* your job should increment this variable each time a job step was successfully completed +* Queue runner will read this variable to determine if your job is stalled or not +* it is recommended to return out of the `process()` function each time you increment this variable +* this allows the queue runner to create a checkpoint by saving your job progress into the job descriptor which is stored in the DB + +### Is complete + +Represents the job state (complete or not). + +* setting this variable to `true` will give a signal to the queue runner to mark the job as successfully completed +* your job should set this variable to `true` only once + +### Example + +This example illustrates how each special variable should be used in your job implementation. + +```php + +items = $items; + } + + /** + * @return string + */ + public function getTitle(): string + { + return 'My awesome job'; + } + + public function setup(): void + { + $this->remaining = $this->items; + + // Set the total steps to the number of items we want to process + $this->totalSteps = count($this->items); + } + + public function process(): void + { + $remaining = $this->remaining; + + // check for trivial case + if (count($remaining) === 0) { + $this->isComplete = true; + + return; + } + + $item = array_shift($remaining); + + // code that will process your item goes here + $this->doSomethingWithTheItem($item); + + $this->remaining = $remaining; + + // Updating current step tells the Queue runner that the job is progressing + $this->currentStep += 1; + + // check for job completion + if (count($remaining) > 0) { + // Note that we do not process more than one item at a time + // this makes the Queue runner save the job progress into DB + // in case something goes wrong the job will be resumed from the last checkpoint + return; + } + + // Queue runner will mark this job as finished + $this->isComplete = true; + } +} +``` ## Troubleshooting @@ -487,119 +599,6 @@ $this->assertNotEquals(QueuedJob::STATUS_BROKEN, $descriptor->JobStatus); For example, this code snippet runs the job and checks if the job ended up in a non-broken state. -## Special job variables - -It's good to be aware of special variables which should be used in your job implementation. - -* `totalSteps` (integer) - defaults to `0`, maps to `TotalSteps` DB column -* `currentStep` (integer) - defaults to `0`, maps to `StepsProcessed` DB column -* `isComplete` (boolean) - defaults to `false`, related to `JobStatus` DB column - -See `copyJobToDescriptor` for more details on the mapping between `Job` and `JobDescriptor`. - -### Total steps - -* represents total number of steps needed to complete the job -* this variable should be set to proper value in the `setup()` function of your job -* the value should not be changed after that -* this variable is not used by the Queue runner and is only meant to indicate how many steps are needed (information only) -* it is recommended to avoid using this variable inside the `process()` function of your job -* instead, determine if your job is complete based on the job data (if there are any more items left to process) - -### Current step - -* represents number of steps processed -* your job should increment this variable each time a job step was successfully completed -* Queue runner will read this variable to determine if your job is stalled or not -* it is recommended to return out of the `process()` function each time you increment this variable -* this allows the queue runner to create a checkpoint by saving your job progress into the job descriptor which is stored in the DB - -### Is complete - -* represents the job state (complete or not) -* setting this variable to `true` will give a signal to the queue runner to mark the job as successfully completed -* your job should set this variable to `true` only once - -### Summary - -* `totalSteps` - information only -* `currentStep` - Queue runner uses this to determine if job is stalled or not -* `isComplete` - Queue runner uses this to determine if job is completed or not - -### Example - -```php - -items = $items; - } - - /** - * @return string - */ - public function getTitle(): string - { - return 'My awesome job'; - } - - public function setup(): void - { - $this->remaining = $this->items; - - // Set the total steps to the number of items we want to process - $this->totalSteps = count($this->items); - } - - public function process(): void - { - $remaining = $this->remaining; - - // check for trivial case - if (count($remaining) === 0) { - $this->isComplete = true; - - return; - } - - $item = array_shift($remaining); - - // code that will process your item goes here - $this->doSomethingWithTheItem($item); - - $this->remaining = $remaining; - - // Updating current step tells the Queue runner that the job is progressing - $this->currentStep += 1; - - // check for job completion - if (count($remaining) > 0) { - // Note that we do not process more than one item at a time - // this makes the Queue runner save the job progress into DB - // in case something goes wrong the job will be resumed from the last checkpoint - return; - } - - // Queue runner will mark this job as finished - $this->isComplete = true; - } -} -``` - ## Advanced job setup This section is recommended for developers who are already familiar with basic concepts and want to take full advantage of the features in this module. From 4c8aa393cbda9d316d0b8851be44bcb1be4c2cc1 Mon Sep 17 00:00:00 2001 From: Steve Boyd Date: Wed, 3 Mar 2021 17:59:42 +1300 Subject: [PATCH 3/3] [CVE-2021-27938] Prevent echoing request variable --- src/Tasks/CreateQueuedJobTask.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tasks/CreateQueuedJobTask.php b/src/Tasks/CreateQueuedJobTask.php index acd6e8b8..4f67de7a 100644 --- a/src/Tasks/CreateQueuedJobTask.php +++ b/src/Tasks/CreateQueuedJobTask.php @@ -54,7 +54,7 @@ public function run($request) $now = time(); if ($start >= $now) { $friendlyStart = date('Y-m-d H:i:s', $start); - echo "Job ".$request['name']. " queued to start at: ".$friendlyStart.""; + echo 'Job queued to start at: ' . $friendlyStart . ''; singleton('Symbiote\\QueuedJobs\\Services\\QueuedJobService')->queueJob($job, $start); } else { echo "'start' parameter must be a date/time in the future, parseable with strtotime";