diff --git a/packages/lib-classifier/src/store/Classification/Classification.js b/packages/lib-classifier/src/store/Classification/Classification.js
index b510646774..6731e14e35 100644
--- a/packages/lib-classifier/src/store/Classification/Classification.js
+++ b/packages/lib-classifier/src/store/Classification/Classification.js
@@ -1,9 +1,10 @@
 import cuid from 'cuid'
-import { types, getSnapshot, getType } from 'mobx-state-tree'
+import { types, getSnapshot, getType, addDisposer } from 'mobx-state-tree'
 import * as tasks from '@plugins/tasks'
 import AnnotationsStore from '@store/AnnotationsStore'
 import Resource from '@store/Resource'
 import ClassificationMetadata  from './ClassificationMetadata'
+import { autorun } from 'mobx'
 
 const annotationModels = Object.values(tasks).map(task => task.AnnotationModel)
 
@@ -19,6 +20,17 @@ const Classification = types
     metadata: types.maybe(ClassificationMetadata)
   })
   .views(self => ({
+    /**
+     * Returns false until we start updating task annotations.
+     */
+    get inProgress() {
+      let inProgress = false
+      self.annotations.forEach(annotation => {
+        inProgress ||= annotation._inProgress
+      })
+      return inProgress
+    },
+
     toSnapshot () {
       let snapshot = getSnapshot(self)
       let annotations = []
@@ -58,5 +70,23 @@ const Classification = types
     newSnapshot.annotations = Object.values(snapshot.annotations)
     return newSnapshot
   })
+  .actions(self => {
+    function _onAnnotationsChange () {
+      // set started at when inProgress changes from false to true
+      if (self.inProgress) {
+        self.setStartedAt()
+      }
+    }
+
+    return ({
+      afterAttach () {
+        addDisposer(self, autorun(_onAnnotationsChange))
+      },
+
+      setStartedAt () {
+        self.metadata.startedAt = new Date().toISOString()
+      }
+    })
+  })
 
 export default types.compose('ClassificationResource', Resource, AnnotationsStore, Classification)
diff --git a/packages/lib-classifier/src/store/ClassificationStore.js b/packages/lib-classifier/src/store/ClassificationStore.js
index 33f53fee3f..bcf0ceeee8 100644
--- a/packages/lib-classifier/src/store/ClassificationStore.js
+++ b/packages/lib-classifier/src/store/ClassificationStore.js
@@ -88,10 +88,6 @@ const ClassificationStore = types
 
       if (validClassificationReference) {
         const classification = self.active
-        if (classification?.annotations.size === 0) {
-          // update startedAt if we're starting a new classification
-          classification.metadata.startedAt = (new Date()).toISOString()
-        }
         if (classification) {
           return classification.addAnnotation(task, annotationValue)
         }
diff --git a/packages/lib-classifier/src/store/ClassificationStore.spec.js b/packages/lib-classifier/src/store/ClassificationStore.spec.js
index 3955d6dcfc..0cdbc17d3d 100644
--- a/packages/lib-classifier/src/store/ClassificationStore.spec.js
+++ b/packages/lib-classifier/src/store/ClassificationStore.spec.js
@@ -132,10 +132,12 @@ describe('Model > ClassificationStore', function () {
         })
         const taskSnapshot = Object.assign({}, singleChoiceTaskSnapshot, { taskKey: singleChoiceAnnotationSnapshot.task })
         taskSnapshot.createAnnotation = () => SingleChoiceAnnotation.create(singleChoiceAnnotationSnapshot)
+        const classification = classifications.active
+        const annotation = classification.createAnnotation(taskSnapshot)
         clock.tick(1 * 60 * 60 * 1000) // wait for one hour before starting the classification.
-        classifications.addAnnotation(taskSnapshot, singleChoiceAnnotationSnapshot.value)
+        annotation.update(0)
         clock.tick(30 * 1000) // wait for 30 seconds before finishing the classification.
-        classifications.addAnnotation(taskSnapshot, 1)
+        annotation.update(1)
         classifications.completeClassification()
       })