diff --git a/webapp/src/Controller/Jury/ConfigController.php b/webapp/src/Controller/Jury/ConfigController.php
index 8005ae6173b..bbddc76be73 100644
--- a/webapp/src/Controller/Jury/ConfigController.php
+++ b/webapp/src/Controller/Jury/ConfigController.php
@@ -85,8 +85,26 @@ public function indexAction(EventLogService $eventLogService, Request $request):
             }
 
             if (empty($errors)) {
-                $this->addFlash('scoreboard_refresh', 'After changing specific ' .
-                    'settings, you might need to refresh the scoreboard.');
+                $needsRefresh = false;
+                $needsRejudging = false;
+                foreach ($diffs as $key => $diff) {
+                    $category = $this->config->getCategory($key);
+                    if ($category === 'Scoring') {
+                        $needsRefresh = true;
+                    }
+                    if ($category === 'Judging') {
+                        $needsRejudging = true;
+                    }
+                }
+
+                if ($needsRefresh) {
+                    $this->addFlash('scoreboard_refresh', 'After changing specific ' .
+                        'scoring related settings, you might need to refresh the scoreboard (cache).');
+                }
+                if ($needsRejudging) {
+                    $this->addFlash('danger', 'After changing specific ' .
+                        'judging related settings, you might need to rejudge affected submissions.');
+                }
 
                 return $this->redirectToRoute('jury_config', ['diffs' => json_encode($diffs)]);
             } else {
diff --git a/webapp/src/Service/ConfigurationService.php b/webapp/src/Service/ConfigurationService.php
index 65457f168a0..c9852033b7c 100644
--- a/webapp/src/Service/ConfigurationService.php
+++ b/webapp/src/Service/ConfigurationService.php
@@ -70,6 +70,12 @@ public function get(string $name, bool $onlyIfPublic = false)
         return $value;
     }
 
+    public function getCategory(string $name): ?string
+    {
+        $spec = $this->getConfigSpecification()[$name] ?? null;
+        return $spec?->category;
+    }
+
     /**
      * Get all the configuration values, indexed by name.
      *