diff --git a/misc-tools/configure-domjudge.in b/misc-tools/configure-domjudge.in
index 360f783d5e..7de6a6901d 100755
--- a/misc-tools/configure-domjudge.in
+++ b/misc-tools/configure-domjudge.in
@@ -17,6 +17,7 @@ import json
 import os.path
 import requests
 import requests.utils
+import shutil
 import sys
 from typing import List, Set
 
@@ -101,11 +102,15 @@ else:
 if os.path.exists('executables'):
     executables = []
     for file in os.listdir('executables'):
-        if file.endswith(".zip"):
-            executables.append(file[:-4])
-    if executables and dj_utils.confirm('Upload language executables (found: ' + ','.join(executables) + ')?', False):
+        if os.path.isdir(f'executables/{file}'):
+            executables.append(file)
+
+    if executables:
+        if dj_utils.confirm('Upload language executables (found: ' + ','.join(executables) + ')?', False):
+            for langid in executables:
+                dj_utils.upload_file(f'languages/{langid}/executable', 'executable', f'executables/{langid}.zip')
         for langid in executables:
-            dj_utils.upload_file(f'languages/{langid}/executable', 'executable', f'executables/{langid}.zip')
+            os.remove(f'executables/{langid}.zip')
 
 
 if os.path.exists('languages.json'):