diff --git a/src/pydiode/decrypt.py b/src/pydiode/decrypt.py
index ca8bd49..4a0c8ab 100644
--- a/src/pydiode/decrypt.py
+++ b/src/pydiode/decrypt.py
@@ -20,8 +20,9 @@
 def main():
     for line in sys.stdin:
         path = Path(line.strip())
+        # Decrypt files ending in .gpg
         if path.exists() and path.suffix == ".gpg":
-            subprocess.run(
+            gpg = subprocess.run(
                 [
                     "gpg",
                     "--batch",
@@ -33,6 +34,9 @@ def main():
                     path,
                 ]
             )
+            # If decryption succeeded, delete the encrypted file
+            if not gpg.returncode:
+                path.unlink()
 
 
 if __name__ == "__main__":
diff --git a/tests/msg.gpg b/tests/msg.gpg
new file mode 100644
index 0000000..1ab70ad
Binary files /dev/null and b/tests/msg.gpg differ
diff --git a/tests/test_decrypt.py b/tests/test_decrypt.py
index 363ec7c..ca59f53 100644
--- a/tests/test_decrypt.py
+++ b/tests/test_decrypt.py
@@ -17,25 +17,40 @@ def setUp(self):
         )
 
     def test_decrypt(self):
-        files = {
-            Path("1.txt.gpg"): "ABC",
-            Path("2.txt.gpg"): "DEF",
-            Path("3.txt.gpg"): "GHI",
+        # We have a secret key for all but the last file
+        encrypted_files = [
+            Path("1.txt.gpg"),
+            Path("2.txt.gpg"),
+            Path("3.txt.gpg"),
+            Path("msg.gpg"),
+        ]
+        decrypted_files = {
+            Path("1.txt"): "ABC",
+            Path("2.txt"): "DEF",
+            Path("3.txt"): "GHI",
         }
 
         with tempfile.TemporaryDirectory() as tmpdir:
             # Copy the encrypted files
-            for file in files:
+            for file in encrypted_files:
                 shutil.copy(PARENT_DIR / file, tmpdir)
 
             # Decrypt the files
             decrypt = subprocess.run(
                 [sys.executable, "-m", "pydiode.decrypt"],
-                input="\n".join([str(tmpdir / file) for file in files.keys()]),
+                input="\n".join([str(tmpdir / f) for f in encrypted_files]),
+                capture_output=True,
+                check=True,
                 text=True,
             )
 
             # Check the files were decrypted
-            for file, data in files.items():
-                with open(tmpdir / file.with_suffix(""), "r") as f:
+            for file, data in decrypted_files.items():
+                with open(tmpdir / file, "r") as f:
                     self.assertEqual(data, f.read().strip())
+
+            # The properly decrypted files should be deleted
+            for file in encrypted_files[:-1]:
+                self.assertFalse((tmpdir / file).exists())
+            # The file that wan't decrypted should exist
+            self.assertTrue((tmpdir / encrypted_files[-1]).exists())