diff --git a/setup.cfg b/setup.cfg
index 3c156a5..b578bfa 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -61,6 +61,7 @@ universal = true
 formats = gztar
 
 [coverage:report]
+include = orthoani/*.py
 show_missing = true
 exclude_lines =
     pragma: no cover
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 9e217c3..e29e5f7 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -4,7 +4,6 @@
 import unittest
 import subprocess
 import sys
-from subprocess import PIPE
 
 from Bio.SeqIO import read
 
@@ -17,9 +16,35 @@ def setUpClass(cls):
         cls.data = pathlib.Path(__file__).parent / "data"
         cls.p1, cls.p2 = map(os.fspath, cls.data.glob("*.fna"))
 
+    def shell(self, args):
+        return subprocess.run(
+            args,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            cwd=self.data.parent.parent,
+        )
+
     def test_orthoani_cli(self):
         # check the score we get is the same as the OrthoANI Java implementation
         args = [sys.executable, "-m", "orthoani", "-q", self.p1, "-r", self.p2]
-        proc = subprocess.run(args, stdout=PIPE, cwd=self.data.parent.parent)
+        proc = self.shell(args)
         proc.check_returncode()
         self.assertEqual(proc.stdout, b"0.5725\n")
+
+    def test_return_code(self):
+        args = [sys.executable, "-m", "orthoani", "-q", self.data, "-r", self.p2]
+        proc = self.shell(args)
+        self.assertEqual(proc.returncode, 21) # IsADirectoryError
+
+        args = [sys.executable, "-m", "orthoani", "-q", "_", "-r", self.p2]
+        proc = self.shell(args)
+        self.assertEqual(proc.returncode, 2) # FileNotFoundError
+
+    def test_return_code_traceback(self):
+        args = [sys.executable, "-m", "orthoani", "-T", "-q", self.data, "-r", self.p2]
+        proc = self.shell(args)
+        self.assertEqual(proc.returncode, 21) # IsADirectoryError
+
+        args = [sys.executable, "-m", "orthoani", "-T", "-q", "_", "-r", self.p2]
+        proc = self.shell(args)
+        self.assertEqual(proc.returncode, 2) # FileNotFoundError