diff --git a/tests/detect_aws_credentials_test.py b/tests/detect_aws_credentials_test.py
index 495c574..2fbb2da 100644
--- a/tests/detect_aws_credentials_test.py
+++ b/tests/detect_aws_credentials_test.py
@@ -12,10 +12,6 @@
     ('ok_json.json', 0),
 )
 
-NO_CREDENTIALS_TEST = (
-    ('with_secrets.txt', 2),
-)
-
 
 @pytest.mark.parametrize(('filename', 'expected_retval'), TESTS)
 def test_detect_aws_credentials(filename, expected_retval):
@@ -26,10 +22,16 @@
     assert ret == expected_retval
 
 
-@pytest.mark.parametrize(('filename', 'expected_retval'), NO_CREDENTIALS_TEST)
-def test_non_existent_credentials(filename, expected_retval):
+def test_non_existent_credentials(capsys):
     # with a non-existent credentials file
-    ret = main(
-        [get_resource_path(filename), "--credentials-file=testing/resources/credentailsfilethatdoesntexist"]
+    ret = main((
+        get_resource_path('with_secrets.txt'),
+        "--credentials-file=testing/resources/credentailsfilethatdoesntexist"
+    ))
+    assert ret == 2
+    out, _ = capsys.readouterr()
+    assert out == (
+        'No aws keys were configured at '
+        'testing/resources/credentailsfilethatdoesntexist\n'
+        'Configure them with --credentials-file\n'
     )
-    assert ret == expected_retval