diff --git a/ssh-pkcs11.c b/ssh-pkcs11.c
index a302c79c0..cae24525b 100644
--- a/ssh-pkcs11.c
+++ b/ssh-pkcs11.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh-pkcs11.c,v 1.47 2020/01/25 00:03:36 djm Exp $ */
+/* $OpenBSD: ssh-pkcs11.c,v 1.48 2020/03/06 18:14:13 markus Exp $ */
 /*
  * Copyright (c) 2010 Markus Friedl.  All rights reserved.
  * Copyright (c) 2014 Pedro Martelletto. All rights reserved.
@@ -1627,6 +1627,8 @@ fail:
 	}
 	if (handle)
 		dlclose(handle);
+	if (ret > 0)
+		ret = -1;
 	return (ret);
 }