diff --git a/.github/run_test.sh b/.github/run_test.sh
new file mode 100755
index 000000000..f8a97f51f
--- /dev/null
+++ b/.github/run_test.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+TARGETS=$@
+
+TEST_TARGET="tests"
+LTESTS=""  # all tests by default
+
+set -ex
+
+for TARGET in $TARGETS; do
+    case $TARGET in
+    --without-openssl)
+        # When built without OpenSSL we can't do the file-based RSA key tests.
+        TEST_TARGET=t-exec
+        ;;
+    esac
+done
+
+if [ -z "$LTESTS" ]; then
+    make $TEST_TARGET
+else
+    make $TEST_TARGET LTESTS="$LTESTS"
+fi
diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml
index 6ea16196b..2189756bb 100644
--- a/.github/workflows/c-cpp.yml
+++ b/.github/workflows/c-cpp.yml
@@ -34,6 +34,6 @@ jobs:
     - name: make
       run: make
     - name: make tests
-      run: make tests
+      run: ./.github/run_test.sh ${{ matrix.configs }}
       env:
         TEST_SSH_UNSAFE_PERMISSIONS: 1