diff --git a/crypto/ts/ts_rsp_verify.c b/crypto/ts/ts_rsp_verify.c
index e13fd90d37..21da8aaa9e 100644
--- a/crypto/ts/ts_rsp_verify.c
+++ b/crypto/ts/ts_rsp_verify.c
@@ -347,36 +347,43 @@ static int int_ts_RESP_verify_token(TS_VERIFY_CTX *ctx,
     unsigned char *imprint = NULL;
     unsigned imprint_len = 0;
     int ret = 0;
+    int flags = ctx->flags;
 
-    if ((ctx->flags & TS_VFY_SIGNATURE)
+    /* Some options require us to also check the signature */
+    if (((flags & TS_VFY_SIGNER) && tsa_name != NULL)
+            || (flags & TS_VFY_TSA_NAME)) {
+        flags |= TS_VFY_SIGNATURE;
+    }
+
+    if ((flags & TS_VFY_SIGNATURE)
         && !TS_RESP_verify_signature(token, ctx->certs, ctx->store, &signer))
         goto err;
-    if ((ctx->flags & TS_VFY_VERSION)
+    if ((flags & TS_VFY_VERSION)
         && TS_TST_INFO_get_version(tst_info) != 1) {
         TSerr(TS_F_INT_TS_RESP_VERIFY_TOKEN, TS_R_UNSUPPORTED_VERSION);
         goto err;
     }
-    if ((ctx->flags & TS_VFY_POLICY)
+    if ((flags & TS_VFY_POLICY)
         && !ts_check_policy(ctx->policy, tst_info))
         goto err;
-    if ((ctx->flags & TS_VFY_IMPRINT)
+    if ((flags & TS_VFY_IMPRINT)
         && !ts_check_imprints(ctx->md_alg, ctx->imprint, ctx->imprint_len,
                               tst_info))
         goto err;
-    if ((ctx->flags & TS_VFY_DATA)
+    if ((flags & TS_VFY_DATA)
         && (!ts_compute_imprint(ctx->data, tst_info,
                                 &md_alg, &imprint, &imprint_len)
             || !ts_check_imprints(md_alg, imprint, imprint_len, tst_info)))
         goto err;
-    if ((ctx->flags & TS_VFY_NONCE)
+    if ((flags & TS_VFY_NONCE)
         && !ts_check_nonces(ctx->nonce, tst_info))
         goto err;
-    if ((ctx->flags & TS_VFY_SIGNER)
+    if ((flags & TS_VFY_SIGNER)
         && tsa_name && !ts_check_signer_name(tsa_name, signer)) {
         TSerr(TS_F_INT_TS_RESP_VERIFY_TOKEN, TS_R_TSA_NAME_MISMATCH);
         goto err;
     }
-    if ((ctx->flags & TS_VFY_TSA_NAME)
+    if ((flags & TS_VFY_TSA_NAME)
         && !ts_check_signer_name(ctx->tsa_name, signer)) {
         TSerr(TS_F_INT_TS_RESP_VERIFY_TOKEN, TS_R_TSA_UNTRUSTED);
         goto err;