runtests: fix test key format for libssh2 WinCNG (and others)

SFTP/SCP tests were failing in CI with WinCNG libssh2 since we first
added such job. With `curl: (67) Authentication failure`.

The reason is that the default `ssh-keygen` RSA private key format
changed to OpenSSH (RFC4716) in 2018. libssh2 does not support this
format with some of its crypto backends.

Fix it by generating keys explicitly in PEM format as necessary via
the `-m` option. This format is universally recognized for RSA keys.

2018-08-24: https://www.openssh.com/txt/release-7.8: OpenSSH format becomes default
2010-08-23: https://www.openssh.com/txt/release-5.6: `-m` option first supported

This fixed the auth issue, just to reveal a known flakiness issue in
libssh2 + WinCNG, causing:
```
curl: (2) Failure establishing ssh session: -8, Unable to exchange encryption keys
```
Ref: https://github.com/curl/curl/actions/runs/14000494428/job/39205633258?pr=16781#step:15:1796
Tracked here: https://github.com/libssh2/libssh2/issues/804
Mitigated in libssh2 tests by retrying them.

Due to this, keep ignoring these test results.

Also:
- add an env to customize key format: `CURL_TEST_SSH_KEY_FORMAT`
- display the generated format in the log.
- GHA/linux: document the wolfSSH error code causing it to fail tests:
  ```
  curl: (79) wolfssh SFTP connect error -1051 / WS_MATCH_KEY_ALGO_E / cannot match key algo with peer
  ```

Follow-up to 4911e7af119c1b7efd46a742d47bca44832c3041 #16735
Follow-up to 0ec72c1ef8d87a29bf2eaa5e36ab173147a4d015 #16672
Follow-up to e53523fef07894991c69d907a7c7794c7ada4ff4 #14859
Follow-up to e26cbe20cbedbea0ca743dd33880517309315cb2 #13979

Closes #16781

.github/workflows/linux.yml

@@ -664,7 +664,7 @@ jobs:
           export TFLAGS='${{ matrix.build.tflags }}'
           if [ -z '${{ matrix.build.torture }}' ]; then
             if [[ '${{ matrix.build.install_steps }}' = *'wolfssh'* ]]; then
-              TFLAGS+=' ~SFTP'
+              TFLAGS+=' ~SFTP'  # curl: (79) wolfssh SFTP connect error -1051 / WS_MATCH_KEY_ALGO_E / cannot match key algo with peer
             fi
             if [[ '${{ matrix.build.install_packages }}' = *'valgrind'* ]]; then
               TFLAGS+=' -j6'

.github/workflows/windows.yml

@@ -345,6 +345,9 @@ jobs:
           export TFLAGS='-j8 ${{ matrix.tflags }}'
           if [ '${{ matrix.sys }}' != 'msys' ]; then
             TFLAGS+=' !498'  # 'Reject too large HTTP response headers on endless redirects' HTTP, HTTP GET (runtests detecting result code 2009 instead of 56 returned by curl)
+            if [[ '${{ matrix.install }}' = *'libssh2-wincng'* ]]; then
+              TFLAGS+=' ~SCP ~SFTP'  # Flaky: `-8, Unable to exchange encryption keys`. https://github.com/libssh2/libssh2/issues/804
+            fi
             TFLAGS+=' ~612'  # 'SFTP post-quote remove file' SFTP, post-quote
           fi
           TFLAGS+=' ~613'  # 'SFTP directory retrieval' SFTP, directory
@@ -933,7 +936,7 @@ jobs:
           export TFLAGS='-j8 ${{ matrix.tflags }}'
           TFLAGS+=' !498'  # 'Reject too large HTTP response headers on endless redirects' HTTP, HTTP GET (runtests detecting result code 2009 instead of 56 returned by curl)
           if [[ '${{ matrix.install }}' = *'libssh2[core,zlib]'* ]]; then
-            TFLAGS+=' !SCP !SFTP'  # Fail with all tested openssh servers: curl: (67) Authentication failure
+            TFLAGS+=' ~SCP ~SFTP'  # Flaky: `-8, Unable to exchange encryption keys`. https://github.com/libssh2/libssh2/issues/804
           fi
           TFLAGS+=' ~612'  # 'SFTP post-quote remove file' SFTP, post-quote
           if [ '${{ matrix.openssh }}' = '' ]; then  # MSYS2 openssh

tests/sshhelp.pm

@@ -50,6 +50,7 @@ BEGIN {
         $hstpubsha256f
         $cliprvkeyf
         $clipubkeyf
+        display_file_top
         display_sshdconfig
         display_sshconfig
         display_sftpconfig
@@ -190,6 +191,21 @@ sub display_file {
 }
 
 
+#***************************************************************************
+# Display first line of the given file
+#
+sub display_file_top {
+    my $filename = $_[0];
+    print "=== Top of file $filename\n";
+    if(open(my $displayfh, "<", "$filename")) {
+        my $line = <$displayfh>;
+        print "$line";
+        close $displayfh;
+    }
+    print "=== End of file $filename\n";
+}
+
+
 #***************************************************************************
 # Display contents of the ssh daemon config file
 #

tests/sshserver.pl

@@ -60,6 +60,7 @@ use sshhelp qw(
     $hstpubsha256f
     $cliprvkeyf
     $clipubkeyf
+    display_file_top
     display_sshdconfig
     display_sshconfig
     display_sftpconfig
@@ -408,16 +409,32 @@ if((! -e pp($hstprvkeyf)) || (! -s pp($hstprvkeyf)) ||
     # Make sure all files are gone so ssh-keygen doesn't complain
     unlink(pp($hstprvkeyf), pp($hstpubkeyf), pp($hstpubmd5f),
            pp($hstpubsha256f), pp($cliprvkeyf), pp($clipubkeyf));
+
+    my $sshkeygenopt = '';
+    if(($sshid =~ /OpenSSH/) && ($sshvernum >= 560)) {
+        # Override the default key format. Necessary to force legacy PEM format
+        # for libssh2 crypto backends that do not understand the OpenSSH (RFC4716)
+        # format, e.g. WinCNG.
+        # Accepted values: RFC4716, PKCS8, PEM (see also 'man ssh-keygen')
+        if($ENV{'CURL_TEST_SSH_KEY_FORMAT'}) {
+            $sshkeygenopt .= ' -m ' . $ENV{'CURL_TEST_SSH_KEY_FORMAT'};
+        }
+        else {
+            $sshkeygenopt .= ' -m PEM';  # Use the most compatible RSA format for tests.
+        }
+    }
     logmsg "generating host keys...\n" if($verbose);
-    if(system "\"$sshkeygen\" -q -t rsa -f " . pp($hstprvkeyf) . " -C 'curl test server' -N ''") {
+    if(system "\"$sshkeygen\" -q -t rsa -f " . pp($hstprvkeyf) . " -C 'curl test server' -N ''" . $sshkeygenopt) {
         logmsg "Could not generate host key\n";
         exit 1;
     }
+    display_file_top(pp($hstprvkeyf));
     logmsg "generating client keys...\n" if($verbose);
-    if(system "\"$sshkeygen\" -q -t rsa -f " . pp($cliprvkeyf) . " -C 'curl test client' -N ''") {
+    if(system "\"$sshkeygen\" -q -t rsa -f " . pp($cliprvkeyf) . " -C 'curl test client' -N ''" . $sshkeygenopt) {
         logmsg "Could not generate client key\n";
         exit 1;
     }
+    display_file_top(pp($cliprvkeyf));
     # Make sure that permissions are restricted so openssh doesn't complain
     system "chmod 600 " . pp($hstprvkeyf);
     system "chmod 600 " . pp($cliprvkeyf);