mirror of
https://github.com/curl/curl.git
synced 2025-03-07 15:27:17 +08:00
Add support for single-block binaries that contain all libtests and unit tests respectively. Enable with: - autotools: `--enable-test-bundles` - cmake: `-DCURL_TEST_BUNDLES=ON` (They are compatible with `--enable-unity` and `-DCMAKE_UNITY_BUILD=ON` options, for further speed-up.) Makes libtests and unit tests build _fast_, needing little disk space even in static mode. Similar to CMake unity mode, but with a custom script, also supporting autotools builds. The price is having to deal with symbols/macros colliding between `lib*.c` and `unit*.c` sources. Maybe with naming conventions or other solutions this can be improved gradually and reduce the need for manual intervention by `mk-bundle.mk`. I've included a script that does the bulk of detecting name collisions. Also: - CI: enable test bundles. - CI: build tests in more jobs. - lib2305: fix FILE handle leak. - unit1661: fix memleak found by torture test by releasing the `bufref` structure in `unit_stop()` that was allocated in `unit_setup()`. ``` test 1661...[bufref unit tests] Leak detected: memory still allocated: 13 bytes allocated by /home/runner/work/curl/curl/tests/unit/unit1661.c:70 1661: torture FAILED: function number 1 in test. ``` Ref: https://github.com/curl/curl/actions/runs/10967279334/job/30456745290?pr=14772#step:8:41 Similar test suite builds with autotools default and cmake+bundle+unity: - GHA/Linux: 33s vs 7s https://github.com/curl/curl/actions/runs/10705668823/job/29681617374 - GHA/macOS 34s vs 2s https://github.com/curl/curl/actions/runs/10705668813/job/29681632885 - GHA/FreeBSD: 15m25 vs 6m21 (full workflow time, ~qemu) https://github.com/curl/curl/actions/runs/10705668811/job/29681607915 - GHA/Cygwin: 9m52 vs 32s https://github.com/curl/curl/actions/runs/10705668809/job/29681609965 - GHA/MSYS2: 3m52 vs 14s https://github.com/curl/curl/actions/runs/10705668808/job/29681624295 - GHA/mingw-w64: 5m45 vs 30s https://github.com/curl/curl/actions/runs/10705668808/job/29681628787 Autotools test suite builds compared between master -> `--enable-test-bundles`: - GHA/Linux: 33s -> 9s (run tests: 22m23 -> 20m44) https://github.com/curl/curl/actions/runs/10710030193/job/29695932185 https://github.com/curl/curl/actions/runs/10967831456/job/30458220344 - GHA/macOS: 25s -> 4s (run tests: 2m58 -> 2m24) https://github.com/curl/curl/actions/runs/10710030195/job/29695938444 https://github.com/curl/curl/actions/runs/10967831452/job/30458225762 - GHA/non-native (FreeBSD): 4m8 -> 3m12 (full workflow time, ~qemu) https://github.com/curl/curl/actions/runs/10710030198/job/29695928401 https://github.com/curl/curl/actions/runs/10967831458/job/30458212692 - GHA/Cygwin: 9m25 -> 1m9 (run tests: 9m19 -> 3m28) https://github.com/curl/curl/actions/runs/10710030212/job/29695928213 https://github.com/curl/curl/actions/runs/10967831453/job/30458213268 - GHA/MSYS2: 3m54 -> 32s (run tests: 6m3 -> 3m59) https://github.com/curl/curl/actions/runs/10710030190/job/29704850591 https://github.com/curl/curl/actions/runs/10967831449/job/30459280005 - GHA/mingw-w64: 5m42 -> 1m5 (run tests: 7m41 -> 5m36) https://github.com/curl/curl/actions/runs/10710030190/job/29704852058 https://github.com/curl/curl/actions/runs/10967831449/job/30459280862 - Azure MSYS2 mingw64 openssl: 38m55 -> 11m58 https://dev.azure.com/daniel0244/curl/_build/results?buildId=25546&view=logs&j=b58b8c59-0f61-52e9-0f9e-fad562a1e77f&t=0f9230a7-3b10-53ca-9938-700ece377c5e https://dev.azure.com/daniel0244/curl/_build/results?buildId=25547&view=logs&jobId=39473db1-3945-55d5-deb5-c218fad88dce&j=b58b8c59-0f61-52e9-0f9e-fad562a1e77f&t=0f9230a7-3b10-53ca-9938-700ece377c5e - Azure Ubuntu default: 2m15 -> 55s (all build) https://dev.azure.com/daniel0244/curl/_build/results?buildId=25546&view=logs&j=9d58b9ac-e1e6-53b6-f83a-1f9f1d912522&t=a6b38d83-e7cf-5a9b-c762-a178412717b7 https://dev.azure.com/daniel0244/curl/_build/results?buildId=25547&view=logs&jobId=39473db1-3945-55d5-deb5-c218fad88dce&j=9d58b9ac-e1e6-53b6-f83a-1f9f1d912522&t=a6b38d83-e7cf-5a9b-c762-a178412717b7 Cmake test suite builds compared between master -> `-DCURL_TEST_BUNDLES=ON` + unity: - GHA/Linux: 29s -> 7s (run tests: 4m50 -> 4m57, 20m43 -> 20m45) https://github.com/curl/curl/actions/runs/10710030193/job/29695941814 https://github.com/curl/curl/actions/runs/10705668823/job/29681622201 - GHA/Linux old: 44s -> 13s (bundle+no unity) (run tests: 5m5 -> 5m6) https://github.com/curl/curl/actions/runs/10718264094/job/29719794727 https://github.com/curl/curl/actions/runs/10718653175/job/29721009613 - GHA/macOS: 32s -> 2s (run tests: 2m43 -> 2m40) https://github.com/curl/curl/actions/runs/10710030195/job/29695931956 https://github.com/curl/curl/actions/runs/10705668813/job/29681638937 - GHA/non-native (*BSD): inconclusive (full workflow time, ~qemu) https://github.com/curl/curl/actions/runs/10710030198 https://github.com/curl/curl/actions/runs/10705668811 - GHA/Cygwin: 3m9 -> 32s https://github.com/curl/curl/actions/runs/10710030212/job/29695929075 https://github.com/curl/curl/actions/runs/10705668809/job/29681609965 - GHA/MSYS2: 2m24 -> 14s https://github.com/curl/curl/actions/runs/10710030190/job/29704850996 https://github.com/curl/curl/actions/runs/10705668808/job/29681624295 - GHA/mingw-w64: 3m56 -> 30s (run tests: 4m2 -> 3m52) https://github.com/curl/curl/actions/runs/10710030190/job/29704852219 https://github.com/curl/curl/actions/runs/10705668808/job/29681631393 - GHA/mingw-w64-old: 7m19 -> 1m44 (run tests: 3m30 -> 2m53) https://github.com/curl/curl/actions/runs/10710030190/job/29704849763 https://github.com/curl/curl/actions/runs/10705668808/job/29681622329 - GHA/MSVC: 3m22 -> 13s (run tests: 9m43 -> 4m22) https://github.com/curl/curl/actions/runs/10710030190/job/29704850411 https://github.com/curl/curl/actions/runs/10705668808/job/29681623313 - AppVeyor CI MSVC 2008: 4m3 -> 45s (full build) - AppVeyor CI MSVC 2010: 2m56 -> 1m8 (full build) - AppVeyor CI MSVC 2022: 10m19 -> 2m23 (full build) https://ci.appveyor.com/project/curlorg/curl/builds/50538455 https://ci.appveyor.com/project/curlorg/curl/builds/50536558 - AppVeyor CI total build time: 10m30 (master) -> 6m48 (unity) -> 4m5 (bundle) -> 3m24 (bundle+unity) -> 5m7 (bundle+unity+all jobs building tests) Closes #14772
356 lines
9.0 KiB
C
356 lines
9.0 KiB
C
/***************************************************************************
|
|
* _ _ ____ _
|
|
* Project ___| | | | _ \| |
|
|
* / __| | | | |_) | |
|
|
* | (__| |_| | _ <| |___
|
|
* \___|\___/|_| \_\_____|
|
|
*
|
|
* Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
|
*
|
|
* This software is licensed as described in the file COPYING, which
|
|
* you should have received as part of this distribution. The terms
|
|
* are also available at https://curl.se/docs/copyright.html.
|
|
*
|
|
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
|
|
* copies of the Software, and permit persons to whom the Software is
|
|
* furnished to do so, under the terms of the COPYING file.
|
|
*
|
|
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
|
* KIND, either express or implied.
|
|
*
|
|
* SPDX-License-Identifier: curl
|
|
*
|
|
***************************************************************************/
|
|
#include "test.h"
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include "testutil.h"
|
|
#include "warnless.h"
|
|
#include "memdebug.h"
|
|
|
|
#define TEST_HANG_TIMEOUT 60 * 1000
|
|
|
|
struct Sockets {
|
|
curl_socket_t *sockets;
|
|
int count; /* number of sockets actually stored in array */
|
|
int max_count; /* max number of sockets that fit in allocated array */
|
|
};
|
|
|
|
struct ReadWriteSockets {
|
|
struct Sockets read, write;
|
|
};
|
|
|
|
/**
|
|
* Remove a file descriptor from a sockets array.
|
|
*/
|
|
static void removeFd(struct Sockets *sockets, curl_socket_t fd, int mention)
|
|
{
|
|
int i;
|
|
|
|
if(mention)
|
|
fprintf(stderr, "Remove socket fd %d\n", (int) fd);
|
|
|
|
for(i = 0; i < sockets->count; ++i) {
|
|
if(sockets->sockets[i] == fd) {
|
|
if(i < sockets->count - 1)
|
|
memmove(&sockets->sockets[i], &sockets->sockets[i + 1],
|
|
sizeof(curl_socket_t) * (sockets->count - (i + 1)));
|
|
--sockets->count;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Add a file descriptor to a sockets array.
|
|
*/
|
|
static void addFd(struct Sockets *sockets, curl_socket_t fd, const char *what)
|
|
{
|
|
/**
|
|
* To ensure we only have each file descriptor once, we remove it then add
|
|
* it again.
|
|
*/
|
|
fprintf(stderr, "Add socket fd %d for %s\n", (int) fd, what);
|
|
removeFd(sockets, fd, 0);
|
|
/*
|
|
* Allocate array storage when required.
|
|
*/
|
|
if(!sockets->sockets) {
|
|
sockets->sockets = malloc(sizeof(curl_socket_t) * 20U);
|
|
if(!sockets->sockets)
|
|
return;
|
|
sockets->max_count = 20;
|
|
}
|
|
else if(sockets->count >= sockets->max_count) {
|
|
/* this can't happen in normal cases */
|
|
fprintf(stderr, "too many file handles error\n");
|
|
exit(2);
|
|
}
|
|
/*
|
|
* Add file descriptor to array.
|
|
*/
|
|
sockets->sockets[sockets->count] = fd;
|
|
++sockets->count;
|
|
}
|
|
|
|
/**
|
|
* Callback invoked by curl to poll reading / writing of a socket.
|
|
*/
|
|
static int curlSocketCallback(CURL *easy, curl_socket_t s, int action,
|
|
void *userp, void *socketp)
|
|
{
|
|
struct ReadWriteSockets *sockets = userp;
|
|
|
|
(void)easy; /* unused */
|
|
(void)socketp; /* unused */
|
|
|
|
if(action == CURL_POLL_IN || action == CURL_POLL_INOUT)
|
|
addFd(&sockets->read, s, "read");
|
|
|
|
if(action == CURL_POLL_OUT || action == CURL_POLL_INOUT)
|
|
addFd(&sockets->write, s, "write");
|
|
|
|
if(action == CURL_POLL_REMOVE) {
|
|
removeFd(&sockets->read, s, 1);
|
|
removeFd(&sockets->write, s, 0);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* Callback invoked by curl to set a timeout.
|
|
*/
|
|
static int curlTimerCallback(CURLM *multi, long timeout_ms, void *userp)
|
|
{
|
|
struct timeval *timeout = userp;
|
|
|
|
(void)multi; /* unused */
|
|
if(timeout_ms != -1) {
|
|
*timeout = tutil_tvnow();
|
|
timeout->tv_usec += (int)timeout_ms * 1000;
|
|
}
|
|
else {
|
|
timeout->tv_sec = -1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* Check for curl completion.
|
|
*/
|
|
static int checkForCompletion(CURLM *curl, int *success)
|
|
{
|
|
int result = 0;
|
|
*success = 0;
|
|
while(1) {
|
|
int numMessages;
|
|
CURLMsg *message = curl_multi_info_read(curl, &numMessages);
|
|
if(!message)
|
|
break;
|
|
if(message->msg == CURLMSG_DONE) {
|
|
result = 1;
|
|
if(message->data.result == CURLE_OK)
|
|
*success = 1;
|
|
else
|
|
*success = 0;
|
|
}
|
|
else {
|
|
fprintf(stderr, "Got an unexpected message from curl: %i\n",
|
|
(int)message->msg);
|
|
result = 1;
|
|
*success = 0;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
static int getMicroSecondTimeout(struct timeval *timeout)
|
|
{
|
|
struct timeval now;
|
|
ssize_t result;
|
|
now = tutil_tvnow();
|
|
result = (ssize_t)((timeout->tv_sec - now.tv_sec) * 1000000 +
|
|
timeout->tv_usec - now.tv_usec);
|
|
if(result < 0)
|
|
result = 0;
|
|
|
|
return curlx_sztosi(result);
|
|
}
|
|
|
|
/**
|
|
* Update a fd_set with all of the sockets in use.
|
|
*/
|
|
static void updateFdSet(struct Sockets *sockets, fd_set* fdset,
|
|
curl_socket_t *maxFd)
|
|
{
|
|
int i;
|
|
for(i = 0; i < sockets->count; ++i) {
|
|
FD_SET(sockets->sockets[i], fdset);
|
|
if(*maxFd < sockets->sockets[i] + 1) {
|
|
*maxFd = sockets->sockets[i] + 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void notifyCurl(CURLM *curl, curl_socket_t s, int evBitmask,
|
|
const char *info)
|
|
{
|
|
int numhandles = 0;
|
|
CURLMcode result = curl_multi_socket_action(curl, s, evBitmask, &numhandles);
|
|
if(result != CURLM_OK) {
|
|
fprintf(stderr, "Curl error on %s: %i (%s)\n",
|
|
info, result, curl_multi_strerror(result));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Invoke curl when a file descriptor is set.
|
|
*/
|
|
static void checkFdSet(CURLM *curl, struct Sockets *sockets, fd_set *fdset,
|
|
int evBitmask, const char *name)
|
|
{
|
|
int i;
|
|
for(i = 0; i < sockets->count; ++i) {
|
|
if(FD_ISSET(sockets->sockets[i], fdset)) {
|
|
notifyCurl(curl, sockets->sockets[i], evBitmask, name);
|
|
}
|
|
}
|
|
}
|
|
|
|
CURLcode test(char *URL)
|
|
{
|
|
CURLcode res = CURLE_OK;
|
|
CURL *curl = NULL;
|
|
FILE *hd_src = NULL;
|
|
int hd;
|
|
struct_stat file_info;
|
|
CURLM *m = NULL;
|
|
struct ReadWriteSockets sockets = {{NULL, 0, 0}, {NULL, 0, 0}};
|
|
struct timeval timeout = {-1, 0};
|
|
int success = 0;
|
|
|
|
assert(test_argc >= 5);
|
|
|
|
start_test_timing();
|
|
|
|
if(!libtest_arg3) {
|
|
fprintf(stderr, "Usage: lib582 [url] [filename] [username]\n");
|
|
return TEST_ERR_USAGE;
|
|
}
|
|
|
|
hd_src = fopen(libtest_arg2, "rb");
|
|
if(!hd_src) {
|
|
fprintf(stderr, "fopen() failed with error: %d (%s)\n",
|
|
errno, strerror(errno));
|
|
fprintf(stderr, "Error opening file: (%s)\n", libtest_arg2);
|
|
return TEST_ERR_FOPEN;
|
|
}
|
|
|
|
/* get the file size of the local file */
|
|
hd = fstat(fileno(hd_src), &file_info);
|
|
if(hd == -1) {
|
|
/* can't open file, bail out */
|
|
fprintf(stderr, "fstat() failed with error: %d (%s)\n",
|
|
errno, strerror(errno));
|
|
fprintf(stderr, "ERROR: cannot open file (%s)\n", libtest_arg2);
|
|
fclose(hd_src);
|
|
return TEST_ERR_FSTAT;
|
|
}
|
|
fprintf(stderr, "Set to upload %d bytes\n", (int)file_info.st_size);
|
|
|
|
res_global_init(CURL_GLOBAL_ALL);
|
|
if(res) {
|
|
fclose(hd_src);
|
|
return res;
|
|
}
|
|
|
|
easy_init(curl);
|
|
|
|
/* enable uploading */
|
|
easy_setopt(curl, CURLOPT_UPLOAD, 1L);
|
|
|
|
/* specify target */
|
|
easy_setopt(curl, CURLOPT_URL, URL);
|
|
|
|
/* go verbose */
|
|
easy_setopt(curl, CURLOPT_VERBOSE, 1L);
|
|
|
|
/* now specify which file to upload */
|
|
easy_setopt(curl, CURLOPT_READDATA, hd_src);
|
|
|
|
easy_setopt(curl, CURLOPT_USERPWD, libtest_arg3);
|
|
easy_setopt(curl, CURLOPT_SSH_PUBLIC_KEYFILE, test_argv[4]);
|
|
easy_setopt(curl, CURLOPT_SSH_PRIVATE_KEYFILE, test_argv[5]);
|
|
easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
|
|
|
|
easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)file_info.st_size);
|
|
|
|
multi_init(m);
|
|
|
|
multi_setopt(m, CURLMOPT_SOCKETFUNCTION, curlSocketCallback);
|
|
multi_setopt(m, CURLMOPT_SOCKETDATA, &sockets);
|
|
|
|
multi_setopt(m, CURLMOPT_TIMERFUNCTION, curlTimerCallback);
|
|
multi_setopt(m, CURLMOPT_TIMERDATA, &timeout);
|
|
|
|
multi_add_handle(m, curl);
|
|
|
|
while(!checkForCompletion(m, &success)) {
|
|
fd_set readSet, writeSet;
|
|
curl_socket_t maxFd = 0;
|
|
struct timeval tv = {10, 0};
|
|
|
|
FD_ZERO(&readSet);
|
|
FD_ZERO(&writeSet);
|
|
updateFdSet(&sockets.read, &readSet, &maxFd);
|
|
updateFdSet(&sockets.write, &writeSet, &maxFd);
|
|
|
|
if(timeout.tv_sec != -1) {
|
|
int usTimeout = getMicroSecondTimeout(&timeout);
|
|
tv.tv_sec = usTimeout / 1000000;
|
|
tv.tv_usec = usTimeout % 1000000;
|
|
}
|
|
else if(maxFd <= 0) {
|
|
tv.tv_sec = 0;
|
|
tv.tv_usec = 100000;
|
|
}
|
|
|
|
select_test((int)maxFd, &readSet, &writeSet, NULL, &tv);
|
|
|
|
/* Check the sockets for reading / writing */
|
|
checkFdSet(m, &sockets.read, &readSet, CURL_CSELECT_IN, "read");
|
|
checkFdSet(m, &sockets.write, &writeSet, CURL_CSELECT_OUT, "write");
|
|
|
|
if(timeout.tv_sec != -1 && getMicroSecondTimeout(&timeout) == 0) {
|
|
/* Curl's timer has elapsed. */
|
|
notifyCurl(m, CURL_SOCKET_TIMEOUT, 0, "timeout");
|
|
}
|
|
|
|
abort_on_test_timeout();
|
|
}
|
|
|
|
if(!success) {
|
|
fprintf(stderr, "Error uploading file.\n");
|
|
res = TEST_ERR_MAJOR_BAD;
|
|
}
|
|
|
|
test_cleanup:
|
|
|
|
/* proper cleanup sequence - type PB */
|
|
|
|
curl_multi_remove_handle(m, curl);
|
|
curl_easy_cleanup(curl);
|
|
curl_multi_cleanup(m);
|
|
curl_global_cleanup();
|
|
|
|
/* close the local file */
|
|
fclose(hd_src);
|
|
|
|
/* free local memory */
|
|
free(sockets.read.sockets);
|
|
free(sockets.write.sockets);
|
|
|
|
return res;
|
|
}
|