mirror/curl
2
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2024-11-27 05:50:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

autotools: add support for 'unity' builds, enable in CI

Browse Source 
Implement the "unity" builds as known from CMake, but for autotools.
It's limited to `lib` and `src` (CMake also supports it in `tests`).

Enable with: `--enable-unity` (disabled by default)

Unity builds speed up builds significantly. Cygwin and Windows builds in
particular, but the effect is noticeable on most systems. It also allows
discovering unity issues with autotools, benefitting also CMake when
building the same combination. In CI it makes turnaround times quicker.

This closes build performance with CMake. autotools still lags behind
because it builds shared and static libcurl in two, separate passes.
CMake does it in one. Manpage compilation isn't batched, it is in CMake.
After unity and test bundle support the slowest parts of the build are
the configuration phase (which is effectively a tedious, non-parallel,
compilation and/or linking of 300+ tiny programs. The next bottleneck
is compiling individual examples and finally test servers (only slow
with autotools).

The autotools implementation is slightly less efficient than CMake,
because 3 sources are permanently excluded while in CMake this isn't
necessary and solved more efficiently while building libtests. There is
also no 'unity' support for tests, making them a less efficient also.

Enable it in CI for most `configure` jobs. Except in GHA/dist (though
it works fine there too), to use the default config there. Also skip for
the Linux AWC-LC job where it made builds time a few seconds longer
(reason undiscovered.)

Autotools test suite builds compared between master -> `--enable-unity`:
- GHA/Linux: 32s -> 12s
  https://github.com/curl/curl/actions/runs/10705668823/job/29681617374
  https://github.com/curl/curl/actions/runs/10742978889/job/29796766297
- GHA/macOS: 37s -> 10s
  https://github.com/curl/curl/actions/runs/10705668813/job/29681632885
  https://github.com/curl/curl/actions/runs/10742978699/job/29796768875
- GHA/FreeBSD: 15m25 -> 10m58 (full workflow time, ~qemu)
  https://github.com/curl/curl/actions/runs/10705668811/job/29681607915
  https://github.com/curl/curl/actions/runs/10742978937/job/29796766115
- GHA/Cygwin: 3m32 -> 1m21
  https://github.com/curl/curl/actions/runs/10705668809/job/29681609965
  https://github.com/curl/curl/actions/runs/10742978645/job/29796756933
- GHA/MSYS2: 2m42 -> 50s
  https://github.com/curl/curl/actions/runs/10705668808/job/29681621166
  https://github.com/curl/curl/actions/runs/10742978662/job/29799739289
- GHA/mingw-w64: 5m32 -> 1m23
  https://github.com/curl/curl/actions/runs/10705668808/job/29681628787
  https://github.com/curl/curl/actions/runs/10742978662/job/29799741568

Closes #14815
This commit is contained in:
Viktor Szakats 2024-09-06 10:26:06 +02:00
parent 45202cbba4
commit 60c3d04465
No known key found for this signature in database
GPG Key ID: B5ABD165E2AEF201
16 changed files with 167 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.circleci/config.yml
View File

@ -33,7 +33,7 @@ commands:
- run:
command: |
autoreconf -fi
./configure --disable-dependency-tracking --enable-warnings --enable-werror --with-openssl \
./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror --with-openssl \
|| { tail -1000 config.log; false; }
configure-openssl-no-verbose:
@ -41,7 +41,7 @@ commands:
- run:
command: |
autoreconf -fi
./configure --disable-dependency-tracking --disable-verbose --enable-werror --with-openssl \
./configure --disable-dependency-tracking --enable-unity --disable-verbose --enable-werror --with-openssl \
|| { tail -1000 config.log; false; }
configure-no-proxy:
@ -49,7 +49,7 @@ commands:
- run:
command: |
autoreconf -fi
./configure --disable-dependency-tracking --disable-proxy --enable-werror --with-openssl \
./configure --disable-dependency-tracking --enable-unity --disable-proxy --enable-werror --with-openssl \
|| { tail -1000 config.log; false; }
install-cares:
@ -110,7 +110,7 @@ commands:
- run:
command: |
autoreconf -fi
./configure --disable-dependency-tracking --enable-warnings --enable-werror --with-openssl --enable-ares \
./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror --with-openssl --enable-ares \
|| { tail -1000 config.log; false; }
configure-wolfssh:
@ -118,7 +118,7 @@ commands:
- run:
command: |
autoreconf -fi
LDFLAGS="-Wl,-rpath,$HOME/wssh/lib" ./configure --disable-dependency-tracking --enable-warnings --enable-werror --with-wolfssl=$HOME/wssl --with-wolfssh=$HOME/wssh \
LDFLAGS="-Wl,-rpath,$HOME/wssh/lib" ./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror --with-wolfssl=$HOME/wssl --with-wolfssh=$HOME/wssh \
|| { tail -1000 config.log; false; }
configure-cares-debug:
@ -126,7 +126,7 @@ commands:
- run:
command: |
autoreconf -fi
./configure --disable-dependency-tracking --enable-debug --enable-werror --with-openssl --enable-ares \
./configure --disable-dependency-tracking --enable-unity --enable-debug --enable-werror --with-openssl --enable-ares \
|| { tail -1000 config.log; false; }
build:

2
.github/workflows/cygwin.yml vendored
View File

@ -81,7 +81,7 @@ jobs:
timeout-minutes: 5
run: |
PATH="/usr/bin:$(cygpath "${SYSTEMROOT}")/System32"
mkdir bld && cd bld && ../configure --enable-warnings --enable-werror \
mkdir bld && cd bld && ../configure --enable-unity --enable-warnings --enable-werror \
--prefix="${HOME}"/install \
--enable-websockets \
--with-openssl \

2
.github/workflows/http3-linux.yml vendored
View File

@ -451,7 +451,7 @@ jobs:
- run: autoreconf -fi
name: 'autoreconf'
- run: ./configure --disable-dependency-tracking ${{ matrix.build.configure }}
- run: ./configure --disable-dependency-tracking --enable-unity ${{ matrix.build.configure }}
name: 'configure'
- run: make V=1

4
.github/workflows/linux-old.yml vendored
View File

@ -113,7 +113,9 @@ jobs:
run: |
mkdir bld-am
cd bld-am
../configure --disable-dependency-tracking --enable-warnings --enable-werror --with-openssl --enable-ares --with-libssh --with-zstd --with-gssapi --prefix="$PWD"/../install-am
../configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror \
--with-openssl --enable-ares --with-libssh --with-zstd --with-gssapi \
--prefix="$PWD"/../install-am
- name: 'autoconf build'
run: |

8
.github/workflows/linux.yml vendored
View File

@ -496,7 +496,10 @@ jobs:
if: ${{ matrix.build.configure }}
name: 'autoreconf'
- run: ${{ matrix.build.configure-prefix }} ./configure --disable-dependency-tracking --enable-warnings --enable-werror ${{ matrix.build.configure }}
- run: |
${{ matrix.build.configure-prefix }} \
./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror \
${{ matrix.build.configure }}
if: ${{ matrix.build.configure }}
name: 'configure (autotools)'
@ -555,8 +558,7 @@ jobs:
- if: contains(matrix.build.install_steps, 'pytest')
# run for `tests/http` directory, so pytest does not pick up any other
# packages we might have built here
run:
pytest -v tests/http
run: pytest -v tests/http
name: 'run pytest'
env:
TFLAGS: "${{ matrix.build.tflags }}"

4
.github/workflows/linux32.yml vendored
View File

@ -69,7 +69,9 @@ jobs:
- run: autoreconf -fi
name: 'autoreconf'
- run: ./configure --disable-dependency-tracking --enable-warnings --enable-werror ${{ matrix.build.configure }}
- run: |
./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror \
${{ matrix.build.configure }}
name: 'configure'
- run: make V=1

4
.github/workflows/macos.yml vendored
View File

@ -203,7 +203,7 @@ jobs:
options+=" --with-sysroot=$(xcrun --sdk macosx --show-sdk-path 2>/dev/null)"
CFLAGS+=" --sysroot=$(xcrun --sdk macosx --show-sdk-path 2>/dev/null)"
fi
mkdir bld && cd bld && ../configure --enable-warnings --enable-werror \
mkdir bld && cd bld && ../configure --enable-unity --enable-warnings --enable-werror \
--disable-dependency-tracking \
--with-libpsl=$(brew --prefix libpsl) \
${{ matrix.configure }} ${options}
@ -605,7 +605,7 @@ jobs:
[ '${{ matrix.config }}' = 'SecureTransport' ] && options+=' --with-secure-transport'
CFLAGS+=' -mmacosx-version-min=${{ matrix.macos-version-min }}'
# would pick up nghttp2, libidn2, but libssh2 is disabled by default
mkdir bld && cd bld && ../configure --enable-warnings --enable-werror \
mkdir bld && cd bld && ../configure --enable-unity --enable-warnings --enable-werror \
--disable-dependency-tracking \
--disable-docs --disable-manual \
--without-nghttp2 --without-libidn2 \

4
.github/workflows/non-native.yml vendored
View File

@ -136,7 +136,7 @@ jobs:
pkgconf brotli openldap26-client libidn2 libnghttp2 nghttp2 stunnel py311-impacket
autoreconf -fi
export CC='${{ matrix.compiler }}'
mkdir bld && cd bld && ../configure --enable-debug --enable-warnings --enable-werror \
mkdir bld && cd bld && ../configure --enable-unity --enable-debug --enable-warnings --enable-werror \
--prefix="${HOME}"/install \
--enable-websockets \
--with-openssl \
@ -198,7 +198,7 @@ jobs:
run: |
ln -s /usr/bin/gcpp /usr/bin/cpp # Some tests expect `cpp`, which is named `gcpp` in this env.
autoreconf -fi
mkdir bld && cd bld && ../configure --enable-debug --enable-warnings --enable-werror \
mkdir bld && cd bld && ../configure --enable-unity --enable-debug --enable-warnings --enable-werror \
--prefix="${HOME}"/install \
--enable-websockets \
--with-openssl \

4
.github/workflows/torture.yml vendored
View File

@ -74,7 +74,9 @@ jobs:
- run: autoreconf -fi
name: 'autoreconf'
- run: ./configure --disable-dependency-tracking --enable-warnings --enable-werror ${{ matrix.build.configure }}
- run: |
./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror \
${{ matrix.build.configure }}
name: 'configure'
- run: make V=1

4
.github/workflows/windows.yml vendored
View File

@ -103,7 +103,7 @@ jobs:
if: ${{ matrix.build == 'autotools' }}
timeout-minutes: 5
run: |
mkdir bld && cd bld && ../configure --enable-warnings --enable-werror \
mkdir bld && cd bld && ../configure --enable-unity --enable-warnings --enable-werror \
--prefix="${HOME}"/install \
--enable-websockets \
--with-openssl \
@ -415,7 +415,7 @@ jobs:
- name: 'autotools configure'
if: ${{ matrix.build == 'autotools' }}
run: |
mkdir bld && cd bld && ../configure --enable-warnings --enable-werror \
mkdir bld && cd bld && ../configure --enable-unity --enable-warnings --enable-werror \
--host=${TRIPLET} \
--with-schannel --with-winidn \
--without-libpsl \

2
.github/workflows/wolfssl.yml vendored
View File

@ -84,7 +84,7 @@ jobs:
- run: autoreconf -fi
name: 'autoreconf'
- run: ./configure --disable-dependency-tracking --enable-warnings --enable-werror ${{ matrix.build.configure }}
- run: ./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror ${{ matrix.build.configure }}
name: 'configure'
- run: make V=1

23
configure.ac
View File

@ -605,6 +605,29 @@ if test "$curl_cv_native_windows" = "yes"; then
[AC_MSG_ERROR([windres not found in PATH. Windows builds require windres. Cannot continue.])])
fi
dnl ----------------------------------------
dnl whether use "unity" mode for lib and src
dnl ----------------------------------------
want_unity='no'
AC_MSG_CHECKING([whether to build libcurl and curl in "unity" mode])
AC_ARG_ENABLE(unity,
AS_HELP_STRING([--enable-unity],[Enable unity mode])
AS_HELP_STRING([--disable-unity],[Disable unity (default)]),
[ case "$enableval" in
yes)
want_unity='yes'
AC_MSG_RESULT([yes])
;;
*)
AC_MSG_RESULT([no])
;;
esac ],
AC_MSG_RESULT([no])
)
AM_CONDITIONAL([USE_UNITY], [test "$want_unity" = 'yes'])
dnl ************************************************************
dnl switch off particular protocols
dnl

19
lib/Makefile.am
View File

@ -70,8 +70,27 @@ AM_CFLAGS =
# Makefile.inc provides the CSOURCES and HHEADERS defines
include Makefile.inc
if USE_UNITY
# Keep these separate to avoid duplicate definitions when linking libtests
# in static mode.
curl_EXCLUDE = curl_threads.c timediff.c warnless.c
if DEBUGBUILD
# We must compile these sources separately to avoid memdebug.h redefinitions
# applying to them.
curl_EXCLUDE += memdebug.c curl_multibyte.c
endif
libcurl_unity.c: $(top_srcdir)/scripts/mk-unity.pl $(CSOURCES)
@PERL@ $(top_srcdir)/scripts/mk-unity.pl $(srcdir) $(CSOURCES) --exclude $(curl_EXCLUDE) > libcurl_unity.c
nodist_libcurl_la_SOURCES = libcurl_unity.c
libcurl_la_SOURCES = $(curl_EXCLUDE)
nodist_libcurlu_la_SOURCES = libcurl_unity.c
libcurlu_la_SOURCES = $(curl_EXCLUDE)
CLEANFILES = libcurl_unity.c
else
libcurl_la_SOURCES = $(CSOURCES) $(HHEADERS)
libcurlu_la_SOURCES = $(CSOURCES) $(HHEADERS)
endif
libcurl_la_CPPFLAGS_EXTRA =
libcurl_la_LDFLAGS_EXTRA =

2
scripts/Makefile.am
View File

@ -23,7 +23,7 @@
###########################################################################
EXTRA_DIST = coverage.sh completion.pl firefox-db2pem.sh checksrc.pl \
mk-ca-bundle.pl schemetable.c cd2nroff nroff2cd cdall cd2cd managen \
mk-ca-bundle.pl mk-unity.pl schemetable.c cd2nroff nroff2cd cdall cd2cd managen \
dmaketgz maketgz release-tools.sh verify-release cmakelint.sh
ZSH_FUNCTIONS_DIR = @ZSH_FUNCTIONS_DIR@

63
scripts/mk-unity.pl Executable file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env perl
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) Viktor Szakats
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
# SPDX-License-Identifier: curl
#
###########################################################################
# Helper script for "unity"-like support in autotools, to generate the umbrella
# C source that includes the individual source files. Reads Makefile.inc and
# accepts the variable name containing all the source files to include. Also
# allow a list of exceptions that are to be excluded from the generated file.
use strict;
use warnings;
if(!@ARGV) {
die "Usage: $0 [<c-sources>] [--exclude <exclude-c-sources>]\n";
}
# Specific sources to exclude or add as an extra source file
my @src;
my %exclude;
my $in_exclude = 0;
foreach my $src (@ARGV) {
if($in_exclude) {
$exclude{$src} = 1;
}
elsif($src eq "--exclude") {
$in_exclude = 1;
}
else {
push @src, $src;
}
}
print <<HEADER
/* !checksrc! disable COPYRIGHT all */
HEADER
;
foreach my $src (@src) {
if($src =~ /\.c$/g && !exists $exclude{$src}) {
print "#include \"$src\"\n";
}
}

33
src/Makefile.am
View File

@ -60,8 +60,30 @@ endif
include Makefile.inc
CLEANFILES =
if USE_UNITY
curl_EXCLUDE =
if DEBUGBUILD
# We must compile this source separately to avoid memdebug.h redefinitions
# applying to them.
curl_EXCLUDE += ../lib/curl_multibyte.c
endif
if USE_CPPFLAG_CURL_STATICLIB
curl_CURLX = $(CURLTOOL_LIBCURL_CFILES)
else
curl_CURLX = $(CURLX_CFILES)
endif
curltool_unity.c: $(top_srcdir)/scripts/mk-unity.pl $(CURL_CFILES) $(curl_CURLX)
@PERL@ $(top_srcdir)/scripts/mk-unity.pl $(srcdir) $(CURL_CFILES) $(curl_CURLX) --exclude $(curl_EXCLUDE) > curltool_unity.c
nodist_curl_SOURCES = curltool_unity.c
curl_SOURCES = $(curl_EXCLUDE)
CLEANFILES += curltool_unity.c
else
# CURL_FILES comes from Makefile.inc
curl_SOURCES = $(CURL_FILES)
endif
if HAVE_WINDRES
curl_SOURCES += $(CURL_RCFILES)
$(CURL_RCFILES): tool_version.h
@ -84,8 +106,17 @@ libcurltool_la_CPPFLAGS = $(AM_CPPFLAGS) \
-DCURL_STATICLIB -DUNITTESTS
libcurltool_la_CFLAGS =
libcurltool_la_LDFLAGS = -static $(LINKFLAGS)
if USE_UNITY
libcurltool_unity.c: $(top_srcdir)/scripts/mk-unity.pl $(CURL_CFILES) $(CURLTOOL_LIBCURL_CFILES)
@PERL@ $(top_srcdir)/scripts/mk-unity.pl $(srcdir) $(CURL_CFILES) $(CURLTOOL_LIBCURL_CFILES) --exclude $(curl_EXCLUDE) > libcurltool_unity.c
nodist_libcurltool_la_SOURCES = libcurltool_unity.c
libcurltool_la_SOURCES = $(curl_EXCLUDE)
CLEANFILES += libcurltool_unity.c
else
libcurltool_la_SOURCES = $(CURL_FILES)
endif
endif
# Use absolute directory to disable VPATH
ASCIIPAGE=$(top_builddir)/docs/cmdline-opts/curl.txt
@ -127,7 +158,7 @@ $(HUGE):
echo '#include "tool_hugehelp.h"' >> $(HUGE)
endif
CLEANFILES = $(HUGE)
CLEANFILES += $(HUGE)
CA_EMBED_CSOURCE = tool_ca_embed.c
CURL_CFILES += $(CA_EMBED_CSOURCE)