postgresql/.cirrus.yml

# CI configuration file for CI utilizing cirrus-ci.org
#
# For instructions on how to enable the CI integration in a repository and
# further details, see src/tools/ci/README


env:
  # Source of images / containers
  GCP_PROJECT: pg-ci-images
  IMAGE_PROJECT: $GCP_PROJECT
  CONTAINER_REPO: us-docker.pkg.dev/${GCP_PROJECT}/ci

  # The lower depth accelerates git clone. Use a bit of depth so that
  # concurrent tasks and retrying older jobs has a chance of working.
  CIRRUS_CLONE_DEPTH: 500
  # Useful to be able to analyse what in a script takes long
  CIRRUS_LOG_TIMESTAMP: true

  CCACHE_MAXSIZE: "250M"

  # target to test, for all but windows
  CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
  CHECKFLAGS: -Otarget
  PROVE_FLAGS: --timer
  MTEST_ARGS: --print-errorlogs --no-rebuild -C build
  PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
  TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
  PG_TEST_EXTRA: kerberos ldap ssl


# What files to preserve in case tests fail
on_failure_ac: &on_failure_ac
  log_artifacts:
    paths:
      - "**/*.log"
      - "**/*.diffs"
      - "**/regress_log_*"
    type: text/plain

on_failure_meson: &on_failure_meson
  testrun_artifacts:
    paths:
      - "build*/testrun/**/*.log"
      - "build*/testrun/**/*.diffs"
      - "build*/testrun/**/regress_log_*"
    type: text/plain

  # In theory it'd be nice to upload the junit files meson generates, so that
  # cirrus will nicely annotate the commit. Unfortunately the files don't
  # contain identifieable file + line numbers right now, so the annotations
  # don't end up useful. We could probably improve on that with a some custom
  # conversion script, but ...
  meson_log_artifacts:
    path: "build*/meson-logs/*.txt"
    type: text/plain


# To avoid unnecessarily spinning up a lot of VMs / containers for entirely
# broken commits, have a minimal task that all others depend on.
task:
  name: SanityCheck

  # If a specific OS is requested, don't run the sanity check. This shortens
  # push-wait-for-ci cycle time a bit when debugging operating system specific
  # failures. Uses skip instead of only_if, as cirrus otherwise warns about
  # only_if conditions not matching.
  skip: $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:.*'

  env:
    CPUS: 4
    BUILD_JOBS: 8
    TEST_JOBS: 8
    CCACHE_DIR: ${CIRRUS_WORKING_DIR}/ccache_dir
    # no options enabled, should be small
    CCACHE_MAXSIZE: "150M"

  # Container starts up quickly, but is slower at runtime, particularly for
  # tests. Good for the briefly running sanity check.
  container:
    image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
    cpu: $CPUS

  ccache_cache:
    folder: $CCACHE_DIR

  create_user_script: |
    useradd -m postgres
    chown -R postgres:postgres .
    mkdir -p ${CCACHE_DIR}
    chown -R postgres:postgres ${CCACHE_DIR}
    echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
    su postgres -c "ulimit -l -H && ulimit -l -S"
    # Can't change container's kernel.core_pattern. Postgres user can't write
    # to / normally. Change that.
    chown root:postgres /
    chmod g+rwx /    

  configure_script: |
    su postgres <<-EOF
      meson setup \
        --buildtype=debug \
        --auto-features=disabled \
        -Dtap_tests=enabled \
        build
    EOF    
  build_script: |
    su postgres <<-EOF
      ninja -C build -j${BUILD_JOBS}
    EOF    
  upload_caches: ccache

  # Run a minimal set of tests. The main regression tests take too long for
  # this purpose. For now this is a random quick pg_regress style test, and a
  # tap test that exercises both a frontend binary and the backend.
  test_minimal_script: |
    su postgres <<-EOF
      ulimit -c unlimited
      meson test $MTEST_ARGS --num-processes ${TEST_JOBS} \
        tmp_install cube/regress pg_ctl/001_start_stop
    EOF    

  on_failure:
    <<: *on_failure_meson
    cores_script: |
      mkdir -m 770 /tmp/cores
      find / -maxdepth 1 -type f -name 'core*' -exec mv '{}' /tmp/cores/ \;
      src/tools/ci/cores_backtrace.sh linux /tmp/cores      


task:
  name: FreeBSD - 13 - Meson

  env:
    # FreeBSD on GCP is slow when running with larger number of CPUS /
    # jobs. Using one more job than cpus seems to work best.
    CPUS: 2
    BUILD_JOBS: 3
    TEST_JOBS: 3

    CCACHE_DIR: /tmp/ccache_dir
    CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
    CFLAGS: -Og -ggdb

  depends_on: SanityCheck
  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'

  compute_engine_instance:
    image_project: $IMAGE_PROJECT
    image: family/pg-ci-freebsd-13
    platform: freebsd
    cpu: $CPUS
    memory: 4G
    disk: 50

  sysinfo_script: |
    id
    uname -a
    ulimit -a -H && ulimit -a -S
    export    

  ccache_cache:
    folder: $CCACHE_DIR
  # Workaround around performance issues due to 32KB block size
  repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
  create_user_script: |
    pw useradd postgres
    chown -R postgres:postgres .
    mkdir -p ${CCACHE_DIR}
    chown -R postgres:postgres ${CCACHE_DIR}    
  setup_core_files_script: |
    mkdir -m 770 /tmp/cores
    chown root:postgres /tmp/cores
    sysctl kern.corefile='/tmp/cores/%N.%P.core'    
  setup_additional_packages_script: |
    #pkg install -y ...    

  # NB: Intentionally build without -Dllvm. The freebsd image size is already
  # large enough to make VM startup slow, and even without llvm freebsd
  # already takes longer than other platforms except for windows.
  configure_script: |
    su postgres <<-EOF
      meson setup \
        --buildtype=debug \
        -Dcassert=true -Dssl=openssl -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \
        -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
        -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
        build
    EOF    
  build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
  upload_caches: ccache

  test_world_script: |
    su postgres <<-EOF
      ulimit -c unlimited
      meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
    EOF    

  on_failure:
    <<: *on_failure_meson
    cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores


# configure feature flags, shared between the task running the linux tests and
# the CompilerWarnings task
LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >-
  --with-gssapi
  --with-icu
  --with-ldap
  --with-libxml
  --with-libxslt
  --with-llvm
  --with-lz4
  --with-pam
  --with-perl
  --with-python
  --with-selinux
  --with-ssl=openssl
  --with-systemd
  --with-tcl --with-tclconfig=/usr/lib/tcl8.6/
  --with-uuid=ossp
  --with-zstd  

LINUX_MESON_FEATURES: &LINUX_MESON_FEATURES >-
  -Dllvm=enabled
  -Dssl=openssl
  -Duuid=e2fs  


task:
  env:
    CPUS: 4
    BUILD_JOBS: 4
    TEST_JOBS: 8 # experimentally derived to be a decent choice

    CCACHE_DIR: /tmp/ccache_dir
    DEBUGINFOD_URLS: "https://debuginfod.debian.net"

    # Enable a reasonable set of sanitizers. Use the linux task for that, as
    # it one of the fastest tasks (without sanitizers). Also several of the
    # sanitizers work best on linux.
    #
    # The overhead of alignment sanitizer is low, undefined behaviour has
    # moderate overhead. Test alignment sanitizer in the meson task, as it
    # does both 32 and 64 bit builds and is thus more likely to expose
    # alignment bugs.
    #
    # Address sanitizer in contrast is somewhat expensive. Enable it in the
    # autoconf task, as the meson task tests both 32 and 64bit.
    #
    # disable_coredump=0, abort_on_error=1: for useful backtraces in case of crashes
    # print_stacktraces=1,verbosity=2, duh
    # detect_leaks=0: too many uninteresting leak errors in short-lived binaries
    UBSAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:verbosity=2
    ASAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:detect_leaks=0

    # SANITIZER_FLAGS is set in the tasks below
    CFLAGS: -Og -ggdb -fno-sanitize-recover=all $SANITIZER_FLAGS
    CXXFLAGS: $CFLAGS
    LDFLAGS: $SANITIZER_FLAGS
    CC: ccache gcc
    CXX: ccache g++

    LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
    LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES

  depends_on: SanityCheck
  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'

  compute_engine_instance:
    image_project: $IMAGE_PROJECT
    image: family/pg-ci-bullseye
    platform: linux
    cpu: $CPUS
    memory: 4G

  ccache_cache:
    folder: ${CCACHE_DIR}

  sysinfo_script: |
    id
    uname -a
    cat /proc/cmdline
    ulimit -a -H && ulimit -a -S
    export    
  create_user_script: |
    useradd -m postgres
    chown -R postgres:postgres .
    mkdir -p ${CCACHE_DIR}
    chown -R postgres:postgres ${CCACHE_DIR}
    echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
    su postgres -c "ulimit -l -H && ulimit -l -S"    
  setup_core_files_script: |
    mkdir -m 770 /tmp/cores
    chown root:postgres /tmp/cores
    sysctl kernel.core_pattern='/tmp/cores/%e-%s-%p.core'    
  setup_additional_packages_script: |
    #apt-get update
    #DEBIAN_FRONTEND=noninteractive apt-get -y install ...    

  matrix:
    - name: Linux - Debian Bullseye - Autoconf

      env:
        SANITIZER_FLAGS: -fsanitize=address

      configure_script: |
        su postgres <<-EOF
          ./configure \
            --enable-cassert --enable-debug --enable-tap-tests \
            --enable-nls \
            \
            ${LINUX_CONFIGURE_FEATURES} \
            \
            CLANG="ccache clang"
        EOF        
      build_script: su postgres -c "make -s -j${BUILD_JOBS} world-bin"
      upload_caches: ccache

      test_world_script: |
        su postgres <<-EOF
          ulimit -c unlimited # default is 0
          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
        EOF        

      on_failure:
        <<: *on_failure_ac

    - name: Linux - Debian Bullseye - Meson

      env:
        CCACHE_MAXSIZE: "400M" # tests two different builds
        SANITIZER_FLAGS: -fsanitize=alignment,undefined

      configure_script: |
        su postgres <<-EOF
          meson setup \
            --buildtype=debug \
            -Dcassert=true \
            ${LINUX_MESON_FEATURES} \
            -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
            build
        EOF        

      # Also build & test in a 32bit build - it's gotten rare to test that
      # locally.
      configure_32_script: |
        su postgres <<-EOF
          export CC='ccache gcc -m32'
          meson setup \
            --buildtype=debug \
            -Dcassert=true \
            ${LINUX_MESON_FEATURES} \
            -Dllvm=disabled \
            --pkg-config-path /usr/lib/i386-linux-gnu/pkgconfig/ \
            -DPERL=perl5.32-i386-linux-gnu \
            -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
            build-32
        EOF        

      build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
      build_32_script: su postgres -c 'ninja -C build-32 -j${BUILD_JOBS}'

      upload_caches: ccache

      test_world_script: |
        su postgres <<-EOF
          ulimit -c unlimited
          meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
        EOF
        # so that we don't upload 64bit logs if 32bit fails
        rm -rf build/        

      # There's currently no coverage of icu with LANG=C in the buildfarm. We
      # can easily provide some here by running one of the sets of tests that
      # way. Newer versions of python insist on changing the LC_CTYPE away
      # from C, prevent that with PYTHONCOERCECLOCALE.
      test_world_32_script: |
        su postgres <<-EOF
          ulimit -c unlimited
          PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
        EOF        

      on_failure:
        <<: *on_failure_meson

  on_failure:
    cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores


task:
  name: macOS - Monterey - Meson

  env:
    CPUS: 12 # always get that much for cirrusci macOS instances
    BUILD_JOBS: $CPUS
    # Test performance regresses noticably when using all cores. 8 seems to
    # work OK. See
    # https://postgr.es/m/20220927040208.l3shfcidovpzqxfh%40awork3.anarazel.de
    TEST_JOBS: 8

    CIRRUS_WORKING_DIR: ${HOME}/pgsql/
    CCACHE_DIR: ${HOME}/ccache
    HOMEBREW_CACHE: ${HOME}/homebrew-cache
    PERL5LIB: ${HOME}/perl5/lib/perl5

    CC: ccache cc
    CXX: ccache c++
    CPPFLAGS: -DRANDOMIZE_ALLOCATED_MEMORY
    CFLAGS: -Og -ggdb
    CXXFLAGS: -Og -ggdb

  depends_on: SanityCheck
  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'

  osx_instance:
    image: monterey-base

  sysinfo_script: |
    id
    uname -a
    ulimit -a -H && ulimit -a -S
    export    

  setup_core_files_script:
    - mkdir ${HOME}/cores
    - sudo sysctl kern.corefile="${HOME}/cores/core.%P"

  perl_cache:
    folder: ~/perl5
  cpan_install_script:
    - perl -mIPC::Run -e 1 || cpan -T IPC::Run
    - perl -mIO::Pty -e 1 || cpan -T IO::Pty
  upload_caches: perl


  # XXX: Could we instead install homebrew into a cached directory? The
  # homebrew installation takes a good bit of time every time, even if the
  # packages do not need to be downloaded.
  homebrew_cache:
    folder: $HOMEBREW_CACHE
  setup_additional_packages_script: |
    brew install \
      ccache \
      icu4c \
      krb5 \
      llvm \
      lz4 \
      make \
      meson \
      openldap \
      openssl \
      python \
      tcl-tk \
      zstd

    brew cleanup -s # to reduce cache size    
  upload_caches: homebrew

  ccache_cache:
    folder: $CCACHE_DIR
  configure_script: |
    brewpath="/usr/local"
    PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"

    for pkg in icu4c krb5 openldap openssl zstd ; do
      pkgpath="${brewpath}/opt/${pkg}"
      PKG_CONFIG_PATH="${pkgpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
      PATH="${pkgpath}/bin:${pkgpath}/sbin:$PATH"
    done

    export PKG_CONFIG_PATH PATH

    meson setup \
      --buildtype=debug \
      -Dextra_include_dirs=${brewpath}/include \
      -Dextra_lib_dirs=${brewpath}/lib \
      -Dcassert=true \
      -Dssl=openssl -Duuid=e2fs -Ddtrace=auto \
      -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
      build    

  build_script: ninja -C build -j${BUILD_JOBS}
  upload_caches: ccache

  test_world_script: |
    ulimit -c unlimited # default is 0
    ulimit -n 1024 # default is 256, pretty low
    meson test $MTEST_ARGS --num-processes ${TEST_JOBS}    

  on_failure:
    <<: *on_failure_meson
    cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"


WINDOWS_ENVIRONMENT_BASE: &WINDOWS_ENVIRONMENT_BASE
  env:
    # Half the allowed per-user CPU cores
    CPUS: 4

    # The default cirrus working dir is in a directory msbuild complains about
    CIRRUS_WORKING_DIR: "c:/cirrus"
    # git's tar doesn't deal with drive letters, see
    # https://postgr.es/m/b6782dc3-a7b0-ed56-175f-f8f54cb08d67%40dunslane.net
    TAR: "c:/windows/system32/tar.exe"
    # Avoids port conflicts between concurrent tap test runs
    PG_TEST_USE_UNIX_SOCKETS: 1
    PG_REGRESS_SOCK_DIR: "c:/cirrus/"

  sysinfo_script: |
    chcp
    systeminfo
    powershell -Command get-psdrive -psprovider filesystem
    set    


task:
  name: Windows - Server 2019, VS 2019 - Meson & ninja
  << : *WINDOWS_ENVIRONMENT_BASE

  env:
    TEST_JOBS: 8 # wild guess, data based value welcome

    # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That
    # prevents crash reporting from working unless binaries do SetErrorMode()
    # themselves. Furthermore, it appears that either python or, more likely,
    # the C runtime has a bug where SEM_NOGPFAULTERRORBOX can very
    # occasionally *trigger* a crash on process exit - which is hard to debug,
    # given that it explicitly prevents crash dumps from working...
    # 0x8001 is SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX
    CIRRUS_WINDOWS_ERROR_MODE: 0x8001

  depends_on: SanityCheck
  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'

  windows_container:
    image: $CONTAINER_REPO/windows_ci_vs_2019:latest
    cpu: $CPUS
    memory: 4G

  setup_additional_packages_script: |
    REM choco install -y --no-progress ...    

  # Use /DEBUG:FASTLINK to avoid high memory usage during linking
  configure_script: |
    vcvarsall x64
    meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Db_pch=true -Dssl=openssl -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build    

  build_script: |
    vcvarsall x64
    ninja -C build    

  check_world_script: |
    vcvarsall x64
    meson test %MTEST_ARGS% --num-processes %TEST_JOBS%    

  on_failure:
    <<: *on_failure_meson
    crashlog_artifacts:
      path: "crashlog-*.txt"
      type: text/plain


task:
  << : *WINDOWS_ENVIRONMENT_BASE
  name: Windows - Server 2019, MinGW64 - Meson

  # due to resource constraints we don't run this task by default for now
  trigger_type: manual
  # worth using only_if despite being manual, otherwise this task will show up
  # when e.g. ci-os-only: linux is used.
  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*'
  # otherwise it'll be sorted before other tasks
  depends_on: SanityCheck

  windows_container:
    image: $CONTAINER_REPO/windows_ci_mingw64:latest
    cpu: $CPUS
    memory: 4G

  env:
    TEST_JOBS: 4 # higher concurrency causes occasional failures
    CCACHE_DIR: C:/msys64/ccache
    CCACHE_MAXSIZE: "500M"
    CCACHE_SLOPPINESS: pch_defines,time_macros
    CCACHE_DEPEND: 1
    # for some reason mingw plpython cannot find it's installation without this
    PYTHONHOME: C:/msys64/ucrt64
    # prevents MSYS bash from resetting error mode
    MSYS: winjitdebug
    # Start bash in current working directory
    CHERE_INVOKING: 1
    BASH: C:\msys64\usr\bin\bash.exe -l

  ccache_cache:
    folder: ${CCACHE_DIR}

  setup_additional_packages_script: |
    REM C:\msys64\usr\bin\pacman.exe -S --noconfirm ...    

  mingw_info_script: |
    %BASH% -c "where gcc"
    %BASH% -c "gcc --version"
    %BASH% -c "where perl"
    %BASH% -c "perl --version"    

  # disable -Dnls as the number of files it creates cause a noticable slowdown
  configure_script: |
    %BASH% -c "meson setup -Ddebug=true -Doptimization=g -Dcassert=true -Db_pch=true -Dnls=disabled -DTAR=%TAR% build"    

  build_script: |
    %BASH% -c "ninja -C build"    

  upload_caches: ccache

  test_world_script: |
    %BASH% -c "meson test %MTEST_ARGS% --num-processes %TEST_JOBS%"    

  on_failure:
    <<: *on_failure_meson
    crashlog_artifacts:
      path: "crashlog-*.txt"
      type: text/plain


task:
  name: CompilerWarnings

  # To limit unnecessary work only run this once the SanityCheck
  # succeeds. This is particularly important for this task as we intentionally
  # use always: to continue after failures. Task that did not run count as a
  # success, so we need to recheck SanityChecks's condition here ...
  depends_on: SanityCheck
  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*'

  env:
    CPUS: 4
    BUILD_JOBS: 4

    # Use larger ccache cache, as this task compiles with multiple compilers /
    # flag combinations
    CCACHE_MAXSIZE: "1GB"
    CCACHE_DIR: "/tmp/ccache_dir"

    LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
    LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES

  container:
    image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
    cpu: $CPUS

  sysinfo_script: |
    id
    uname -a
    cat /proc/cmdline
    ulimit -a -H && ulimit -a -S
    gcc -v
    clang -v
    export    

  ccache_cache:
    folder: $CCACHE_DIR

  setup_additional_packages_script: |
    #apt-get update
    #DEBIAN_FRONTEND=noninteractive apt-get -y install ...    

  ###
  # Test that code can be built with gcc/clang without warnings
  ###

  setup_script: echo "COPT=-Werror" > src/Makefile.custom

  # Trace probes have a history of getting accidentally broken. Use the
  # different compilers to build with different combinations of dtrace on/off
  # and cassert on/off.

  # gcc, cassert off, dtrace on
  always:
    gcc_warning_script: |
      time ./configure \
        --cache gcc.cache \
        --enable-dtrace \
        ${LINUX_CONFIGURE_FEATURES} \
        CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
      make -s -j${BUILD_JOBS} clean
      time make -s -j${BUILD_JOBS} world-bin      

  # gcc, cassert on, dtrace off
  always:
    gcc_a_warning_script: |
      time ./configure \
        --cache gcc.cache \
        --enable-cassert \
        ${LINUX_CONFIGURE_FEATURES} \
        CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
      make -s -j${BUILD_JOBS} clean
      time make -s -j${BUILD_JOBS} world-bin      

  # clang, cassert off, dtrace off
  always:
    clang_warning_script: |
      time ./configure \
        --cache clang.cache \
        ${LINUX_CONFIGURE_FEATURES} \
        CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
      make -s -j${BUILD_JOBS} clean
      time make -s -j${BUILD_JOBS} world-bin      

  # clang, cassert on, dtrace on
  always:
    clang_a_warning_script: |
      time ./configure \
        --cache clang.cache \
        --enable-cassert \
        --enable-dtrace \
        ${LINUX_CONFIGURE_FEATURES} \
        CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
      make -s -j${BUILD_JOBS} clean
      time make -s -j${BUILD_JOBS} world-bin      

  # cross-compile to windows
  always:
    mingw_cross_warning_script: |
      time ./configure \
        --host=x86_64-w64-mingw32 \
        --enable-cassert \
        CC="ccache x86_64-w64-mingw32-gcc" \
        CXX="ccache x86_64-w64-mingw32-g++"
      make -s -j${BUILD_JOBS} clean
      time make -s -j${BUILD_JOBS} world-bin      

  ###
  # Verify docs can be built
  ###
  # XXX: Only do this if there have been changes in doc/ since last build
  always:
    docs_build_script: |
      time ./configure \
        --cache gcc.cache \
        CC="ccache gcc" \
        CXX="ccache g++" \
        CLANG="ccache clang"
      make -s -j${BUILD_JOBS} clean
      time make -s -j${BUILD_JOBS} -C doc      

  ###
  # Verify headerscheck / cpluspluscheck succeed
  #
  # - Don't use ccache, the files are uncacheable, polluting ccache's
  #   cache
  # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
  # - XXX have to disable ICU to avoid errors:
  #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
  ###
  always:
    headers_headerscheck_script: |
      time ./configure \
        ${LINUX_CONFIGURE_FEATURES} \
        --without-icu \
        --quiet \
        CC="gcc" CXX"=g++" CLANG="clang"
      make -s -j${BUILD_JOBS} clean
      time make -s headerscheck EXTRAFLAGS='-fmax-errors=10'      
    headers_cpluspluscheck_script: |
      time make -s cpluspluscheck EXTRAFLAGS='-fmax-errors=10'      

  always:
    upload_caches: ccache