From e8a007de0de526972ecde27ccabe3564be7722f4 Mon Sep 17 00:00:00 2001
From: Viktor Szakats <commit@vsz.me>
Date: Sat, 12 Oct 2024 10:45:31 +0200
Subject: [PATCH] GHA: optimize test prereq steps

- Linux: move test and pytest prereqs right before test run.
  - returns build phase results faster.
  - allows skipping steps for jobs that don't need them.
  - makes dependencies more transparent.
- sync prereq install step names.
- use `tests/requirements.txt` more.

Closes #15275
---
 .github/workflows/http3-linux.yml | 64 ++++++++++++++------------
 .github/workflows/linux.yml       | 76 +++++++++++++++++--------------
 .github/workflows/linux32.yml     |  5 +-
 .github/workflows/macos.yml       |  8 ++--
 .github/workflows/torture.yml     |  5 +-
 .github/workflows/windows.yml     |  8 ++--
 6 files changed, 92 insertions(+), 74 deletions(-)

diff --git a/.github/workflows/http3-linux.yml b/.github/workflows/http3-linux.yml
index 79ee25c80d..a0a7e32166 100644
--- a/.github/workflows/http3-linux.yml
+++ b/.github/workflows/http3-linux.yml
@@ -157,7 +157,7 @@ jobs:
         run: |
           echo 'needs-build=true' >> $GITHUB_OUTPUT
 
-      - name: install build prerequisites
+      - name: install build prereqs
         if: steps.settings.outputs.needs-build == 'true'
         run: |
           sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
@@ -330,7 +330,7 @@ jobs:
             apache2 apache2-dev libnghttp2-dev vsftpd
           echo 'CC=gcc-12' >> $GITHUB_ENV
           echo 'CXX=g++-12' >> $GITHUB_ENV
-        name: 'install prereqs and impacket, pytest, crypto, apache2'
+        name: 'install prereqs'
 
       - name: cache quictls
         uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4
@@ -444,36 +444,8 @@ jobs:
           # $HOME/quiche/quiche/deps/boringssl/src/lib
         name: 'build quiche and boringssl'
 
-      - name: cache mod_h2
-        uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4
-        id: cache-mod_h2
-        env:
-          cache-name: cache-mod_h2
-        with:
-          path: /home/runner/mod_h2
-          key: ${{ runner.os }}-http3-build-${{ env.cache-name }}-${{ env.mod_h2-version }}
-
-      - name: 'build mod_h2'
-        if: steps.cache-mod_h2.outputs.cache-hit != 'true'
-        run: |
-          cd $HOME
-          git clone --quiet --depth=1 -b v${{ env.mod_h2-version }} https://github.com/icing/mod_h2
-          cd mod_h2
-          autoreconf -fi
-          ./configure
-          make
-
-      - name: 'install mod_h2'
-        run: |
-          cd $HOME/mod_h2
-          sudo make install
-
       - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
 
-      - run: |
-          sudo python3 -m pip install -r tests/requirements.txt -r tests/http/requirements.txt
-        name: 'install python test prereqs'
-
       - run: autoreconf -fi
         if: ${{ matrix.build.configure }}
         name: 'autoreconf'
@@ -527,6 +499,10 @@ jobs:
             make V=1 -C tests
           fi
 
+      - name: 'install test prereqs'
+        run: |
+          sudo python3 -m pip install -r tests/requirements.txt
+
       - name: 'run tests'
         env:
           TFLAGS: "${{ matrix.build.tflags }}"
@@ -537,6 +513,34 @@ jobs:
             make V=1 test-ci
           fi
 
+      - name: 'install pytest prereqs'
+        run: |
+          sudo python3 -m pip install -r tests/http/requirements.txt
+
+      - name: cache mod_h2
+        uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4
+        id: cache-mod_h2
+        env:
+          cache-name: cache-mod_h2
+        with:
+          path: /home/runner/mod_h2
+          key: ${{ runner.os }}-http3-build-${{ env.cache-name }}-${{ env.mod_h2-version }}
+
+      - name: 'build mod_h2'
+        if: steps.cache-mod_h2.outputs.cache-hit != 'true'
+        run: |
+          cd $HOME
+          git clone --quiet --depth=1 -b v${{ env.mod_h2-version }} https://github.com/icing/mod_h2
+          cd mod_h2
+          autoreconf -fi
+          ./configure
+          make
+
+      - name: 'install mod_h2'
+        run: |
+          cd $HOME/mod_h2
+          sudo make install
+
       - name: 'run pytest'
         env:
           TFLAGS: "${{ matrix.build.tflags }}"
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index f9de7206ee..10ef656e61 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -273,8 +273,12 @@ jobs:
           sudo apt-get update -y
           sudo apt-get install libtool autoconf automake ninja-build pkgconf stunnel4 libpsl-dev libbrotli-dev libzstd-dev \
             ${{ matrix.build.install_packages }}
-          sudo python3 -m pip install impacket
-        name: 'install prereqs and impacket'
+        name: 'install prereqs'
+
+      - if: contains(matrix.build.install_steps, 'pytest')
+        run: |
+          sudo apt-get install apache2 apache2-dev libnghttp2-dev vsftpd
+        name: 'install prereqs for pytest'
 
       - if: startsWith(matrix.build.container, 'alpine')
         run: |
@@ -549,38 +553,6 @@ jobs:
           printenv >> $GITHUB_ENV
         name: 'install Intel compilers'
 
-      - if: contains(matrix.build.install_steps, 'pytest')
-        run: |
-          sudo apt-get install apache2 apache2-dev libnghttp2-dev vsftpd
-          sudo python3 -m pip install -r tests/http/requirements.txt
-        name: 'install pytest and apach2-dev'
-
-      - name: cache mod_h2
-        if: contains(matrix.build.install_steps, 'pytest')
-        uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4
-        id: cache-mod_h2
-        env:
-          cache-name: cache-mod_h2
-        with:
-          path: /home/runner/mod_h2
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.mod_h2-version }}
-
-      - name: 'build mod_h2'
-        if: contains(matrix.build.install_steps, 'pytest') && steps.cache-mod_h2.outputs.cache-hit != 'true'
-        run: |
-          cd $HOME
-          git clone --quiet --depth=1 -b v${{ env.mod_h2-version }} https://github.com/icing/mod_h2
-          cd mod_h2
-          autoreconf -fi
-          ./configure
-          make
-
-      - name: 'install mod_h2'
-        if: contains(matrix.build.install_steps, 'pytest')
-        run: |
-          cd $HOME/mod_h2
-          sudo make install
-
       - run: autoreconf -fi
         if: ${{ matrix.build.configure }}
         name: 'autoreconf'
@@ -652,6 +624,11 @@ jobs:
             make V=1 -C tests
           fi
 
+      - name: 'install test prereqs'
+        if: ${{ matrix.build.install_steps != 'skipall' && !startsWith(matrix.build.container, 'alpine') && matrix.build.container == null }}
+        run: |
+          sudo python3 -m pip install -r tests/requirements.txt
+
       - name: 'run tests'
         if: ${{ matrix.build.install_steps != 'skipall' && matrix.build.install_steps != 'skiprun' }}
         timeout-minutes: ${{ contains(matrix.build.install_packages, 'valgrind') && 30 || 15 }}
@@ -669,6 +646,37 @@ jobs:
             make V=1 test-ci
           fi
 
+      - name: 'install pytest prereqs'
+        if: contains(matrix.build.install_steps, 'pytest')
+        run: |
+          sudo python3 -m pip install -r tests/http/requirements.txt
+
+      - name: cache mod_h2
+        if: contains(matrix.build.install_steps, 'pytest')
+        uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4
+        id: cache-mod_h2
+        env:
+          cache-name: cache-mod_h2
+        with:
+          path: /home/runner/mod_h2
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.mod_h2-version }}
+
+      - name: 'build mod_h2'
+        if: contains(matrix.build.install_steps, 'pytest') && steps.cache-mod_h2.outputs.cache-hit != 'true'
+        run: |
+          cd $HOME
+          git clone --quiet --depth=1 -b v${{ env.mod_h2-version }} https://github.com/icing/mod_h2
+          cd mod_h2
+          autoreconf -fi
+          ./configure
+          make
+
+      - name: 'install mod_h2'
+        if: contains(matrix.build.install_steps, 'pytest')
+        run: |
+          cd $HOME/mod_h2
+          sudo make install
+
       - name: 'run pytest'
         if: contains(matrix.build.install_steps, 'pytest')
         env:
diff --git a/.github/workflows/linux32.yml b/.github/workflows/linux32.yml
index eb5f7bfc43..d39dae828b 100644
--- a/.github/workflows/linux32.yml
+++ b/.github/workflows/linux32.yml
@@ -61,7 +61,6 @@ jobs:
           sudo dpkg --add-architecture i386
           sudo apt-get update -y
           sudo apt-get install -y --no-install-suggests --no-install-recommends libtool autoconf automake pkgconf stunnel4 ${{ matrix.build.install_packages }}
-          sudo python3 -m pip install --break-system-packages impacket
         name: 'install prereqs'
 
       - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
@@ -88,6 +87,10 @@ jobs:
       - run: make V=1 -C tests
         name: 'make tests'
 
+      - name: 'install test prereqs'
+        run: |
+          sudo python3 -m pip install --break-system-packages -r tests/requirements.txt
+
       - run: make V=1 test-ci
         name: 'run tests'
         env:
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
index c447bfb11e..31beb5ee25 100644
--- a/.github/workflows/macos.yml
+++ b/.github/workflows/macos.yml
@@ -222,11 +222,11 @@ jobs:
       - name: 'make tests'
         run: make -C bld V=1 -C tests
 
-      - name: 'pip3 install'
+      - name: 'make install test prereqs'
         run: |
           python3 -m venv $HOME/venv
           source $HOME/venv/bin/activate
-          python3 -m pip install impacket
+          python3 -m pip install -r tests/requirements.txt
 
       - name: 'run tests'
         timeout-minutes: 20
@@ -380,11 +380,11 @@ jobs:
       - name: 'cmake build tests'
         run: ninja -C bld testdeps
 
-      - name: 'pip3 install'
+      - name: 'cmake install test prereqs'
         run: |
           python3 -m venv $HOME/venv
           source $HOME/venv/bin/activate
-          python3 -m pip install impacket
+          python3 -m pip install -r tests/requirements.txt
 
       - name: 'cmake run tests'
         timeout-minutes: ${{ matrix.build.torture && 20 || 10 }}
diff --git a/.github/workflows/torture.yml b/.github/workflows/torture.yml
index 740db73949..eb127dd28b 100644
--- a/.github/workflows/torture.yml
+++ b/.github/workflows/torture.yml
@@ -61,7 +61,6 @@ jobs:
     steps:
       - run: |
           sudo apt-get install cmake ninja-build pkgconf stunnel4 ${{ matrix.build.install }}
-          python3 -m pip install --break-system-packages impacket
         name: 'install prereqs'
 
       - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
@@ -86,6 +85,10 @@ jobs:
       - run: cmake --build . --verbose --target testdeps
         name: 'build tests'
 
+      - name: 'install test prereqs'
+        run: |
+          python3 -m pip install --break-system-packages -r tests/requirements.txt
+
       - run: cmake --build . --verbose --target test-torture
         name: 'run tests'
         env:
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index f62f1c25de..a3bc542374 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -281,7 +281,7 @@ jobs:
         timeout-minutes: 10
         run: make -C bld -j5 V=1 -C tests
 
-      - name: 'install test tools'
+      - name: 'install test prereqs'
         if: ${{ matrix.build == 'autotools' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
         timeout-minutes: 5
         run: |
@@ -372,7 +372,7 @@ jobs:
         timeout-minutes: 10
         run: cmake --build bld --config '${{ matrix.type }}' --target testdeps
 
-      - name: 'install test tools'
+      - name: 'install test prereqs'
         if: ${{ matrix.build == 'cmake' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
         timeout-minutes: 5
         run: |
@@ -500,7 +500,7 @@ jobs:
           PATH="$(cygpath "${USERPROFILE}")/my-cache/${{ matrix.dir }}/bin:/c/msys64/usr/bin:$PATH"
           cmake --build bld --config '${{ matrix.type }}' --parallel 5 --target testdeps
 
-      - name: 'install test tools'
+      - name: 'install test prereqs'
         if: ${{ matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
         timeout-minutes: 5
         run: |
@@ -783,7 +783,7 @@ jobs:
         timeout-minutes: 10
         run: cmake --build bld --config '${{ matrix.type }}' --parallel 5 --target testdeps
 
-      - name: 'install test tools'
+      - name: 'install test prereqs'
         if: ${{ matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
         timeout-minutes: 5
         run: |