Add requirements-reference.txt for reproducible Python deps

.github/pull_request_template.md

@@ -10,7 +10,9 @@ TODO
   - Add a [sidebar entry](https://github.com/precice/precice.github.io/blob/master/_data/sidebars/tutorial_sidebar.yml)
   - Add it to the [overview](https://github.com/precice/precice.github.io/blob/master/content/tutorials/tutorials.md)
 
+For **release PRs** (new distribution): update `tools/tests/requirements-reference.txt` if `reference_versions.yaml` changed (`python3 tools/tests/update_requirements_reference.py`).
+
 ## Resources
 
 - [Contributing tutorials](https://precice.org/community-contribute-to-precice.html#contributing-tutorials)
-- [System tests documentation](https://precice.org/dev-docs-system-tests.html)
+- [System tests documentation](https://precice.org/dev-docs-system-tests.html)

.github/workflows/check-requirements-reference.yml

@@ -0,0 +1,28 @@
+name: Check requirements-reference
+on:
+  push:
+    branches:
+      - master
+      - develop
+    paths:
+      - tools/tests/requirements-reference.txt
+      - tools/tests/reference_versions.yaml
+      - tools/tests/update_requirements_reference.py
+  pull_request:
+    branches:
+      - master
+      - develop
+    paths:
+      - tools/tests/requirements-reference.txt
+      - tools/tests/reference_versions.yaml
+      - tools/tests/update_requirements_reference.py
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Validate requirements-reference
+        run: python3 tools/tests/validate_requirements_reference.py

.github/workflows/generate-reference-results-manual.yml

@@ -0,0 +1,41 @@
+name: Generate reference results (manual)
+on:
+  workflow_dispatch:
+    inputs:
+      suites:
+        description: 'Test suites to execute (comma-separated, see tests.yaml)'
+        default: 'release'
+        required: true
+        type: string
+      from_ref:
+        description: 'Git branch to use and commit to'
+        required: true
+        type: string
+      commit_msg:
+        description: 'Commit message'
+        default: "Add reference results"
+        type: string
+      clean_docker:
+        description: 'Run docker-system-prune before running tests'
+        default: 'FALSE'
+        type: choice
+        options:
+            - 'FALSE'
+            - 'TRUE'
+      log_level:
+        description: 'Logging verbosity of systemtests.py'
+        default: 'INFO'
+        type: choice
+        options:
+            - 'INFO'
+            - 'DEBUG'
+
+jobs:
+  generate_reference_results_manual:
+    uses: ./.github/workflows/generate-reference-results.yml
+    with: 
+      from_ref: ${{ inputs.from_ref }}
+      commit_msg: ${{ inputs.commit_msg }}
+      suites: ${{ inputs.suites }}
+      clean_docker: ${{ inputs.clean_docker }}
+      log_level: ${{ inputs.log_level }}

.github/workflows/generate_reference_results_workflow.yml → .github/workflows/generate-reference-results.yml

@@ -1,35 +1,42 @@
-name: Generate reference results workflow
+name: Generate reference results (called)
 on:
   workflow_call:
     inputs:
-      from_ref:
-        description: 'Use the systemtests + tutorial metadata + reference_version from this ref'
+      suites:
+        description: 'Test suites to execute (comma-separated, see tests.yaml)'
         required: true
         type: string
+      from_ref:
+        description: 'Git branch to use and commit to'
+        default: 'develop'
+        type: string
       commit_msg:
-        description: 'Commit msg for commit that adds the reference results'
-        default: "Adding reference results"
+        description: 'Commit message'
+        default: "Add reference results"
         type: string
-      suites:
-        description: 'Comma-separated test suites to generate reference results for. If empty, all suites are generated.'
-        default: ''
-        required: false
+      clean_docker:
+        description: 'Run docker-system-prune before running tests'
+        default: 'FALSE'
         type: string
       log_level:
-        description: 'Logging verbosity level used for the systemtests'
+        description: 'Logging verbosity of systemtests.py'
         required: true
         type: string
 jobs:
   generate_reference_results:
     runs-on: [self-hosted, linux, x64, precice-tests-vm]
     steps:
-    - name: Display a quick job summary
+    - name: Report the input values in the summary
       run: |
-        echo "Initiated by: ${{ github.actor }}"
-        echo "Running generate_reference_results.py --log-level ${{inputs.log_level}}"
-        echo "Using Ref: ${{ inputs.from_ref }}"
-        echo "Suites filter: ${{ inputs.suites || 'all (no filter)' }}"
-        echo "Commit message on success:  ${{ inputs.commit_msg }}"
+        {
+        echo "- Initiated by: @${{ github.actor }}"
+        echo "- Test suites: \`${{ inputs.suites || 'all (no filter)' }}\`"
+        echo "- Git branch to commit to: \`${{ inputs.from_ref }}\`"
+        echo "- Commit message:  \`${{ inputs.commit_msg }}\`"
+        echo "- Run \`docker-system-prune\` before running tests: \`${{ inputs.clean_docker }}\`"
+        echo "- Log level: \`${{ inputs.log_level }}\`"
+        echo "- Running \`generate_reference_results.py --suite ${{inputs.suites}} --log_level ${{inputs.log_level}}\`"
+        } >> "$GITHUB_STEP_SUMMARY"
     - name: Move LFS URL to local LFS server
       run: |
         /home/precice/runners_root/scripts/make_lfs_local.sh
@@ -51,21 +58,19 @@ jobs:
       run: |
         pip install --user -r tools/tests/requirements.txt
     - name: Cleanup Docker cache
+      if: ${{  inputs.clean_docker == 'TRUE' }}
       # Remove all Docker containers, images, and build caches
       # to ensure consistent results and as storage cleanup
       run: |
         docker system prune --all --force
     - name: Run tests
       run: |
         cd tools/tests
-        test -f generate_reference_results.py && export GENERATE_REF_RESULTS=generate_reference_results.py
-        test -f generate_reference_data.py && export GENERATE_REF_RESULTS=generate_reference_data.py
-        echo "Selected $GENERATE_REF_RESULTS to run"
         SUITES_ARGS=()
         if [ -n "${{ inputs.suites }}" ]; then
           SUITES_ARGS+=(--suites "${{ inputs.suites }}")
         fi
-        python "$GENERATE_REF_RESULTS" --log-level="${{inputs.log_level}}" "${SUITES_ARGS[@]}"
+        python generate_reference_results.py --log_level="${{inputs.log_level}}" "${SUITES_ARGS[@]}"
         cd ../../
     - name: Create commit
       if: success()
@@ -84,18 +89,21 @@ jobs:
         fi
         git commit -m "${{inputs.commit_msg}}"
         git push
-    - name: Archive run files
-      if: failure()
+    - name: Archive system test logs
+      if: ${{ always() }}
       uses: actions/upload-artifact@v7
       with:
-        name: system_tests_run_${{ github.run_id }}_${{ github.run_attempt }}_reference_full
+        name: system_tests_run_${{ github.run_id }}_${{ github.run_attempt }}_reference_logs
         path: |
-          runs/*
-    - name: Archive main log files
+          runs/*/system-tests-build.log
+          runs/*/system-tests-run.log
+          runs/*/system-tests-compare.log
+          runs/*/*/system-tests_*.log
+    - name: Archive run files
+      if: ${{ always() }}
       uses: actions/upload-artifact@v7
       with:
-        name: system_tests_run_${{ github.run_id }}_${{ github.run_attempt }}_reference_logs
+        name: system_tests_run_${{ github.run_id }}_${{ github.run_attempt }}_reference_full
         path: |
-          runs/*/system-tests-stdout.log
-          runs/*/system-tests-stderr.log
-          runs/*/*/system-tests_*.log
+          runs/*
+