Files
Actions/.github/workflows/UnitTesting.yml

550 lines
24 KiB
YAML

# ==================================================================================================================== #
# Authors: #
# Patrick Lehmann #
# Unai Martinez-Corral #
# #
# ==================================================================================================================== #
# Copyright 2020-2026 The pyTooling Authors #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); #
# you may not use this file except in compliance with the License. #
# You may obtain a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# #
# SPDX-License-Identifier: Apache-2.0 #
# ==================================================================================================================== #
name: Unit Testing (Matrix)
on:
workflow_call:
inputs:
jobs:
description: 'JSON list with environment fields, telling the system and Python versions to run tests with.'
required: true
type: string
apt:
description: 'Ubuntu dependencies to be installed through apt.'
required: false
default: ''
type: string
brew:
description: 'macOS dependencies to be installed through brew.'
required: false
default: ''
type: string
pacboy:
description: 'MSYS2 dependencies to be installed through pacboy (pacman).'
required: false
default: ''
type: string
requirements:
description: 'Python dependencies to be installed through pip.'
required: false
default: '-r ./requirements.txt'
type: string
mingw_requirements:
description: 'Override Python dependencies to be installed through pip on MSYS2 (MINGW64) only.'
required: false
default: ''
type: string
macos_before_script:
description: 'Scripts to execute before pytest on macOS (Intel).'
required: false
default: ''
type: string
macos_arm_before_script:
description: 'Scripts to execute before pytest on macOS (ARM).'
required: false
default: ''
type: string
ubuntu_before_script:
description: 'Scripts to execute before pytest on Ubuntu.'
required: false
default: ''
type: string
windows_before_script:
description: 'Scripts to execute before pytest on Windows (x64-64).'
required: false
default: ''
type: string
windows_arm_before_script:
description: 'Scripts to execute before pytest on Windows (aarch64).'
required: false
default: ''
type: string
mingw64_before_script:
description: 'Scripts to execute before pytest on Windows (x64-64) within MSYS2 MinGW64.'
required: false
default: ''
type: string
ucrt64_before_script:
description: 'Scripts to execute before pytest on Windows (x64-64) within MSYS2 UCRT64.'
required: false
default: ''
type: string
root_directory:
description: 'Working directory for running tests.'
required: false
default: '.'
type: string
tests_directory:
description: 'Path to the directory containing tests (relative from root_directory).'
required: false
default: 'tests'
type: string
unittest_directory:
description: 'Path to the directory containing unit tests (relative from tests_directory).'
required: false
default: 'unit'
type: string
unittest_report_xml:
description: 'JSON object describing the path where to save the unittest summary report XML.'
required: false
default: >-
{ "directory": "report/unit",
"filename": "TestReportSummary.xml",
"fullpath": "report/unit/TestReportSummary.xml"
}
type: string
coverage_config:
description: 'Path to the .coveragerc file. Use pyproject.toml by default.'
required: false
default: 'pyproject.toml'
type: string
coverage_report_xml:
description: 'JSON object describing the path where the coverage report in XML format will be generated.'
required: false
default: >-
{ "directory": "report/coverage",
"filename": "coverage.xml",
"fullpath": "report/coverage/coverage.xml"
}
type: string
coverage_report_json:
description: 'JSON object describing the path where the coverage report in JSON format will be generated.'
required: false
default: >-
{ "directory": "report/coverage",
"filename": "coverage.json",
"fullpath": "report/coverage/coverage.json"
}
type: string
coverage_report_html:
description: 'JSON object describing the path where the coverage report in HTML format will be generated.'
required: false
default: >-
{ "directory": "report/coverage/html",
}
# "fullpath": "report/coverage/html"
type: string
unittest_xml_artifact:
description: "Generate unit test report with junitxml and upload results as an artifact."
required: false
default: ''
type: string
unittest_html_artifact:
description: "Generate unit test report with junitxml and upload results as an artifact."
required: false
default: ''
type: string
coverage_sqlite_artifact:
description: 'Name of the SQLite coverage artifact.'
required: false
default: ''
type: string
coverage_xml_artifact:
description: 'Name of the XML coverage artifact.'
required: false
default: ''
type: string
coverage_json_artifact:
description: 'Name of the JSON coverage artifact.'
required: false
default: ''
type: string
coverage_html_artifact:
description: 'Name of the HTML coverage artifact.'
required: false
default: ''
type: string
jobs:
UnitTesting:
name: ${{ matrix.sysicon }} ${{ matrix.pyicon }} Unit Tests - Python ${{ matrix.python }}
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(inputs.jobs) }}
defaults:
run:
shell: ${{ matrix.shell }}
steps:
- name: ⏬ Checkout repository
uses: actions/checkout@v6
with:
lfs: true
submodules: true
# Package Manager steps
- name: 🔧 Install homebrew dependencies on macOS
if: ( matrix.system == 'macos' || matrix.system == 'macos-arm' ) && inputs.brew != ''
run: brew install ${{ inputs.brew }}
- name: 🔧 Install apt dependencies on Ubuntu
if: ( matrix.system == 'ubuntu' || matrix.system == 'ubuntu-arm' ) && inputs.apt != ''
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends ${{ inputs.apt }}
# Compute Dependencies for MSYS2 steps
# - name: 🔧 Install dependencies (system Python for Python shell)
# if: matrix.system == 'msys2'
# shell: pwsh
# run: |
# py -3.12 -m pip install --disable-pip-version-check --break-system-packages -U tomli
# TODO: extract step to an Action so package, so code can be shared with AppTesting.yml
- name: Compute path to requirements file
id: requirements
shell: python
run: |
from os import getenv
from pathlib import Path
from sys import version
print(f"Python: {version}")
requirements = "${{ inputs.requirements }}"
if requirements.startswith("-r"):
requirements = requirements[2:].lstrip()
if requirements.startswith("./"):
requirementsFile = Path("${{ inputs.root_directory || '.' }}") / Path("${{ inputs.tests_directory || '.' }}") / Path("${{ inputs.unittest_directory || '.' }}") / Path(requirements[2:])
else:
requirementsFile = Path(requirements)
if not requirementsFile.exists():
print(f"::error title=FileNotFoundError::{requirementsFile}")
exit(1)
print(f"requirements file: {requirementsFile.as_posix()}")
# Write requirements path to special file
github_output = Path(getenv("GITHUB_OUTPUT"))
print(f"GITHUB_OUTPUT: {github_output}")
with github_output.open("a+") as f:
f.write(f"requirements=-r {requirementsFile.as_posix()}\n")
else:
print(f"requirements list: {requirements}")
# TODO: extract step to an Action so package lists are shared with UnitTesting (and GHDL?)
- name: Compute pacman/pacboy packages
id: pacboy
if: matrix.system == 'msys2'
shell: python
run: |
from os import getenv
from pathlib import Path
from re import compile
from sys import version
def loadRequirementsFile(requirementsFile: Path):
requirements = []
with requirementsFile.open("r") as file:
for line in file.readlines():
line = line.strip()
if line.startswith("#") or line.startswith("https") or line == "":
continue
elif line.startswith("-r"):
# Remove the first word/argument (-r)
requirements += loadRequirementsFile(requirementsFile.parent / line[2:].lstrip())
else:
requirements.append(line)
return requirements
requirements = "${{ steps.requirements.outputs.requirements }}"
if requirements.startswith("-r"):
try:
dependencies = loadRequirementsFile(Path(requirements[2:].lstrip()))
except FileNotFoundError as ex:
print(f"::error title=FileNotFoundError::{ex}")
exit(1)
else:
dependencies = [req.strip() for req in requirements.split(" ")]
packages = {
"aiohttp": "python-aiohttp:p",
"coverage": "python-coverage:p",
"docstr_coverage": "python-pyaml:p python-types-pyyaml:p",
"igraph": "igraph:p",
"jinja2": "python-markupsafe:p",
"lxml": "python-lxml:p",
"numpy": "python-numpy:p",
"markupsafe": "python-markupsafe:p",
"pip": "python-pip:p",
"pyyaml": "python-pyaml:p python-types-pyyaml:p",
"ruamel.yaml": "python-ruamel-yaml:p",
# "ruamel.yaml": "python-ruamel-yaml:p python-ruamel.yaml.clib:p",
"sphinx": "python-markupsafe:p",
"tomli": "python-tomli:p", # outdated, now part of Python as tomllib
"wheel": "python-wheel:p",
"pyedaa.projectmodel": "python-ruamel-yaml:p python-ruamel.yaml.clib:p python-lxml:p",
"pyedaa.reports": "python-ruamel-yaml:p python-ruamel.yaml.clib:p python-lxml:p",
"sphinx-reports": "python-markupsafe:p python-pyaml:p python-types-pyyaml:p",
}
subPackages = {
"pytooling": {
"yaml": "python-ruamel-yaml:p python-ruamel.yaml.clib:p",
"pypi": "python-aiohttp:p",
},
}
regExp = compile(r"(?P<PackageName>[\w_\-\.]+)(?:\[(?P<SubPackages>(?:\w+)(?:\s*,\s*\w+)*)\])?(?:\s*(?P<Comperator>[<>~=]+)\s*)(?P<Version>\d+(?:\.\d+)*)(?:-(?P<VersionExtension>\w+))?")
pacboyPackages = set(("python-pip:p", "python-wheel:p", "python-tomli:p"))
print(f"Processing dependencies ({len(dependencies)}):")
for dependency in dependencies:
print(f" {dependency}")
match = regExp.match(dependency.lower())
if not match:
print(f" Wrong format: {dependency}")
print(f"::error title=Identifying Pacboy Packages::Unrecognized dependency format '{dependency}'")
continue
package = match["PackageName"]
if package in packages:
rewrite = packages[package]
print(f" Found rewrite rule for '{package}': {rewrite}")
pacboyPackages.add(rewrite)
if match["SubPackages"] and package in subPackages:
for subPackage in match["SubPackages"].split(","):
if subPackage in subPackages[package]:
rewrite = subPackages[package][subPackage]
print(f" Found rewrite rule for '{package}[..., {subPackage}, ...]': {rewrite}")
pacboyPackages.add(rewrite)
# Write jobs to special file
github_output = Path(getenv("GITHUB_OUTPUT"))
print(f"GITHUB_OUTPUT: {github_output}")
with github_output.open("a+") as f:
f.write(f"pacboy_packages={' '.join(pacboyPackages)}\n")
# Python setup
- name: '🟦 Setup MSYS2 for ${{ matrix.runtime }}'
uses: msys2/setup-msys2@v2
if: matrix.system == 'msys2'
with:
msystem: ${{ matrix.runtime }}
update: true
pacboy: >-
${{ steps.pacboy.outputs.pacboy_packages }}
${{ inputs.pacboy }}
- name: 🐍 Setup Python ${{ matrix.python }}
uses: actions/setup-python@v6
if: matrix.system != 'msys2'
with:
python-version: ${{ matrix.python }}
# Python Dependency steps
- name: 🔧 Install wheel and pip dependencies (native)
if: matrix.system != 'msys2'
run: |
python -m pip install --disable-pip-version-check -U wheel
python -m pip install --disable-pip-version-check ${{ steps.requirements.outputs.requirements }}
- name: 🔧 Install pip dependencies (MSYS2)
if: matrix.system == 'msys2'
run: |
if [ -n '${{ inputs.mingw_requirements }}' ]; then
python -m pip install --disable-pip-version-check --break-system-packages ${{ inputs.mingw_requirements }}
else
python -m pip install --disable-pip-version-check --break-system-packages ${{ steps.requirements.outputs.requirements }}
fi
# Before scripts
- name: 🍎 macOS (Intel) before scripts
if: matrix.system == 'macos' && inputs.macos_before_script != ''
run: ${{ inputs.macos_before_script }}
- name: 🍏 macOS (ARM) before scripts
if: matrix.system == 'macos-arm' && inputs.macos_arm_before_script != ''
run: ${{ inputs.macos_arm_before_script }}
- name: 🐧 Ubuntu before scripts
if: ( matrix.system == 'ubuntu' || matrix.system == 'ubuntu-arm' ) && inputs.ubuntu_before_script != ''
run: ${{ inputs.ubuntu_before_script }}
- name: 🪟 Windows (x86-64) before scripts
if: matrix.system == 'windows' && inputs.windows_before_script != ''
run: ${{ inputs.windows_before_script }}
- name: 🏢 Windows (aarch64) before scripts
if: matrix.system == 'windows-arm' && inputs.windows_arm_before_script != ''
run: ${{ inputs.windows_arm_before_script }}
- name: 🪟🟦 Windows (x86-64) + MinGW64 before scripts
if: matrix.system == 'msys2' && matrix.runtime == 'MINGW64' && inputs.mingw64_before_script != ''
run: ${{ inputs.mingw64_before_script }}
- name: 🪟🟨 Windows (x86-64) + UCRT64 before scripts
if: matrix.system == 'msys2' && matrix.runtime == 'UCRT64' && inputs.ucrt64_before_script != ''
run: ${{ inputs.ucrt64_before_script }}
# Run pytests
# TODO: allow configuration of pytest_args
- name: ✅ Run unit tests (Ubuntu/macOS)
id: pytest_bash
if: ( matrix.system != 'windows' && matrix.system != 'windows-arm' )
continue-on-error: true
run: |
export ENVIRONMENT_NAME="${{ matrix.envname }}"
export PYTHONPATH=$(pwd)
cd "${{ inputs.root_directory || '.' }}"
[ -n '${{ inputs.unittest_xml_artifact }}' ] && PYTEST_ARGS='--junitxml=${{ fromJson(inputs.unittest_report_xml).fullpath }}' || unset PYTEST_ARGS
if [ -n '${{ inputs.coverage_config }}' ]; then
printf "%s\n" "coverage run --data-file=.coverage --rcfile=pyproject.toml -m pytest -raP $PYTEST_ARGS --color=yes ${{ inputs.tests_directory || '.' }}/${{ inputs.unittest_directory }}"
coverage run --data-file=.coverage --rcfile=pyproject.toml -m pytest -raP $PYTEST_ARGS --color=yes ${{ inputs.tests_directory || '.' }}/${{ inputs.unittest_directory }}
else
printf "%s\n" "python -m pytest -raP $PYTEST_ARGS --color=yes ${{ inputs.tests_directory || '.' }}/${{ inputs.unittest_directory }}"
python -m pytest -raP $PYTEST_ARGS --color=yes ${{ inputs.tests_directory || '.' }}/${{ inputs.unittest_directory }}
fi
- name: ✅ Run unit tests (Windows)
id: pytest_posh
if: ( matrix.system == 'windows' || matrix.system == 'windows-arm' )
continue-on-error: true
run: |
$env:ENVIRONMENT_NAME = "${{ matrix.envname }}"
$env:PYTHONPATH = (Get-Location).ToString()
cd "${{ inputs.root_directory || '.' }}"
$PYTEST_ARGS = if ("${{ inputs.unittest_xml_artifact }}") { "--junitxml=${{ fromJson(inputs.unittest_report_xml).fullpath }}" } else { "" }
if ("${{ inputs.coverage_config }}") {
Write-Host "coverage run --data-file=.coverage --rcfile=pyproject.toml -m pytest -raP --color=yes ${{ inputs.tests_directory || '.' }}/${{ inputs.unittest_directory }}"
coverage run --data-file=.coverage --rcfile=pyproject.toml -m pytest -raP $PYTEST_ARGS --color=yes ${{ inputs.tests_directory || '.' }}/${{ inputs.unittest_directory }}
} else {
Write-Host "python -m pytest -raP $PYTEST_ARGS --color=yes ${{ inputs.tests_directory || '.' }}/${{ inputs.unittest_directory }}"
python -m pytest -raP $PYTEST_ARGS --color=yes ${{ inputs.tests_directory || '.' }}/${{ inputs.unittest_directory }}
}
- name: Convert coverage to XML format (Cobertura)
id: convert_xml
if: inputs.coverage_xml_artifact != ''
continue-on-error: true
run: coverage xml --data-file=.coverage
- name: Convert coverage to JSON format
id: convert_json
if: inputs.coverage_json_artifact != ''
continue-on-error: true
run: coverage json --data-file=.coverage
- name: Convert coverage to HTML format
id: convert_html
if: inputs.coverage_html_artifact != ''
continue-on-error: true
run: |
coverage html --data-file=.coverage -d ${{ fromJson(inputs.coverage_report_html).directory }}
rm ${{ fromJson(inputs.coverage_report_html).directory }}/.gitignore
# Upload artifacts
- name: 📤 Upload '${{ fromJson(inputs.unittest_report_xml).filename }}' artifact
uses: pyTooling/upload-artifact@v6
if: inputs.unittest_xml_artifact != ''
continue-on-error: true
with:
name: ${{ inputs.unittest_xml_artifact }}-${{ matrix.system }}-${{ matrix.runtime }}-${{ matrix.python }}
working-directory: ${{ fromJson(inputs.unittest_report_xml).directory }}
path: ${{ fromJson(inputs.unittest_report_xml).filename }}
if-no-files-found: error
retention-days: 1
# - name: 📤 Upload 'Unit Tests HTML Report' artifact
# if: inputs.unittest_html_artifact != ''
# continue-on-error: true
# uses: pyTooling/upload-artifact@v6
# with:
# name: ${{ inputs.unittest_html_artifact }}-${{ matrix.system }}-${{ matrix.runtime }}-${{ matrix.python }}
# path: ${{ inputs.unittest_report_html_directory }}
# if-no-files-found: error
# retention-days: 1
- name: 📤 Upload 'Coverage SQLite Database' artifact
if: inputs.coverage_sqlite_artifact != ''
continue-on-error: true
uses: pyTooling/upload-artifact@v6
with:
name: ${{ inputs.coverage_sqlite_artifact }}-${{ matrix.system }}-${{ matrix.runtime }}-${{ matrix.python }}
path: .coverage
include-hidden-files: true
if-no-files-found: error
retention-days: 1
- name: 📤 Upload 'Coverage XML Report' artifact
if: inputs.coverage_xml_artifact != '' && steps.convert_xml.outcome == 'success'
continue-on-error: true
uses: pyTooling/upload-artifact@v6
with:
name: ${{ inputs.coverage_xml_artifact }}-${{ matrix.system }}-${{ matrix.runtime }}-${{ matrix.python }}
working-directory: ${{ fromJson(inputs.coverage_report_xml).directory }}
path: ${{ fromJson(inputs.coverage_report_xml).filename }}
if-no-files-found: error
retention-days: 1
- name: 📤 Upload 'Coverage JSON Report' artifact
if: inputs.coverage_json_artifact != '' && steps.convert_json.outcome == 'success'
continue-on-error: true
uses: pyTooling/upload-artifact@v6
with:
name: ${{ inputs.coverage_json_artifact }}-${{ matrix.system }}-${{ matrix.runtime }}-${{ matrix.python }}
working-directory: ${{ fromJson(inputs.coverage_report_json).directory }}
path: ${{ fromJson(inputs.coverage_report_json).filename }}
if-no-files-found: error
retention-days: 1
- name: 📤 Upload 'Coverage HTML Report' artifact
if: inputs.coverage_html_artifact != '' && steps.convert_html.outcome == 'success'
continue-on-error: true
uses: pyTooling/upload-artifact@v6
with:
name: ${{ inputs.coverage_html_artifact }}-${{ matrix.system }}-${{ matrix.runtime }}-${{ matrix.python }}
working-directory: ${{ fromJson(inputs.coverage_report_html).directory }}
path: '*'
if-no-files-found: error
retention-days: 1
- name: Generate error messages
shell: bash
run: |
exitCode=0
if [[ "${{ steps.pytest_bash.outcome }}" == "failure" || "${{ steps.pytest_posh.outcome }}" == "failure" ]]; then
printf "❌ pytest: %s\n" "Error in pytest execution."
printf "::error title=%s::%s\n" "pytest" "Error in pytest execution."
exitCode=1
else
printf "✅ pytest: No errors.\n"
fi
exit $exitCode