From e2e8b39c4194baff7f87876805dea46876b0fce1 Mon Sep 17 00:00:00 2001 From: Patrick Lehmann Date: Sun, 14 Sep 2025 00:10:19 +0200 Subject: [PATCH] Rework StaticTypeCheck. --- .github/workflows/CompletePipeline.yml | 12 +- .github/workflows/ExtractConfiguration.yml | 37 +++-- .github/workflows/Parameters.yml | 161 +++++++++++++++------ .github/workflows/StaticTypeCheck.yml | 78 +++++++++- pyproject.toml | 8 +- tests/requirements.txt | 2 +- 6 files changed, 231 insertions(+), 67 deletions(-) diff --git a/.github/workflows/CompletePipeline.yml b/.github/workflows/CompletePipeline.yml index 3fd44f4..b01a418 100644 --- a/.github/workflows/CompletePipeline.yml +++ b/.github/workflows/CompletePipeline.yml @@ -195,9 +195,15 @@ jobs: - ConfigParams - UnitTestingParams with: - python_version: ${{ needs.UnitTestingParams.outputs.python_version }} - html_report: ${{ needs.ConfigParams.outputs.typing_report_html_directory }} - html_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).statictyping_html }} + python_version: ${{ needs.UnitTestingParams.outputs.python_version }} + junit_report_directory: ${{ needs.ConfigParams.outputs.typing_report_junit_directory }} + junit_report_file: ${{ needs.ConfigParams.outputs.typing_report_junit_file }} + junit_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).statictyping_junit }} + cobertura_report_directory: ${{ needs.ConfigParams.outputs.typing_report_cobertura_directory }} + cobertura_report_file: ${{ needs.ConfigParams.outputs.typing_report_cobertura_file }} + cobertura_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).statictyping_cobertura }} + html_report: ${{ needs.ConfigParams.outputs.typing_report_html_directory }} + html_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).statictyping_html }} DocCoverage: uses: pyTooling/Actions/.github/workflows/CheckDocumentation.yml@dev diff --git a/.github/workflows/ExtractConfiguration.yml b/.github/workflows/ExtractConfiguration.yml index 35cb764..cfc1774 100644 --- a/.github/workflows/ExtractConfiguration.yml +++ b/.github/workflows/ExtractConfiguration.yml @@ -56,9 +56,6 @@ on: package_directory: description: "" value: ${{ jobs.Extract.outputs.package_directory }} - mypy_prepare_command: - description: "" - value: ${{ jobs.Extract.outputs.mypy_prepare_command }} unittest_report_xml_directory: description: "" value: ${{ jobs.Extract.outputs.unittest_report_xml_directory }} @@ -98,11 +95,22 @@ on: coverage_report_json: description: "" value: ${{ jobs.Extract.outputs.coverage_report_json }} + typing_report_cobertura_directory: + description: "" + value: ${{ jobs.Extract.outputs.typing_report_cobertura_directory }} + typing_report_cobertura_file: + description: "" + value: ${{ jobs.Extract.outputs.typing_report_cobertura_file }} + typing_report_junit_directory: + description: "" + value: ${{ jobs.Extract.outputs.typing_report_junit_directory }} + typing_report_junit_file: + description: "" + value: ${{ jobs.Extract.outputs.typing_report_junit_file }} typing_report_html_directory: description: "" value: ${{ jobs.Extract.outputs.typing_report_html_directory }} - jobs: Extract: name: 📓 Extract configurations from pyproject.toml @@ -110,7 +118,6 @@ jobs: outputs: package_fullname: ${{ steps.getPackageName.outputs.package_fullname }} package_directory: ${{ steps.getPackageName.outputs.package_directory }} - mypy_prepare_command: ${{ steps.getPackageName.outputs.mypy_prepare_command }} unittest_report_xml_directory: ${{ steps.getVariables.outputs.unittest_report_xml_directory }} unittest_report_xml_filename: ${{ steps.getVariables.outputs.unittest_report_xml_filename }} unittest_report_xml: ${{ steps.getVariables.outputs.unittest_report_xml }} @@ -124,6 +131,10 @@ jobs: coverage_report_json_directory: ${{ steps.getVariables.outputs.coverage_report_json_directory }} coverage_report_json_filename: ${{ steps.getVariables.outputs.coverage_report_json_filename }} coverage_report_json: ${{ steps.getVariables.outputs.coverage_report_json }} + typing_report_cobertura_directory: ${{ steps.getVariables.outputs.typing_report_cobertura_directory }} + typing_report_cobertura_file: ${{ steps.getVariables.outputs.typing_report_cobertura_file }} + typing_report_junit_directory: ${{ steps.getVariables.outputs.typing_report_junit_directory }} + typing_report_junit_file: ${{ steps.getVariables.outputs.typing_report_junit_file }} typing_report_html_directory: ${{ steps.getVariables.outputs.typing_report_html_directory }} steps: @@ -159,17 +170,14 @@ jobs: if namespace == "" or namespace == ".": fullname = f"{name}" directory = f"{name}" - mypy_prepare_command = "" else: fullname = f"{namespace}.{name}" directory = f"{namespace}/{name}" - mypy_prepare_command = f"touch {namespace}/__init__.py" print(dedent(f"""\ OUTPUTS: package_fullname: {fullname} package_directory: {directory} - mypy_prepare_command: {mypy_prepare_command} """)) github_output = Path(getenv("GITHUB_OUTPUT")) @@ -178,7 +186,6 @@ jobs: f.write(dedent(f"""\ package_fullname={fullname} package_directory={directory} - mypy_prepare_command={mypy_prepare_command} """)) - name: 🔁 Extract configurations from pyproject.toml @@ -199,6 +206,8 @@ jobs: coverageXMLFile = Path("./coverage.xml") coverageJSONFile = Path("./coverage.json") coverageRC = "${{ inputs.coverage_config }}".strip() + typingCoberturaFile = Path("report/typing/cobertura.xml") + typingJUnitFile = Path("report/typing/StaticTypingSummary.xml") typingHTMLDirectory = Path("htmlmypy") # Read output paths from 'pyproject.toml' file @@ -213,6 +222,8 @@ jobs: coverageHTMLDirectory = Path(pyProjectSettings["tool"]["coverage"]["html"]["directory"]) coverageXMLFile = Path(pyProjectSettings["tool"]["coverage"]["xml"]["output"]) coverageJSONFile= Path(pyProjectSettings["tool"]["coverage"]["json"]["output"]) + typingCoberturaFile = Path(pyProjectSettings["tool"]["mypy"]["cobertura_xml_report"]) / "cobertura.xml" + typingJUnitFile = Path(pyProjectSettings["tool"]["mypy"]["junit_xml"]) typingHTMLDirectory = Path(pyProjectSettings["tool"]["mypy"]["html_report"]) else: print(f"File '{pyProjectFile}' not found.") @@ -221,6 +232,8 @@ jobs: # Read output paths from '.coveragerc' file elif len(coverageRC) > 0: + print(f"::warning title=Deprecated::Using '{coverageRCFile}' is deprecated. Please use 'pyproject.toml'.") + coverageRCFile = Path(coverageRC) if coverageRCFile.exists(): with coverageRCFile.open("rb") as file: @@ -252,6 +265,10 @@ jobs: coverage_report_json_directory={coverageJSONFile.parent.as_posix()} coverage_report_json_filename={coverageJSONFile.name} coverage_report_json={coverageJSONFile.as_posix()} + typing_report_cobertura_directory={typingCoberturaFile.parent.as_posix()} + typing_report_cobertura_file={typingCoberturaFile.name} + typing_report_junit_directory={typingJUnitFile.parent.as_posix()} + typing_report_junit_file={typingJUnitFile.name} typing_report_html_directory={typingHTMLDirectory.as_posix()} """)) @@ -262,5 +279,7 @@ jobs: coverage html: {coverageHTMLDirectory} coverage xml: {coverageXMLFile} coverage json: {coverageJSONFile} + typing cobertura: {typingCoberturaFile} + typing junit: {typingJUnitFile} typing html: {typingHTMLDirectory} """)) diff --git a/.github/workflows/Parameters.yml b/.github/workflows/Parameters.yml index 8b580d4..cd2f415 100644 --- a/.github/workflows/Parameters.yml +++ b/.github/workflows/Parameters.yml @@ -115,25 +115,33 @@ on: python_version: description: "Default Python version for other jobs." value: ${{ jobs.Parameters.outputs.python_version }} - python_jobs: - description: "List of Python versions (and system combinations) to be used in the matrix of other jobs." - value: ${{ jobs.Parameters.outputs.python_jobs }} + package_fullname: + description: "The package's full name." + value: ${{ jobs.Parameters.outputs.package_fullname }} + package_directory: + description: "The package's directory." + value: ${{ jobs.Parameters.outputs.package_directory }} + artifact_basename: + description: "Artifact base name." + value: ${{ jobs.Parameters.outputs.artifact_basename }} artifact_names: description: "Pre-defined artifact names for other jobs." value: ${{ jobs.Parameters.outputs.artifact_names }} - params: - description: "Parameters to be used in other jobs." - value: ${{ jobs.Parameters.outputs.params }} + python_jobs: + description: "List of Python versions (and system combinations) to be used in the matrix of other jobs." + value: ${{ jobs.Parameters.outputs.python_jobs }} jobs: Parameters: name: ✎ Generate pipeline parameters runs-on: "ubuntu-${{ inputs.ubuntu_image_version }}" outputs: - python_version: ${{ steps.params.outputs.python_version }} - python_jobs: ${{ steps.params.outputs.python_jobs }} - artifact_names: ${{ steps.params.outputs.artifact_names }} - params: ${{ steps.params.outputs.params }} + python_version: ${{ steps.variables.outputs.python_version }} + package_fullname: ${{ steps.variables.outputs.package_fullname }} + package_directory: ${{ steps.variables.outputs.package_directory }} + artifact_basename: ${{ steps.variables.outputs.artifact_basename }} + artifact_names: ${{ steps.artifacts.outputs.artifact_names }} + python_jobs: ${{ steps.jobs.outputs.python_jobs }} steps: - name: Generate a startup delay of ${{ inputs.pipeline-delay }} seconds @@ -142,8 +150,91 @@ jobs: run: | sleep ${{ inputs.pipeline-delay }} - - name: Generate 'params' and 'python_jobs' - id: params + - name: Generate 'python_version' + id: variables + shell: python + run: | + from os import getenv + from pathlib import Path + from textwrap import dedent + + python_version = "${{ inputs.python_version }}".strip() + package_namespace = "${{ inputs.package_namespace }}".strip() + package_name = "${{ inputs.package_name }}".strip() + name = "${{ inputs.name }}".strip() + + if package_namespace == "" or package_namespace == ".": + package_fullname = f"{package_name}" + package_directory = f"{package_name}" + else: + package_fullname = f"{package_namespace}.{package_name}" + package_directory = f"{package_namespace}/{package_name}" + + artifact_basename = package_fullname if name == "" else name + + print("Variables:") + print(f" python_version: {python_version}") + print(f" package_fullname: {package_fullname}") + print(f" package_directory: {package_directory}") + print(f" artifact_basename: {artifact_basename}") + + # Write jobs to special file + github_output = Path(getenv("GITHUB_OUTPUT")) + print(f"GITHUB_OUTPUT: {github_output}") + with github_output.open("a+", encoding="utf-8") as f: + f.write(dedent(f"""\ + python_version={python_version} + package_fullname={package_fullname} + package_directory={package_directory} + artifact_basename={artifact_basename} + """)) + + - name: Generate 'artifact_names' + id: artifacts + shell: python + run: | + from json import dumps as json_dumps + from os import getenv + from pathlib import Path + from textwrap import dedent + + package_namespace = "${{ inputs.package_namespace }}".strip() + package_name = "${{ inputs.package_name }}".strip() + artifact_basename = "${{ steps.variables.outputs.artifact_basename }}" + + artifact_names = { + "unittesting_xml": f"{artifact_basename}-UnitTestReportSummary-XML", + "unittesting_html": f"{artifact_basename}-UnitTestReportSummary-HTML", + "perftesting_xml": f"{artifact_basename}-PerformanceTestReportSummary-XML", + "benchtesting_xml": f"{artifact_basename}-BenchmarkTestReportSummary-XML", + "apptesting_xml": f"{artifact_basename}-ApplicationTestReportSummary-XML", + "codecoverage_sqlite": f"{artifact_basename}-CodeCoverage-SQLite", + "codecoverage_xml": f"{artifact_basename}-CodeCoverage-XML", + "codecoverage_json": f"{artifact_basename}-CodeCoverage-JSON", + "codecoverage_html": f"{artifact_basename}-CodeCoverage-HTML", + "statictyping_cobertura": f"{artifact_basename}-StaticTyping-Cobertura-XML", + "statictyping_junit": f"{artifact_basename}-StaticTyping-JUnit-XML", + "statictyping_html": f"{artifact_basename}-StaticTyping-HTML", + "package_all": f"{artifact_basename}-Packages", + "documentation_html": f"{artifact_basename}-Documentation-HTML", + "documentation_latex": f"{artifact_basename}-Documentation-LaTeX", + "documentation_pdf": f"{artifact_basename}-Documentation-PDF", + } + + print("Artifacts Names ({len(artifact_names)}):") + for id, artifactName in artifact_names.items(): + print(f" {id:>24}: {artifactName}") + + # Write jobs to special file + github_output = Path(getenv("GITHUB_OUTPUT")) + print(f"GITHUB_OUTPUT: {github_output}") + with github_output.open("a+", encoding="utf-8") as f: + f.write(dedent(f"""\ + artifact_names={json_dumps(artifact_names)} + """)) + + - name: Generate 'python_jobs' + id: jobs shell: python run: | from json import dumps as json_dumps @@ -152,22 +243,14 @@ jobs: from textwrap import dedent from typing import Iterable - package_namespace = "${{ inputs.package_namespace }}".strip() - package_name = "${{ inputs.package_name }}".strip() - name = "${{ inputs.name }}".strip() - python_version = "${{ inputs.python_version }}".strip() + python_version = "${{ steps.variables.outputs.python_version }}" + name = "${{ steps.artifacts.outputs.artifact_base }}" systems = "${{ inputs.system_list }}".strip() versions = "${{ inputs.python_version_list }}".strip() include_list = "${{ inputs.include_list }}".strip() exclude_list = "${{ inputs.exclude_list }}".strip() disable_list = "${{ inputs.disable_list }}".strip() - if name == "": - if package_namespace == "" or package_namespace == ".": - name = f"{package_name}" - else: - name = f"{package_namespace}.{package_name}" - currentMSYS2Version = "3.12" currentAlphaVersion = "3.14" currentAlphaRelease = "3.14.0-rc.2" @@ -322,31 +405,11 @@ jobs: for runtime, version in combinations if runtime not in data["sys"] ] - artifact_names = { - "unittesting_xml": f"{name}-UnitTestReportSummary-XML", - "unittesting_html": f"{name}-UnitTestReportSummary-HTML", - "perftesting_xml": f"{name}-PerformanceTestReportSummary-XML", - "benchtesting_xml": f"{name}-BenchmarkTestReportSummary-XML", - "apptesting_xml": f"{name}-ApplicationTestReportSummary-XML", - "codecoverage_sqlite": f"{name}-CodeCoverage-SQLite", - "codecoverage_xml": f"{name}-CodeCoverage-XML", - "codecoverage_json": f"{name}-CodeCoverage-JSON", - "codecoverage_html": f"{name}-CodeCoverage-HTML", - "statictyping_html": f"{name}-StaticTyping-HTML", - "package_all": f"{name}-Packages", - "documentation_html": f"{name}-Documentation-HTML", - "documentation_latex": f"{name}-Documentation-LaTeX", - "documentation_pdf": f"{name}-Documentation-PDF", - } - print("Parameters:") print(f" python_version: {python_version}") print(f" python_jobs ({len(jobs)}):\n" + "".join([f" {{ " + ", ".join([f"\"{key}\": \"{value}\"" for key, value in job.items()]) + f" }},\n" for job in jobs]) ) - print(f" artifact_names ({len(artifact_names)}):") - for id, name in artifact_names.items(): - print(f" {id:>20}: {name}") # Write jobs to special file github_output = Path(getenv("GITHUB_OUTPUT")) @@ -355,13 +418,17 @@ jobs: f.write(dedent(f"""\ python_version={python_version} python_jobs={json_dumps(jobs)} - artifact_names={json_dumps(artifact_names)} """)) - name: Verify out parameters id: verify run: | - printf "python_version: %s\n" '${{ steps.params.outputs.python_version }}' - printf "python_jobs: %s\n" '${{ steps.params.outputs.python_jobs }}' - printf "artifact_names: %s\n" '${{ steps.params.outputs.artifact_names }}' - printf "params: %s\n" '${{ steps.params.outputs.params }}' + printf "python_version: %s\n" '${{ steps.variables.outputs.python_version }}' + printf "package_fullname: %s\n" '${{ steps.variables.outputs.package_fullname }}' + printf "package_directory: %s\n" '${{ steps.variables.outputs.package_directory }}' + printf "artifact_basename: %s\n" '${{ steps.variables.outputs.artifact_basename }}' + printf "====================\n" + printf "artifact_names: %s\n" '${{ steps.artifacts.outputs.artifact_names }}' + printf "====================\n" + printf "python_jobs: %s\n" '${{ steps.jobs.outputs.python_jobs }}' + printf "====================\n" diff --git a/.github/workflows/StaticTypeCheck.yml b/.github/workflows/StaticTypeCheck.yml index ecf1371..9a415ce 100644 --- a/.github/workflows/StaticTypeCheck.yml +++ b/.github/workflows/StaticTypeCheck.yml @@ -40,16 +40,36 @@ on: required: false default: '-r tests/requirements.txt' type: string + mypy_options: + description: 'Additional mypy options.' + required: false + default: '' + type: string html_report: description: 'Directory to upload as an artifact.' required: false + default: 'report/typing/html' + type: string + junit_report_directory: + description: 'JUnit file to upload as an artifact.' + required: false default: 'report/typing' type: string - junit_report: - description: 'junit file to upload as an artifact.' + junit_report_file: + description: 'JUnit file to upload as an artifact.' required: false default: 'StaticTypingSummary.xml' type: string + cobertura_report_directory: + description: 'Cobertura file to upload as an artifact.' + required: false + default: 'report/typing' + type: string + cobertura_report_file: + description: 'Cobertura file to upload as an artifact.' + required: false + default: 'cobertura.xml' + type: string html_artifact: description: 'Name of the typing artifact (HTML report).' required: false @@ -60,6 +80,11 @@ on: required: false default: '' type: string + cobertura_artifact: + description: 'Name of the typing cobertura artifact (Cobertura XML).' + required: false + default: '' + type: string jobs: StaticTypeCheck: @@ -80,7 +105,40 @@ jobs: - name: Check Static Typing continue-on-error: true - run: ${{ inputs.commands }} + run: mypy ${{ inputs.mypy_options }} + + - name: Debug output directories + continue-on-error: true + run: | + # List directory contents + set +e + + ANSI_LIGHT_RED=$'\x1b[91m' + ANSI_LIGHT_GREEN=$'\x1b[92m' + ANSI_LIGHT_YELLOW=$'\x1b[93m' + ANSI_LIGHT_BLUE=$'\x1b[94m' + ANSI_NOCOLOR=$'\x1b[0m' + + if [[ "${{ inputs.html_report }}" != "" ]]; then + printf "::group::${ANSI_LIGHT_BLUE}%s${ANSI_NOCOLOR}\n" "Content of '${{ inputs.html_report }}' ..." + tree ${{ inputs.html_report }} + printf "::endgroup::\n" + fi + + if [[ "${{ inputs.junit_report_directory }}" != "" ]]; then + printf "::group::${ANSI_LIGHT_BLUE}%s${ANSI_NOCOLOR}\n" "Content of '${{ inputs.junit_report_directory }}' ..." + tree ${{ inputs.junit_report_directory }} + printf "::endgroup::\n" + if [[ "${{ inputs.cobertura_report_directory }}" != "" && "${{ inputs.junit_report_directory }}" != "${{ inputs.cobertura_report_directory }}" ]]; then + printf "::group::${ANSI_LIGHT_BLUE}%s${ANSI_NOCOLOR}\n" "Content of '${{ inputs.cobertura_report_directory }}' ..." + tree ${{ inputs.cobertura_report_directory }} + printf "::endgroup::\n" + fi + elif [[ "${{ inputs.cobertura_report_directory }}" != "" ]]; then + printf "::group::${ANSI_LIGHT_BLUE}%s${ANSI_NOCOLOR}\n" "Content of '${{ inputs.cobertura_report_directory }}' ..." + tree ${{ inputs.cobertura_report_directory }} + printf "::endgroup::\n" + fi - name: 📤 Upload 'Static Typing Report' HTML artifact uses: pyTooling/upload-artifact@v4 @@ -99,6 +157,18 @@ jobs: continue-on-error: true with: name: ${{ inputs.junit_artifact }} - path: ${{ inputs.junit_report }} + working-directory: ${{ inputs.junit_report_directory }} + path: ${{ inputs.junit_report_file }} + if-no-files-found: error + retention-days: 1 + + - name: 📤 Upload 'Static Typing Report' Cobertura artifact + uses: pyTooling/upload-artifact@v4 + if: ${{ inputs.cobertura_artifact != '' }} + continue-on-error: true + with: + name: ${{ inputs.cobertura_artifact }} + working-directory: ${{ inputs.cobertura_report_directory }} + path: ${{ inputs.cobertura_report_file }} if-no-files-found: error retention-days: 1 diff --git a/pyproject.toml b/pyproject.toml index fa426fd..8f7ecf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,15 +10,17 @@ build-backend = "setuptools.build_meta" line-length = 120 [tool.mypy] -packages = ["myPackage", "myFramework"] +packages = ["myPackage", "myFramework.Extension"] python_version = "3.13" -#ignore_missing_imports = true strict = true pretty = true show_error_context = true show_error_codes = true namespace_packages = true -html_report = "report/typing" +follow_untyped_imports = true +html_report = "report/typing/html" +junit_xml = "report/typing/StaticTypingSummary.xml" +cobertura_xml_report = "report/typing" [tool.pytest] junit_xml = "report/unit/UnittestReportSummary.xml" diff --git a/tests/requirements.txt b/tests/requirements.txt index 000aea4..c8b94c7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -8,6 +8,6 @@ pytest ~= 8.4 pytest-cov ~= 7.0 # Static Type Checking -mypy ~= 1.17 +mypy[reports] ~= 1.18 typing_extensions ~= 4.15 lxml ~= 6.0