From 05e5d1f86c9fc034c969c35c3012ef61504728cc Mon Sep 17 00:00:00 2001 From: Patrick Lehmann Date: Sat, 4 Oct 2025 21:30:24 +0200 Subject: [PATCH 1/3] Improved pyproject.toml reading if settings don't exist. --- .github/workflows/ExtractConfiguration.yml | 38 ++++++++++---- .github/workflows/Parameters.yml | 2 +- .github/workflows/PrepareJob.yml | 59 ++++++++++++++++++---- 3 files changed, 80 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ExtractConfiguration.yml b/.github/workflows/ExtractConfiguration.yml index e32bc17..a52b043 100644 --- a/.github/workflows/ExtractConfiguration.yml +++ b/.github/workflows/ExtractConfiguration.yml @@ -114,7 +114,7 @@ jobs: coverageRC = "${{ inputs.coverage_config }}".strip() typingCoberturaFile = Path("report/typing/cobertura.xml") typingJUnitFile = Path("report/typing/StaticTypingSummary.xml") - typingHTMLDirectory = Path("htmlmypy") + typingHTMLDirectory = Path("report/typing/html") # Read output paths from 'pyproject.toml' file if coverageRC == "pyproject.toml": @@ -123,14 +123,34 @@ jobs: with pyProjectFile.open("rb") as file: pyProjectSettings = tomli_load(file) - unittestXMLFile = Path(pyProjectSettings["tool"]["pytest"]["junit_xml"]) - mergedUnittestXMLFile = Path(pyProjectSettings["tool"]["pyedaa-reports"]["junit_xml"]) - coverageHTMLDirectory = Path(pyProjectSettings["tool"]["coverage"]["html"]["directory"]) - coverageXMLFile = Path(pyProjectSettings["tool"]["coverage"]["xml"]["output"]) - coverageJSONFile= Path(pyProjectSettings["tool"]["coverage"]["json"]["output"]) - typingCoberturaFile = Path(pyProjectSettings["tool"]["mypy"]["cobertura_xml_report"]) / "cobertura.xml" - typingJUnitFile = Path(pyProjectSettings["tool"]["mypy"]["junit_xml"]) - typingHTMLDirectory = Path(pyProjectSettings["tool"]["mypy"]["html_report"]) + toolSection = pyProjectSettings["tool"] + if "pytest" in toolSection: + section = toolSection["pytest"] + if "junit_xml" in section: + unittestXMLFile = Path(section["junit_xml"]) + + if "pyedaa-reports" in toolSection: + section = toolSection["pyedaa-reports"] + if "junit_xml" in section: + mergedUnittestXMLFile = Path(section["junit_xml"]) + + if "coverage" in toolSection: + section = toolSection["coverage"] + if "html" in section: + coverageHTMLDirectory = Path(section["html"]["directory"]) + if "xml" in section: + coverageXMLFile = Path(section["xml"]["output"]) + if "json" in section: + coverageJSONFile= Path(section["json"]["output"]) + + if "mypy" in toolSection: + section = toolSection["mypy"] + if "cobertura_xml_report" in section: + typingCoberturaFile = Path(section["cobertura_xml_report"]) / "cobertura.xml" + if "junit_xml" in section: + typingJUnitFile = Path(section["junit_xml"]) + if "html_report" in section: + typingHTMLDirectory = Path(section["html_report"]) else: print(f"File '{pyProjectFile}' not found.") print(f"::error title=FileNotFoundError::File '{pyProjectFile}' not found.") diff --git a/.github/workflows/Parameters.yml b/.github/workflows/Parameters.yml index c7eadd6..5d6b91f 100644 --- a/.github/workflows/Parameters.yml +++ b/.github/workflows/Parameters.yml @@ -164,7 +164,7 @@ jobs: package_name = "${{ inputs.package_name }}".strip() name = "${{ inputs.name }}".strip() - if package_namespace == "": # or package_namespace == ".": + if package_namespace == "": package_fullname = package_name package_directory = package_name elif package_namespace[-2:] == ".*": diff --git a/.github/workflows/PrepareJob.yml b/.github/workflows/PrepareJob.yml index 1cbdbf3..517a0f0 100644 --- a/.github/workflows/PrepareJob.yml +++ b/.github/workflows/PrepareJob.yml @@ -191,13 +191,14 @@ jobs: printf "${ANSI_LIGHT_GREEN} [OK]\n" default_branch="${defaultBranch}" - printf " default_branch=%s\n" "${default_branch}" + printf " Default branch ${ANSI_LIGHT_BLUE}'%s'${ANSI_NOCOLOR}\n" "${default_branch}" else printf "${ANSI_LIGHT_RED} [FAILED]\n" - printf " %s\n" "${default_branch}" + printf " ${ANSI_LIGHT_RED}%s${ANSI_NOCOLOR}\n" "${default_branch}" fi printf "Commit checks:\n" + printf " Commit: %s\n" "${{ github.sha }}" printf " Commit kind " if [[ -z "$(git rev-list -1 --merges ${{ github.sha }}~1..${{ github.sha }})" ]]; then is_regular_commit="true" @@ -208,24 +209,37 @@ jobs: fi printf "Branch checks:\n" + printf " Branch: %s\n" "${branch}" + printf " Commit on default branch ${ANSI_LIGHT_BLUE}'%s'${ANSI_NOCOLOR} " "${defaultBranch}" if [[ "${branch}" == "${defaultBranch}" ]]; then on_default_branch="true" - printf " Commit on default branch ${ANSI_LIGHT_BLUE}'%s'${ANSI_NOCOLOR}\n" "${defaultBranch}" + printf "${ANSI_LIGHT_GREEN}[YES]${ANSI_NOCOLOR}\n" + else + printf "${ANSI_LIGHT_RED}[NO]${ANSI_NOCOLOR}\n" fi + printf " Commit on main branch ${ANSI_LIGHT_BLUE}'%s'${ANSI_NOCOLOR} " "${{ inputs.main_branch }}" if [[ "${branch}" == "${{ inputs.main_branch }}" ]]; then on_main_branch="true" - printf " Commit on main branch ${ANSI_LIGHT_BLUE}'%s'${ANSI_NOCOLOR}\n" "${{ inputs.main_branch }}" + printf "${ANSI_LIGHT_GREEN}[YES]${ANSI_NOCOLOR}\n" + else + printf "${ANSI_LIGHT_RED}[NO]${ANSI_NOCOLOR}\n" fi + printf " Commit on release branch ${ANSI_LIGHT_BLUE}'%s'${ANSI_NOCOLOR} " "${{ inputs.release_branch }}" if [[ "${branch}" == "${{ inputs.release_branch }}" ]]; then on_release_branch="true" - printf " Commit on release branch ${ANSI_LIGHT_BLUE}'%s'${ANSI_NOCOLOR}\n" "${{ inputs.release_branch }}" + printf "${ANSI_LIGHT_GREEN}[YES]${ANSI_NOCOLOR}\n" + else + printf "${ANSI_LIGHT_RED}[NO]${ANSI_NOCOLOR}\n" fi + printf " Commit on development branch ${ANSI_LIGHT_BLUE}'%s'${ANSI_NOCOLOR} " "${{ inputs.development_branch }}" if [[ "${branch}" == "${{ inputs.development_branch }}" ]]; then on_dev_branch="true" - printf " Commit on development branch ${ANSI_LIGHT_BLUE}'%s'${ANSI_NOCOLOR}\n" "${{ inputs.development_branch }}" + printf "${ANSI_LIGHT_GREEN}[YES]${ANSI_NOCOLOR}\n" + else + printf "${ANSI_LIGHT_RED}[NO]${ANSI_NOCOLOR}\n" fi if [[ "${is_merge_commit}" == "true" ]]; then @@ -261,11 +275,14 @@ jobs: NIGHTLY_TAG_PATTERN='^${{ inputs.nightly_tag_pattern }}$' RELEASE_TAG_PATTERN='^${{ inputs.release_tag_pattern }}$' - printf " Check tag name against regexp '%s' ... " "${RELEASE_TAG_PATTERN}" - if [[ "${tag}" =~ NIGHTLY_TAG_PATTERN ]]; then + + printf "Tag checks:\n" + printf " Tag: %s\n" "${tag}" + printf " Check tag '%s' against regexp ... " "${tag}" + if [[ "${tag}" =~ ${NIGHTLY_TAG_PATTERN} ]]; then printf "${ANSI_LIGHT_GREEN}[NIGHTLY]${ANSI_NOCOLOR}\n" is_nightly_tag="true" - elif [[ "${tag}" =~ $RELEASE_TAG_PATTERN ]]; then + elif [[ "${tag}" =~ ${RELEASE_TAG_PATTERN} ]]; then printf "${ANSI_LIGHT_GREEN}[RELEASE]${ANSI_NOCOLOR}\n" version="${tag}" is_release_tag="true" @@ -277,6 +294,30 @@ jobs: printf "::error title=RexExpCheck::Tag name '%s' doesn't conform to regexp '%s' nor '%s'.\n" "${tag}" "${NIGHTLY_TAG_PATTERN}" "${RELEASE_TAG_PATTERN}" exit 1 fi + + if [[ "${is_nightly_tag}" == "true" ]]; then + printf " Check if nightly tag is on main branch '%s' ... " "${{ inputs.main_branch }}" + git branch --remotes --contains $(git rev-parse --verify "tags/${tag}~0") | grep "origin/${{ inputs.main_branch }}" > /dev/null + if [[ $? -eq 0 ]]; then + printf "${ANSI_LIGHT_GREEN}[OK]${ANSI_NOCOLOR}\n" + else + printf "${ANSI_LIGHT_RED}[FAILED]${ANSI_NOCOLOR}\n" + printf " ${ANSI_LIGHT_RED}Tag '%s' isn't on branch '%s'.${ANSI_NOCOLOR}\n" "${tag}" "${{ inputs.main_branch }}" + printf "::error title=TagCheck::Tag '%s' isn't on branch '%s'.\n" "${tag}" "${{ inputs.main_branch }}" + exit 1 + fi + elif [[ "${is_release_tag}" == "true" ]]; then + printf " Check if release tag is on main branch '%s' ... " "${{ inputs.main_branch }}" + git branch --remotes --contains $(git rev-parse --verify "tags/${tag}~0") | grep "origin/${{ inputs.main_branch }}" > /dev/null + if [[ $? -eq 0 ]]; then + printf "${ANSI_LIGHT_GREEN}[OK]${ANSI_NOCOLOR}\n" + else + printf "${ANSI_LIGHT_RED}[FAILED]${ANSI_NOCOLOR}\n" + printf " ${ANSI_LIGHT_RED}Tag '%s' isn't on branch '%s'.${ANSI_NOCOLOR}\n" "${tag}" "${{ inputs.main_branch }}" + printf "::error title=TagCheck::Tag '%s' isn't on branch '%s'.\n" "${tag}" "${{ inputs.main_branch }}" + exit 1 + fi + fi elif [[ "${ref:0:10}" == "refs/pull/" ]]; then printf "${ANSI_LIGHT_YELLOW}[PULL REQUEST]\n" ref_kind="pullrequest" From 953d0698c96414fa821aaee85cd186fb553ac5a1 Mon Sep 17 00:00:00 2001 From: Patrick Lehmann Date: Sat, 4 Oct 2025 23:48:00 +0200 Subject: [PATCH 2/3] Fixed wheel package installations on Windows. --- .github/workflows/ApplicationTesting.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ApplicationTesting.yml b/.github/workflows/ApplicationTesting.yml index 1c3ef4d..b8d94e4 100644 --- a/.github/workflows/ApplicationTesting.yml +++ b/.github/workflows/ApplicationTesting.yml @@ -220,11 +220,16 @@ jobs: python -m pip install --disable-pip-version-check ${{ inputs.requirements }} fi - - name: 🔧 Install wheel from artifact + - name: 🔧 Install wheel from artifact (Ubuntu/macOS) + if: ( matrix.system != 'windows' && matrix.system != 'windows-arm' ) run: | - ls -l install python -m pip install --disable-pip-version-check -U install/*.whl + - name: 🔧 Install wheel from artifact (Windows) + if: ( matrix.system == 'windows' || matrix.system == 'windows-arm' ) + run: | + python -m pip install -v --disable-pip-version-check (Get-Item .\install\*.whl).FullName + - name: ✅ Run application tests (Ubuntu/macOS) if: ( matrix.system != 'windows' && matrix.system != 'windows-arm' ) run: | From fc08112235cf905aec63c808a71b51653fbba310 Mon Sep 17 00:00:00 2001 From: Patrick Lehmann Date: Sun, 5 Oct 2025 15:14:39 +0200 Subject: [PATCH 3/3] Added parameters to enable bandit and pylint checks. --- .github/workflows/CompletePipeline.yml | 13 ++++++++++++- dist/requirements.txt | 2 +- doc/index.rst | 2 +- doc/requirements.txt | 6 +++--- myPackage/__init__.py | 1 + pyproject.toml | 3 ++- requirements.txt | 2 +- 7 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.github/workflows/CompletePipeline.yml b/.github/workflows/CompletePipeline.yml index bd07107..b7cecf0 100644 --- a/.github/workflows/CompletePipeline.yml +++ b/.github/workflows/CompletePipeline.yml @@ -93,6 +93,16 @@ on: required: false default: 'windows-arm:pypy-3.10 windows-arm:pypy-3.11' type: string + bandit: + description: 'Run Static Application Security Testing (SAST) using Bandit.' + required: false + default: 'false' + type: string + pylint: + description: 'Run Python linting using pylint.' + required: false + default: 'false' + type: string codecov: description: 'Publish merged coverage and unittest reports to Codecov.' required: false @@ -205,6 +215,8 @@ jobs: with: python_version: ${{ needs.UnitTestingParams.outputs.python_version }} package_directory: ${{ needs.UnitTestingParams.outputs.package_directory }} + bandit: ${{ inputs.bandit }} + pylint: ${{ inputs.pylint }} artifact: CodeQuality DocCoverage: @@ -219,7 +231,6 @@ jobs: uses: pyTooling/Actions/.github/workflows/Package.yml@dev needs: - UnitTestingParams -# - UnitTesting with: python_version: ${{ needs.UnitTestingParams.outputs.python_version }} artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).package_all }} diff --git a/dist/requirements.txt b/dist/requirements.txt index 778498a..cacbc6f 100644 --- a/dist/requirements.txt +++ b/dist/requirements.txt @@ -1,2 +1,2 @@ wheel ~= 0.45 -twine ~= 6.1 +twine ~= 6.2 diff --git a/doc/index.rst b/doc/index.rst index eb1c3b2..0242c18 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -164,7 +164,7 @@ Example Pipelines .. code-block:: toml [build-system] - requires = ["setuptools >= 80.0", "wheel ~= 0.45", "pyTooling ~= 8.5"] + requires = ["setuptools >= 80.0", "wheel ~= 0.45", "pyTooling ~= 8.7"] build-backend = "setuptools.build_meta" [tool.mypy] diff --git a/doc/requirements.txt b/doc/requirements.txt index 709b41b..63b74c1 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,6 +1,6 @@ -r ../requirements.txt -pyTooling ~= 8.5 +pyTooling ~= 8.7 # Enforce latest version on ReadTheDocs sphinx ~= 8.2 @@ -13,7 +13,7 @@ sphinx_rtd_theme ~= 3.0 # Sphinx Extenstions sphinxcontrib-mermaid ~= 1.0 autoapi >= 2.0.1 -sphinx_design ~= 0.6.1 -sphinx-copybutton >= 0.5.2 +sphinx_design ~= 0.6 +sphinx-copybutton >= 0.5 sphinx_autodoc_typehints ~= 3.2 sphinx_reports ~= 0.9 diff --git a/myPackage/__init__.py b/myPackage/__init__.py index 9ea7328..e545d62 100644 --- a/myPackage/__init__.py +++ b/myPackage/__init__.py @@ -40,6 +40,7 @@ __version__ = "0.4.5" __keywords__ = ["GitHub Actions"] __issue_tracker__ = "https://GitHub.com/pyTooling/Actions/issues" +from pickle import dumps from subprocess import check_call from pyTooling.Decorators import export, readonly diff --git a/pyproject.toml b/pyproject.toml index 6388d1c..8108040 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,13 +2,14 @@ requires = [ "setuptools >= 80.0", "wheel ~= 0.45", - "pyTooling ~= 8.5" + "pyTooling ~= 8.7" ] build-backend = "setuptools.build_meta" [tool.pylint.format] indent-string="\t" max-line-length = 120 +ignore-long-lines = "^.{0,110}#: .*" [tool.pylint.basic] argument-naming-style = "camelCase" diff --git a/requirements.txt b/requirements.txt index 3781d6d..4fcd219 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -pyTooling ~= 8.5 +pyTooling ~= 8.7