MINIFICPP-2870 Lint/Check python code with ruff

.github/github_scripts/github_actions_cache_cleanup.py

@@ -1,21 +1,22 @@
 #!/bin/python3
 
-import requests
-import re
 import argparse
 import logging
-from typing import Dict, List
+import re
+
+import requests
 
+logger = logging.getLogger(__name__)
 
 logging.basicConfig(level=logging.INFO)
 
 
 class GithubRequestSender:
     def __init__(self, token: str, repository: str):
         self.headers = {
-            'Accept': 'application/vnd.github+json',
-            'Authorization': 'Bearer ' + token,
-            'X-GitHub-Api-Version': '2022-11-28'
+            "Accept": "application/vnd.github+json",
+            "Authorization": "Bearer " + token,
+            "X-GitHub-Api-Version": "2022-11-28",
         }
         self.repository = repository
 
@@ -25,7 +26,7 @@ def _send_get_request(self, url: str):
 
         return response.json()
 
-    def _send_delete_request(self, url: str, params: Dict[str, str]):
+    def _send_delete_request(self, url: str, params: dict[str, str]):
         response = requests.delete(url, headers=self.headers, params=params)
         response.raise_for_status()
 
@@ -58,24 +59,29 @@ class GithubActionsCacheCleaner:
     def __init__(self, token: str, repository: str):
         self.github_request_sender = GithubRequestSender(token, repository)
 
-    def _list_open_pr_ids(self) -> List[str]:
+    def _list_open_pr_ids(self) -> list[str]:
         open_tickets = []
         for request in self.github_request_sender.list_open_pull_requests():
-            open_tickets.append(request['number'])
+            open_tickets.append(request["number"])
         return open_tickets
 
-    def _get_cache_entries(self) -> List[CacheEntry]:
+    def _get_cache_entries(self) -> list[CacheEntry]:
         entries = []
         json_result = self.github_request_sender.list_caches()
         for json_entry in json_result["actions_caches"]:
             entries.append(CacheEntry(int(json_entry["id"]), json_entry["key"]))
         return entries
 
     def _is_pr_already_closed(self, entry, open_tickets):
-        match = re.search(r'refs/pull/([\d]+)/merge', entry.key)
+        match = re.search(r"refs/pull/([\d]+)/merge", entry.key)
         return match and int(match[1]) not in open_tickets
 
-    def _remove_non_latest_branch_caches(self, entry: CacheEntry, latest_branch_cache_map: Dict[str, CacheEntry], removable_entries: List[str]):
+    def _remove_non_latest_branch_caches(
+        self,
+        entry: CacheEntry,
+        latest_branch_cache_map: dict[str, CacheEntry],
+        removable_entries: list[str],
+    ):
         cache_mapping_key = "-".join(entry.key.split("-")[0:-1])
         if cache_mapping_key not in latest_branch_cache_map:
             latest_branch_cache_map[cache_mapping_key] = entry
@@ -86,32 +92,44 @@ def _remove_non_latest_branch_caches(self, entry: CacheEntry, latest_branch_cach
             else:
                 removable_entries.append(entry.key)
 
-    def _get_removable_cache_entries(self) -> List[str]:
+    def _get_removable_cache_entries(self) -> list[str]:
         removable_entries = []
-        latest_branch_cache_map = dict()
+        latest_branch_cache_map = {}
         open_prs = self._list_open_pr_ids()
         for entry in self._get_cache_entries():
             if self._is_pr_already_closed(entry, open_prs):
                 removable_entries.append(entry.key)
                 continue
 
-            self._remove_non_latest_branch_caches(entry, latest_branch_cache_map, removable_entries)
+            self._remove_non_latest_branch_caches(
+                entry, latest_branch_cache_map, removable_entries
+            )
 
         return removable_entries
 
-    def _remove_cache_entries(self, entries_to_remove: List[str]):
+    def _remove_cache_entries(self, entries_to_remove: list[str]):
         for key in entries_to_remove:
-            logging.info('Removing cache entry: %s', key)
+            logger.info("Removing cache entry: %s", key)
             self.github_request_sender.delete_cache(key)
 
     def remove_obsolete_cache_entries(self):
         self._remove_cache_entries(self._get_removable_cache_entries())
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(prog='Github Actions Cache Cleaner', description='Cleans up obsolete cache entries of github actions')
-    parser.add_argument('-r', '--repository', help='The name of the repository in <owner>/<repository> format', required=True)
-    parser.add_argument('-t', '--token', help='Github token for API access', required=True)
+    parser = argparse.ArgumentParser(
+        prog="Github Actions Cache Cleaner",
+        description="Cleans up obsolete cache entries of github actions",
+    )
+    parser.add_argument(
+        "-r",
+        "--repository",
+        help="The name of the repository in <owner>/<repository> format",
+        required=True,
+    )
+    parser.add_argument(
+        "-t", "--token", help="Github token for API access", required=True
+    )
     args = parser.parse_args()
     cache_cleaner = GithubActionsCacheCleaner(args.token, args.repository)
     cache_cleaner.remove_obsolete_cache_entries()

.github/github_scripts/github_actions_cache_cleanup_tests.py

@@ -2,6 +2,7 @@
 
 import unittest
 from unittest.mock import MagicMock
+
 from github_actions_cache_cleanup import GithubActionsCacheCleaner
 
 
@@ -21,7 +22,7 @@ def create_mock_github_request_sender(self):
             {
                 "number": 229,
                 "title": "MINIFICPP-123 TEST3",
-            }
+            },
         ]
         mock.list_open_pull_requests.return_value = open_pull_requests
         caches = {
@@ -57,7 +58,7 @@ def create_mock_github_request_sender(self):
                 {
                     "id": 55555,
                     "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567",
-                }
+                },
             ]
         }
         mock.list_caches = MagicMock()
@@ -102,7 +103,7 @@ def create_empty_open_pr_mock_github_request_sender(self):
                 {
                     "id": 55555,
                     "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567",
-                }
+                },
             ]
         }
         mock.list_caches = MagicMock()
@@ -125,13 +126,11 @@ def create_empty_caches_mock_github_request_sender(self):
             {
                 "number": 229,
                 "title": "MINIFICPP-123 TEST3",
-            }
+            },
         ]
         mock.list_open_pull_requests.return_value = open_pull_requests
         mock.list_caches = MagicMock()
-        caches = {
-            "actions_caches": []
-        }
+        caches = {"actions_caches": []}
         mock.list_caches.return_value = caches
         mock.delete_cache = MagicMock()
         return mock
@@ -140,30 +139,54 @@ def test_cache_cleanup(self):
         cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp")
         cleaner.github_request_sender = self.create_mock_github_request_sender()
         cleaner.remove_obsolete_cache_entries()
-        self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]),
-                         {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123",
-                          "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456",
-                          "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
-                          "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"})
+        self.assertEqual(
+            {
+                call[0][0]
+                for call in cleaner.github_request_sender.delete_cache.call_args_list
+            },
+            {
+                "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123",
+                "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456",
+                "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
+                "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456",
+            },
+        )
 
     def test_cache_cleanup_with_zero_open_prs(self):
         cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp")
-        cleaner.github_request_sender = self.create_empty_open_pr_mock_github_request_sender()
+        cleaner.github_request_sender = (
+            self.create_empty_open_pr_mock_github_request_sender()
+        )
         cleaner.remove_obsolete_cache_entries()
-        self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]),
-                         {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123",
-                          "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
-                          "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
-                          "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536",
-                          "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456",
-                          "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"})
+        self.assertEqual(
+            {
+                call[0][0]
+                for call in cleaner.github_request_sender.delete_cache.call_args_list
+            },
+            {
+                "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123",
+                "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
+                "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
+                "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536",
+                "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456",
+                "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456",
+            },
+        )
 
     def test_cache_cleanup_with_zero_action_caches(self):
         cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp")
-        cleaner.github_request_sender = self.create_empty_caches_mock_github_request_sender()
+        cleaner.github_request_sender = (
+            self.create_empty_caches_mock_github_request_sender()
+        )
         cleaner.remove_obsolete_cache_entries()
-        self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), set())
+        self.assertEqual(
+            {
+                call[0][0]
+                for call in cleaner.github_request_sender.delete_cache.call_args_list
+            },
+            set(),
+        )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()

.github/workflows/ci.yml

@@ -603,15 +603,15 @@ jobs:
           name: minifi_rs_behave
           path: minifi_rust/minifi_rs_behave/output
   linters:
-    name: "C++ lint + Shellcheck + Flake8 + Cargo fmt check + Clippy check"
-    runs-on: ubuntu-22.04-arm
+    name: "C++ lint + Shellcheck + Ruff + Cargo fmt check + Clippy check"
+    runs-on: ubuntu-24.04-arm
     timeout-minutes: 15
     steps:
       - id: checkout
         uses: actions/checkout@v6
 
       - id: install_deps
-        run: sudo apt update && sudo apt install -y flake8
+        run: pipx install ruff
 
       - id: cpp_lint
         name: C++ linter
@@ -623,10 +623,10 @@ jobs:
         continue-on-error: true
         run: ./run_shellcheck.sh .
 
-      - id: flake8_check
-        name: Flake8 check
+      - id: python_check
+        name: Python check
         continue-on-error: true
-        run: ./run_flake8.sh .
+        run: ruff check && ruff format --check
 
       - id: cargo_fmt_check
         name: Cargo fmt check
@@ -641,7 +641,7 @@ jobs:
         working-directory: minifi_rust
 
       - name: Check Linter Statuses
-        if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' || steps.cargo_fmt_check.outcome == 'failure' || steps.clippy_check.outcome == 'failure'
+        if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.python_check.outcome == 'failure' || steps.cargo_fmt_check.outcome == 'failure' || steps.clippy_check.outcome == 'failure'
         run: |
           echo "One or more linters failed. Failing the workflow."
           exit 1

CMakeLists.txt

@@ -829,9 +829,6 @@ set_target_properties(linter PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD TRUE)
 if(NOT WIN32)
 add_custom_target(shellcheck
     COMMAND ${CMAKE_SOURCE_DIR}/run_shellcheck.sh ${CMAKE_SOURCE_DIR})
-
-add_custom_target(flake8
-    COMMAND ${CMAKE_SOURCE_DIR}/run_flake8.sh ${CMAKE_SOURCE_DIR})
 endif(NOT WIN32)
 
 feature_summary(WHAT ALL FILENAME ${CMAKE_BINARY_DIR}/all.log)

CONTRIBUTING.md

@@ -55,14 +55,28 @@ it doesn't contradict any of the Google Style Guide or the above exceptions.
 C++ is a powerful language and "with great power comes great responsibility". Please aim for simple, readable and
 maintainable solutions and avoid footguns. Be open for respectful debate in pull request reviews.
 
+### Shell scripts
+
 Shell script files shall follow the guidelines and best practices defined by the [shellcheck](https://github.com/koalaman/shellcheck) analysis tool.
 New contributions are expected to pass the shellcheck analysis as part of the verification process.
 If a shellcheck requested change is unfeasible it shall be disabled on per-line basis and will be subjected to review.
 For more information on an issue please check the [shellcheck wiki page](https://github.com/koalaman/shellcheck/wiki).
-You can run shellcheck by invoking the shellcheck cmake target, e.g. `make shellcheck`.
+You can run shellcheck by invoking the shellcheck cmake target.
+```bash
+make shellcheck
+```
 
-Python script files shall follow the PEP8 guidelines and best practices. The project includes [flake8](https://flake8.pycqa.org/en/latest/) checks
+### Python
+Python script files shall follow the PEP8 guidelines and best practices. The project includes [ruff](https://docs.astral.sh/ruff/) checks
 as part of the verification process, that is applied to all new contributions.
+You can run the linter on Python code via:
+```bash
+ruff check
+```
+You can also auto-format the code via 
+```bash
+ruff format
+```
 
 ## Issues and Pull Requests
 
@@ -127,4 +141,3 @@ register_extension(minifi-kubernetes-extensions "KUBERNETES EXTENSIONS" KUBERNET
 # the next three arguments are used for documentation purposes
 # the fifth optional argument designates the directory of the extension's tests
 ```
-