diff --git a/.github/github_scripts/github_actions_cache_cleanup.py b/.github/github_scripts/github_actions_cache_cleanup.py index 8e70e95383..a82e71fdc3 100755 --- a/.github/github_scripts/github_actions_cache_cleanup.py +++ b/.github/github_scripts/github_actions_cache_cleanup.py @@ -1,11 +1,12 @@ #!/bin/python3 -import requests -import re import argparse import logging -from typing import Dict, List +import re + +import requests +logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -13,9 +14,9 @@ class GithubRequestSender: def __init__(self, token: str, repository: str): self.headers = { - 'Accept': 'application/vnd.github+json', - 'Authorization': 'Bearer ' + token, - 'X-GitHub-Api-Version': '2022-11-28' + "Accept": "application/vnd.github+json", + "Authorization": "Bearer " + token, + "X-GitHub-Api-Version": "2022-11-28", } self.repository = repository @@ -25,7 +26,7 @@ def _send_get_request(self, url: str): return response.json() - def _send_delete_request(self, url: str, params: Dict[str, str]): + def _send_delete_request(self, url: str, params: dict[str, str]): response = requests.delete(url, headers=self.headers, params=params) response.raise_for_status() @@ -58,13 +59,13 @@ class GithubActionsCacheCleaner: def __init__(self, token: str, repository: str): self.github_request_sender = GithubRequestSender(token, repository) - def _list_open_pr_ids(self) -> List[str]: + def _list_open_pr_ids(self) -> list[str]: open_tickets = [] for request in self.github_request_sender.list_open_pull_requests(): - open_tickets.append(request['number']) + open_tickets.append(request["number"]) return open_tickets - def _get_cache_entries(self) -> List[CacheEntry]: + def _get_cache_entries(self) -> list[CacheEntry]: entries = [] json_result = self.github_request_sender.list_caches() for json_entry in json_result["actions_caches"]: @@ -72,10 +73,15 @@ def _get_cache_entries(self) -> List[CacheEntry]: return entries def _is_pr_already_closed(self, entry, open_tickets): - match = re.search(r'refs/pull/([\d]+)/merge', entry.key) + match = re.search(r"refs/pull/([\d]+)/merge", entry.key) return match and int(match[1]) not in open_tickets - def _remove_non_latest_branch_caches(self, entry: CacheEntry, latest_branch_cache_map: Dict[str, CacheEntry], removable_entries: List[str]): + def _remove_non_latest_branch_caches( + self, + entry: CacheEntry, + latest_branch_cache_map: dict[str, CacheEntry], + removable_entries: list[str], + ): cache_mapping_key = "-".join(entry.key.split("-")[0:-1]) if cache_mapping_key not in latest_branch_cache_map: latest_branch_cache_map[cache_mapping_key] = entry @@ -86,22 +92,24 @@ def _remove_non_latest_branch_caches(self, entry: CacheEntry, latest_branch_cach else: removable_entries.append(entry.key) - def _get_removable_cache_entries(self) -> List[str]: + def _get_removable_cache_entries(self) -> list[str]: removable_entries = [] - latest_branch_cache_map = dict() + latest_branch_cache_map = {} open_prs = self._list_open_pr_ids() for entry in self._get_cache_entries(): if self._is_pr_already_closed(entry, open_prs): removable_entries.append(entry.key) continue - self._remove_non_latest_branch_caches(entry, latest_branch_cache_map, removable_entries) + self._remove_non_latest_branch_caches( + entry, latest_branch_cache_map, removable_entries + ) return removable_entries - def _remove_cache_entries(self, entries_to_remove: List[str]): + def _remove_cache_entries(self, entries_to_remove: list[str]): for key in entries_to_remove: - logging.info('Removing cache entry: %s', key) + logger.info("Removing cache entry: %s", key) self.github_request_sender.delete_cache(key) def remove_obsolete_cache_entries(self): @@ -109,9 +117,19 @@ def remove_obsolete_cache_entries(self): if __name__ == "__main__": - parser = argparse.ArgumentParser(prog='Github Actions Cache Cleaner', description='Cleans up obsolete cache entries of github actions') - parser.add_argument('-r', '--repository', help='The name of the repository in / format', required=True) - parser.add_argument('-t', '--token', help='Github token for API access', required=True) + parser = argparse.ArgumentParser( + prog="Github Actions Cache Cleaner", + description="Cleans up obsolete cache entries of github actions", + ) + parser.add_argument( + "-r", + "--repository", + help="The name of the repository in / format", + required=True, + ) + parser.add_argument( + "-t", "--token", help="Github token for API access", required=True + ) args = parser.parse_args() cache_cleaner = GithubActionsCacheCleaner(args.token, args.repository) cache_cleaner.remove_obsolete_cache_entries() diff --git a/.github/github_scripts/github_actions_cache_cleanup_tests.py b/.github/github_scripts/github_actions_cache_cleanup_tests.py index 2f2f7772e7..d6b410f0e4 100755 --- a/.github/github_scripts/github_actions_cache_cleanup_tests.py +++ b/.github/github_scripts/github_actions_cache_cleanup_tests.py @@ -2,6 +2,7 @@ import unittest from unittest.mock import MagicMock + from github_actions_cache_cleanup import GithubActionsCacheCleaner @@ -21,7 +22,7 @@ def create_mock_github_request_sender(self): { "number": 229, "title": "MINIFICPP-123 TEST3", - } + }, ] mock.list_open_pull_requests.return_value = open_pull_requests caches = { @@ -57,7 +58,7 @@ def create_mock_github_request_sender(self): { "id": 55555, "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", - } + }, ] } mock.list_caches = MagicMock() @@ -102,7 +103,7 @@ def create_empty_open_pr_mock_github_request_sender(self): { "id": 55555, "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", - } + }, ] } mock.list_caches = MagicMock() @@ -125,13 +126,11 @@ def create_empty_caches_mock_github_request_sender(self): { "number": 229, "title": "MINIFICPP-123 TEST3", - } + }, ] mock.list_open_pull_requests.return_value = open_pull_requests mock.list_caches = MagicMock() - caches = { - "actions_caches": [] - } + caches = {"actions_caches": []} mock.list_caches.return_value = caches mock.delete_cache = MagicMock() return mock @@ -140,30 +139,54 @@ def test_cache_cleanup(self): cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") cleaner.github_request_sender = self.create_mock_github_request_sender() cleaner.remove_obsolete_cache_entries() - self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), - {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", - "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", - "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", - "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) + self.assertEqual( + { + call[0][0] + for call in cleaner.github_request_sender.delete_cache.call_args_list + }, + { + "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", + "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", + "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", + }, + ) def test_cache_cleanup_with_zero_open_prs(self): cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") - cleaner.github_request_sender = self.create_empty_open_pr_mock_github_request_sender() + cleaner.github_request_sender = ( + self.create_empty_open_pr_mock_github_request_sender() + ) cleaner.remove_obsolete_cache_entries() - self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), - {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", - "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", - "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", - "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536", - "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", - "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) + self.assertEqual( + { + call[0][0] + for call in cleaner.github_request_sender.delete_cache.call_args_list + }, + { + "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", + "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", + "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536", + "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", + "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", + }, + ) def test_cache_cleanup_with_zero_action_caches(self): cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") - cleaner.github_request_sender = self.create_empty_caches_mock_github_request_sender() + cleaner.github_request_sender = ( + self.create_empty_caches_mock_github_request_sender() + ) cleaner.remove_obsolete_cache_entries() - self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), set()) + self.assertEqual( + { + call[0][0] + for call in cleaner.github_request_sender.delete_cache.call_args_list + }, + set(), + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c33fff8919..fb5b346570 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -603,15 +603,15 @@ jobs: name: minifi_rs_behave path: minifi_rust/minifi_rs_behave/output linters: - name: "C++ lint + Shellcheck + Flake8 + Cargo fmt check + Clippy check" - runs-on: ubuntu-22.04-arm + name: "C++ lint + Shellcheck + Ruff + Cargo fmt check + Clippy check" + runs-on: ubuntu-24.04-arm timeout-minutes: 15 steps: - id: checkout uses: actions/checkout@v6 - id: install_deps - run: sudo apt update && sudo apt install -y flake8 + run: pipx install ruff - id: cpp_lint name: C++ linter @@ -623,10 +623,10 @@ jobs: continue-on-error: true run: ./run_shellcheck.sh . - - id: flake8_check - name: Flake8 check + - id: python_check + name: Python check continue-on-error: true - run: ./run_flake8.sh . + run: ruff check && ruff format --check - id: cargo_fmt_check name: Cargo fmt check @@ -641,7 +641,7 @@ jobs: working-directory: minifi_rust - name: Check Linter Statuses - if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' || steps.cargo_fmt_check.outcome == 'failure' || steps.clippy_check.outcome == 'failure' + if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.python_check.outcome == 'failure' || steps.cargo_fmt_check.outcome == 'failure' || steps.clippy_check.outcome == 'failure' run: | echo "One or more linters failed. Failing the workflow." exit 1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 47993663f1..54d2c1e9dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -829,9 +829,6 @@ set_target_properties(linter PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD TRUE) if(NOT WIN32) add_custom_target(shellcheck COMMAND ${CMAKE_SOURCE_DIR}/run_shellcheck.sh ${CMAKE_SOURCE_DIR}) - -add_custom_target(flake8 - COMMAND ${CMAKE_SOURCE_DIR}/run_flake8.sh ${CMAKE_SOURCE_DIR}) endif(NOT WIN32) feature_summary(WHAT ALL FILENAME ${CMAKE_BINARY_DIR}/all.log) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1d25c6ca7..a0338b098b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,14 +55,28 @@ it doesn't contradict any of the Google Style Guide or the above exceptions. C++ is a powerful language and "with great power comes great responsibility". Please aim for simple, readable and maintainable solutions and avoid footguns. Be open for respectful debate in pull request reviews. +### Shell scripts + Shell script files shall follow the guidelines and best practices defined by the [shellcheck](https://github.com/koalaman/shellcheck) analysis tool. New contributions are expected to pass the shellcheck analysis as part of the verification process. If a shellcheck requested change is unfeasible it shall be disabled on per-line basis and will be subjected to review. For more information on an issue please check the [shellcheck wiki page](https://github.com/koalaman/shellcheck/wiki). -You can run shellcheck by invoking the shellcheck cmake target, e.g. `make shellcheck`. +You can run shellcheck by invoking the shellcheck cmake target. +```bash +make shellcheck +``` -Python script files shall follow the PEP8 guidelines and best practices. The project includes [flake8](https://flake8.pycqa.org/en/latest/) checks +### Python +Python script files shall follow the PEP8 guidelines and best practices. The project includes [ruff](https://docs.astral.sh/ruff/) checks as part of the verification process, that is applied to all new contributions. +You can run the linter on Python code via: +```bash +ruff check +``` +You can also auto-format the code via +```bash +ruff format +``` ## Issues and Pull Requests @@ -127,4 +141,3 @@ register_extension(minifi-kubernetes-extensions "KUBERNETES EXTENSIONS" KUBERNET # the next three arguments are used for documentation purposes # the fifth optional argument designates the directory of the extension's tests ``` - diff --git a/behave_framework/src/minifi_behave/containers/container_linux.py b/behave_framework/src/minifi_behave/containers/container_linux.py index da0dcbc0ab..4e5cbc7f58 100644 --- a/behave_framework/src/minifi_behave/containers/container_linux.py +++ b/behave_framework/src/minifi_behave/containers/container_linux.py @@ -23,22 +23,31 @@ import tarfile import tempfile import uuid +from typing import TYPE_CHECKING + from docker.models.networks import Network from minifi_behave.containers.container_protocol import ContainerProtocol from minifi_behave.containers.directory import Directory from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile -from typing import TYPE_CHECKING import docker +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from minifi_behave.core.minifi_test_context import MinifiTestContext class LinuxContainer(ContainerProtocol): - def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, - entrypoint: str | None = None): + def __init__( + self, + image_name: str, + container_name: str, + network: Network, + command: str | None = None, + entrypoint: str | None = None, + ): super().__init__() self.image_name: str = image_name self.container_name: str = container_name @@ -64,24 +73,28 @@ def add_host_file(self, host_path: str, container_path: str, mode: str = "ro"): def add_file_to_running_container(self, content: str, path: str): if not self.container: - logging.error("Container is not running. Cannot add file.") + logger.error("Container is not running. Cannot add file.") raise RuntimeError("Container is not running. Cannot add file.") mkdir_command = f"mkdir -p {shlex.quote(path)}" exit_code, output = self.exec_run(mkdir_command) if exit_code != 0: - logging.error(f"Error creating directory '{path}' in container: {output}") - raise RuntimeError(f"Error creating directory '{path}' in container: {output}") + logger.error(f"Error creating directory '{path}' in container: {output}") + raise RuntimeError( + f"Error creating directory '{path}' in container: {output}" + ) full_path = os.path.join(path, str(uuid.uuid4())) tmp_path = os.path.join("/tmp", str(uuid.uuid4())) pipe_command = f"printf %s {shlex.quote(content)} > {shlex.quote(tmp_path)} && mv {shlex.quote(tmp_path)} {shlex.quote(full_path)}" exit_code, output = self.exec_run(f"sh -c {shlex.quote(pipe_command)}") if exit_code != 0: - logging.error(f"Error adding file to running container: {output}") + logger.error(f"Error adding file to running container: {output}") raise RuntimeError(f"Error adding file to running container: {output}") - def _write_content_to_file(self, filepath: str, permissions: int | None, content: str | bytes): + def _write_content_to_file( + self, filepath: str, permissions: int | None, content: str | bytes + ): write_mode = "w" if isinstance(content, bytes): write_mode = "wb" @@ -105,35 +118,44 @@ def _configure_volumes_of_container_dirs(self): self._write_content_to_file(file_path, None, content) self.volumes[temp_path] = {"bind": directory.path, "mode": directory.mode} - def is_deployed(self) -> bool: - return self.container is not None - def deploy(self, context: MinifiTestContext | None) -> bool: if self.is_deployed(): - logging.info(f"Container '{self.container_name}' is already deployed.") + logger.info(f"Container '{self.container_name}' is already deployed.") return True self._temp_dir = tempfile.TemporaryDirectory() self._configure_volumes_of_container_files() self._configure_volumes_of_container_dirs() for host_file in self.host_files: - self.volumes[host_file.host_path] = {"bind": host_file.container_path, "mode": host_file.mode} + self.volumes[host_file.host_path] = { + "bind": host_file.container_path, + "mode": host_file.mode, + } try: existing_container = self.client.containers.get(self.container_name) - logging.warning(f"Found existing container '{self.container_name}'. Removing it first.") + logger.warning( + f"Found existing container '{self.container_name}'. Removing it first." + ) existing_container.remove(force=True) except docker.errors.NotFound: pass try: - logging.info(f"Creating and starting container '{self.container_name}'...") - self.container = self.client.containers.run(image=self.image_name, name=self.container_name, - ports=self.ports, environment=self.environment, - volumes=self.volumes, network=self.network.name, - command=self.command, entrypoint=self.entrypoint, - user=self.user, detach=True) + logger.info(f"Creating and starting container '{self.container_name}'...") + self.container = self.client.containers.run( + image=self.image_name, + name=self.container_name, + ports=self.ports, + environment=self.environment, + volumes=self.volumes, + network=self.network.name, + command=self.command, + entrypoint=self.entrypoint, + user=self.user, + detach=True, + ) except Exception as e: - logging.error(f"Error starting container: {e}") + logger.error(f"Error starting container: {e}") raise return True @@ -141,32 +163,34 @@ def start(self): if self.container: self.container.start() else: - logging.error("Container does not exist. Cannot start.") + logger.error("Container does not exist. Cannot start.") def stop(self): if self.container: self.container.stop() else: - logging.error("Container does not exist. Cannot stop.") + logger.error("Container does not exist. Cannot stop.") def kill(self): if self.container: self.container.kill() else: - logging.error("Container does not exist. Cannot kill.") + logger.error("Container does not exist. Cannot kill.") def restart(self): if self.container: self.container.restart() else: - logging.error("Container does not exist. Cannot restart.") + logger.error("Container does not exist. Cannot restart.") def clean_up(self): if self.container: try: self.container.remove(force=True) except Exception as e: - logging.error(f"Error cleaning up container '{self.container_name}': {e}") + logger.error( + f"Error cleaning up container '{self.container_name}': {e}" + ) def exec_run(self, command) -> tuple[int | None, str]: if self.container: @@ -177,67 +201,88 @@ def exec_run(self, command) -> tuple[int | None, str]: def nonempty_dir_exists(self, directory_path: str) -> bool: if not self.container: return False - dir_exists_exit_code, dir_exists_output = self.exec_run( - "sh -c {}".format(shlex.quote(f"test -d {directory_path}"))) + dir_exists_exit_code, _dir_exists_output = self.exec_run( + "sh -c {}".format(shlex.quote(f"test -d {directory_path}")) + ) if dir_exists_exit_code != 0: return False - dir_not_empty_ec, dir_not_empty_output = self.exec_run( - "sh -c {}".format(shlex.quote(f'[ "$(ls -A {directory_path})" ]'))) + dir_not_empty_ec, _dir_not_empty_output = self.exec_run( + "sh -c {}".format(shlex.quote(f'[ "$(ls -A {directory_path})" ]')) + ) return dir_not_empty_ec == 0 def directory_contains_empty_file(self, directory_path: str) -> bool: if not self.container or not self.nonempty_dir_exists(directory_path): return False - command = "sh -c {}".format(shlex.quote(f"find {directory_path} -maxdepth 1 -type f -empty")) + command = "sh -c {}".format( + shlex.quote(f"find {directory_path} -maxdepth 1 -type f -empty") + ) exit_code, _ = self.exec_run(command) return exit_code == 0 - def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool: + def directory_contains_file_with_content( + self, directory_path: str, expected_content: str + ) -> bool: if not self.container or not self.nonempty_dir_exists(directory_path): return False quoted_content = shlex.quote(expected_content) - command = "sh -c {}".format(shlex.quote(f"grep -l -F -- {quoted_content} {directory_path}/*")) + command = "sh -c {}".format( + shlex.quote(f"grep -l -F -- {quoted_content} {directory_path}/*") + ) exit_code, _ = self.exec_run(command) return exit_code == 0 - def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool: + def directory_contains_file_with_regex( + self, directory_path: str, regex_str: str + ) -> bool: if not self.container or not self.nonempty_dir_exists(directory_path): return False safe_dir_path = shlex.quote(directory_path) safe_regex_str = shlex.quote(regex_str) - command = (f"find {safe_dir_path} -maxdepth 1 -type f -print0 | " - f"xargs -0 -r grep -l -E -- {safe_regex_str}") + command = ( + f"find {safe_dir_path} -maxdepth 1 -type f -print0 | " + f"xargs -0 -r grep -l -E -- {safe_regex_str}" + ) - exit_code, output = self.exec_run("sh -c {}".format(shlex.quote(command))) + exit_code, output = self.exec_run(f"sh -c {shlex.quote(command)}") if exit_code != 0: - logging.debug("While looking for regex %s in directory %s, grep returned exit code %d, output: %s", - regex_str, directory_path, exit_code, output) + logger.debug( + "While looking for regex %s in directory %s, grep returned exit code %d, output: %s", + regex_str, + directory_path, + exit_code, + output, + ) return exit_code == 0 def path_with_content_exists(self, path: str, content: str) -> bool: count_command = f"sh -c 'cat {path} | grep \"^{content}$\" | wc -l'" exit_code, output = self.exec_run(count_command) if exit_code != 0: - logging.error(f"Error running command '{count_command}': {output}") + logger.error(f"Error running command '{count_command}': {output}") return False try: file_count = int(output.strip()) except (ValueError, IndexError): - logging.error(f"Error parsing output '{output}' from command '{count_command}'") + logger.error( + f"Error parsing output '{output}' from command '{count_command}'" + ) return False return file_count == 1 - def directory_has_single_file_with_content(self, directory_path: str, expected_content: str) -> bool: + def directory_has_single_file_with_content( + self, directory_path: str, expected_content: str + ) -> bool: if not self.container or not self.nonempty_dir_exists(directory_path): return False @@ -245,36 +290,40 @@ def directory_has_single_file_with_content(self, directory_path: str, expected_c exit_code, output = self.exec_run(count_command) if exit_code != 0: - logging.error(f"Error running command '{count_command}': {output}") + logger.error(f"Error running command '{count_command}': {output}") return False try: file_count = int(output.strip()) except (ValueError, IndexError): - logging.error(f"Error parsing output '{output}' from command '{count_command}'") + logger.error( + f"Error parsing output '{output}' from command '{count_command}'" + ) return False if file_count != 1: - logging.error(f"{directory_path} has too many or too few ({file_count}) files") + logger.error( + f"{directory_path} has too many or too few ({file_count}) files" + ) return False content_command = f"sh -c 'cat {directory_path}/*'" exit_code, output = self.exec_run(content_command) if exit_code != 0: - logging.error(f"Error running command '{content_command}': {output}") + logger.error(f"Error running command '{content_command}': {output}") return False actual_content = output.strip() - logging.debug(f"Comparing: '{actual_content}' vs {expected_content}") + logger.debug(f"Comparing: '{actual_content}' vs {expected_content}") return actual_content == expected_content.strip() def get_logs(self) -> str: - logging.debug("Getting logs from container '%s'", self.container_name) + logger.debug("Getting logs from container '%s'", self.container_name) if not self.container: return "" logs_as_bytes = self.container.logs() - return logs_as_bytes.decode('utf-8') + return logs_as_bytes.decode("utf-8") @property def exited(self) -> bool: @@ -282,7 +331,7 @@ def exited(self) -> bool: return False try: self.container.reload() - return self.container.status == 'exited' + return self.container.status == "exited" except docker.errors.NotFound: self.container = None return False @@ -291,39 +340,45 @@ def exited(self) -> bool: def get_number_of_files(self, directory_path: str) -> int: if not self.container: - logging.warning("Container not running") + logger.warning("Container not running") return -1 if not self.nonempty_dir_exists(directory_path): - logging.warning(f"Container directory does not exist: {directory_path}") + logger.warning(f"Container directory does not exist: {directory_path}") return 0 count_command = f"sh -c 'find {directory_path} -maxdepth 1 -type f | wc -l'" exit_code, output = self.exec_run(count_command) if exit_code != 0: - logging.error(f"Error running command '{count_command}': {output}") + logger.error(f"Error running command '{count_command}': {output}") return -1 try: file_count = int(output.strip()) - logging.debug(f"Number of files in '{directory_path}': {file_count}") + logger.debug(f"Number of files in '{directory_path}': {file_count}") return file_count except (ValueError, IndexError): - logging.error(f"Error parsing output '{output}' from command '{count_command}'") + logger.error( + f"Error parsing output '{output}' from command '{count_command}'" + ) return -1 - def _get_contents_of_all_files_in_directory(self, directory_path: str) -> list[str] | None: + def _get_contents_of_all_files_in_directory( + self, directory_path: str + ) -> list[str] | None: safe_dir_path = shlex.quote(directory_path) - list_files_command = f"find {safe_dir_path} -mindepth 1 -maxdepth 1 -type f -print0" + list_files_command = ( + f"find {safe_dir_path} -mindepth 1 -maxdepth 1 -type f -print0" + ) - exit_code, output = self.exec_run(f"sh -c \"{list_files_command}\"") + exit_code, output = self.exec_run(f'sh -c "{list_files_command}"') if exit_code != 0: - logging.error(f"Error running command '{list_files_command}': {output}") + logger.error(f"Error running command '{list_files_command}': {output}") return None - actual_filepaths = [path for path in output.split('\0') if path] + actual_filepaths = [path for path in output.split("\0") if path] actual_file_contents = [] for path in actual_filepaths: @@ -333,34 +388,46 @@ def _get_contents_of_all_files_in_directory(self, directory_path: str) -> list[s exit_code, content = self.exec_run(read_command) if exit_code != 0: - error_message = f"Command to read file '{path}' failed with exit code {exit_code}" - logging.error(error_message) + error_message = ( + f"Command to read file '{path}' failed with exit code {exit_code}" + ) + logger.error(error_message) return None actual_file_contents.append(content) return actual_file_contents - def _verify_file_contents_in_running_container(self, directory_path: str, expected_contents: list[str]) -> bool: + def _verify_file_contents_in_running_container( + self, directory_path: str, expected_contents: list[str] + ) -> bool: if not self.nonempty_dir_exists(directory_path): return False - actual_file_contents = self._get_contents_of_all_files_in_directory(directory_path) + actual_file_contents = self._get_contents_of_all_files_in_directory( + directory_path + ) if actual_file_contents is None: return False if len(actual_file_contents) != len(expected_contents): - logging.debug(f"Expected {len(expected_contents)} files, but found {len(actual_file_contents)}") + logger.debug( + f"Expected {len(expected_contents)} files, but found {len(actual_file_contents)}" + ) return False return sorted(actual_file_contents) == sorted(expected_contents) - def _verify_file_contents_in_stopped_container(self, directory_path: str, expected_contents: list[str]) -> bool: + def _verify_file_contents_in_stopped_container( + self, directory_path: str, expected_contents: list[str] + ) -> bool: if not self.container: return False with tempfile.TemporaryDirectory() as temp_dir: - extracted_dir = self._extract_directory_from_container(directory_path, temp_dir) + extracted_dir = self._extract_directory_from_container( + directory_path, temp_dir + ) if not extracted_dir: return False @@ -370,22 +437,24 @@ def _verify_file_contents_in_stopped_container(self, directory_path: str, expect return sorted(actual_file_contents) == sorted(expected_contents) - def _extract_directory_from_container(self, directory_path: str, temp_dir: str) -> str | None: + def _extract_directory_from_container( + self, directory_path: str, temp_dir: str + ) -> str | None: try: bits, _ = self.container.get_archive(directory_path) temp_tar_path = os.path.join(temp_dir, "archive.tar") - with open(temp_tar_path, 'wb') as f: - for chunk in bits: - f.write(chunk) + with open(temp_tar_path, "wb") as f: + f.writelines(bits) with tarfile.open(temp_tar_path) as tar: tar.extractall(path=temp_dir) - return os.path.join(temp_dir, os.path.basename(directory_path.strip('/'))) + return os.path.join(temp_dir, os.path.basename(directory_path.strip("/"))) except Exception as e: - logging.error( - f"Error extracting files from directory path '{directory_path}' from container '{self.container_name}': {e}") + logger.error( + f"Error extracting files from directory path '{directory_path}' from container '{self.container_name}': {e}" + ) return None def _read_files_from_directory(self, directory_path: str) -> list[str] | None: @@ -393,57 +462,71 @@ def _read_files_from_directory(self, directory_path: str) -> list[str] | None: file_contents = [] for entry in os.scandir(directory_path): if entry.is_file(): - with open(entry.path, 'r') as f: + with open(entry.path, "r") as f: file_contents.append(f.read()) return file_contents except Exception as e: - logging.error(f"Error reading extracted files: {e}") + logger.error(f"Error reading extracted files: {e}") return None - def verify_file_contents(self, directory_path: str, expected_contents: list[str]) -> bool: + def verify_file_contents( + self, directory_path: str, expected_contents: list[str] + ) -> bool: if not self.container: return False self.container.reload() if self.container.status == "running": - return self._verify_file_contents_in_running_container(directory_path, expected_contents) + return self._verify_file_contents_in_running_container( + directory_path, expected_contents + ) - return self._verify_file_contents_in_stopped_container(directory_path, expected_contents) + return self._verify_file_contents_in_stopped_container( + directory_path, expected_contents + ) def log_app_output(self) -> bool: logs = self.get_logs() - logging.info("Logs of container '%s':", self.container_name) + logger.info("Logs of container '%s':", self.container_name) for line in logs.splitlines(): - logging.info(line) + logger.info(line) return False - def verify_path_with_json_content(self, directory_path: str, expected_str: str) -> bool: + def verify_path_with_json_content( + self, directory_path: str, expected_str: str + ) -> bool: if not self.container or not self.nonempty_dir_exists(directory_path): - logging.warning(f"Container not running or directory does not exist: {directory_path}") + logger.warning( + f"Container not running or directory does not exist: {directory_path}" + ) return False count_command = f"sh -c 'find {directory_path} -maxdepth 1 -type f | wc -l'" exit_code, output = self.exec_run(count_command) if exit_code != 0: - logging.error(f"Error running command '{count_command}': {output}") + logger.error(f"Error running command '{count_command}': {output}") return False try: file_count = int(output.strip()) except (ValueError, IndexError): - logging.error(f"Error parsing output '{output}' from command '{count_command}'") + logger.error( + f"Error parsing output '{output}' from command '{count_command}'" + ) return False if file_count != 1: - logging.error(f"{directory_path} has too many or too few ({file_count}) files") + logger.error( + f"{directory_path} has too many or too few ({file_count}) files" + ) return False content_command = f"sh -c 'cat {directory_path}/*'" exit_code, output = self.exec_run(content_command) if exit_code != 0: - logging.error(f"Error running command '{content_command}': {output}") + logger.error(f"Error running command '{content_command}': {output}") return False actual_content = output.strip() @@ -452,12 +535,18 @@ def verify_path_with_json_content(self, directory_path: str, expected_str: str) return actual_json == expected_json - def directory_contains_file_with_json_content(self, directory_path: str, expected_content: str) -> bool: + def directory_contains_file_with_json_content( + self, directory_path: str, expected_content: str + ) -> bool: if not self.container or not self.nonempty_dir_exists(directory_path): - logging.warning(f"Container not running or directory does not exist: {directory_path}") + logger.warning( + f"Container not running or directory does not exist: {directory_path}" + ) return False - actual_file_contents = self._get_contents_of_all_files_in_directory(directory_path) + actual_file_contents = self._get_contents_of_all_files_in_directory( + directory_path + ) if actual_file_contents is None: return False @@ -467,32 +556,37 @@ def directory_contains_file_with_json_content(self, directory_path: str, expecte expected_json = json.loads(expected_content) if actual_json == expected_json: return True - logging.warning(f"File content does not match expected JSON: {file_content}") + logger.warning( + f"File content does not match expected JSON: {file_content}" + ) except json.JSONDecodeError: - logging.error("Error decoding JSON content from file.") + logger.error("Error decoding JSON content from file.") continue return False - def directory_contains_file_with_minimum_size(self, directory_path: str, expected_size: int) -> bool: + def directory_contains_file_with_minimum_size( + self, directory_path: str, expected_size: int + ) -> bool: if not self.container or not self.nonempty_dir_exists(directory_path): return False - command = f"find \"{directory_path}\" -maxdepth 1 -type f -size +{expected_size}c" + command = f'find "{directory_path}" -maxdepth 1 -type f -size +{expected_size}c' exit_code, output = self.exec_run(command) if exit_code != 0: - logging.error(f"Error running command to get file sizes: {output}") + logger.error(f"Error running command to get file sizes: {output}") return False - if len(output.strip()) > 0: - return True - - return False + return len(output.strip()) > 0 def get_memory_usage(self) -> int | None: - exit_code, output = self.exec_run(["awk", "/VmRSS.*kB/ { printf \"%d\", $2 }", "/proc/1/status"]) + exit_code, output = self.exec_run( + ["awk", '/VmRSS.*kB/ { printf "%d", $2 }', "/proc/1/status"] + ) if exit_code != 0: return None memory_usage_in_bytes = int(output) * 1024 - logging.info(f"{self.container_name} memory usage: {memory_usage_in_bytes} bytes") + logger.info( + f"{self.container_name} memory usage: {memory_usage_in_bytes} bytes" + ) return memory_usage_in_bytes diff --git a/behave_framework/src/minifi_behave/containers/container_protocol.py b/behave_framework/src/minifi_behave/containers/container_protocol.py index 2b2fdcc721..8d38650f04 100644 --- a/behave_framework/src/minifi_behave/containers/container_protocol.py +++ b/behave_framework/src/minifi_behave/containers/container_protocol.py @@ -27,36 +27,31 @@ class ContainerProtocol(Protocol): files: list[File] host_files: list[HostFile] - def deploy(self, context) -> bool: - ... + def deploy(self, context) -> bool: ... - def clean_up(self): - ... + def clean_up(self): ... - def exec_run(self, command): - ... + def exec_run(self, command): ... - def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool: - ... + def directory_contains_file_with_content( + self, directory_path: str, expected_content: str + ) -> bool: ... - def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool: - ... + def directory_contains_file_with_regex( + self, directory_path: str, regex_str: str + ) -> bool: ... - def path_with_content_exists(self, path: str, content: str) -> bool: - ... + def path_with_content_exists(self, path: str, content: str) -> bool: ... - def get_logs(self) -> str: - ... + def get_logs(self) -> str: ... @property - def exited(self) -> bool: - ... + def exited(self) -> bool: ... - def get_number_of_files(self, directory_path: str) -> int: - ... + def get_number_of_files(self, directory_path: str) -> int: ... - def verify_file_contents(self, directory_path: str, expected_contents: list[str]) -> bool: - ... + def verify_file_contents( + self, directory_path: str, expected_contents: list[str] + ) -> bool: ... - def log_app_output(self) -> bool: - ... + def log_app_output(self) -> bool: ... diff --git a/behave_framework/src/minifi_behave/containers/container_windows.py b/behave_framework/src/minifi_behave/containers/container_windows.py index 87b0a6243b..d2d5576610 100644 --- a/behave_framework/src/minifi_behave/containers/container_windows.py +++ b/behave_framework/src/minifi_behave/containers/container_windows.py @@ -15,29 +15,39 @@ # limitations under the License. # from __future__ import annotations + +import base64 +import io import logging import os -import tempfile -import base64 import tarfile -import io +import tempfile from typing import TYPE_CHECKING -import docker -from docker.models.networks import Network from docker.models.containers import Container - +from docker.models.networks import Network from minifi_behave.containers.container_protocol import ContainerProtocol from minifi_behave.containers.directory import Directory from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile +import docker + +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from minifi_behave.core.minifi_test_context import MinifiTestContext class WindowsContainer(ContainerProtocol): - def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None): + def __init__( + self, + image_name: str, + container_name: str, + network: Network, + command: str | None = None, + entrypoint: str | None = None, + ): super().__init__() self.image_name: str = image_name self.container_name: str = container_name @@ -57,8 +67,7 @@ def __init__(self, image_name: str, container_name: str, network: Network, comma def _normalize_path(self, path: str) -> str: clean_path = path.strip().replace("/", "\\") - if clean_path.startswith("\\"): - clean_path = clean_path[1:] + clean_path = clean_path.removeprefix("\\") # If it doesn't already have a drive letter, assume C: if ":" not in clean_path: @@ -78,28 +87,33 @@ def deploy(self, context: MinifiTestContext | None) -> bool: for file_name, content in directory.files.items(): file_path = os.path.join(temp_subdir, file_name) with open(file_path, "w", encoding="utf-8") as temp_file: - logging.info(f"writing content into {temp_file.name}") + logger.info(f"writing content into {temp_file.name}") temp_file.write(content) container_bind_path = self._normalize_path(directory.path) self.volumes[temp_subdir] = { "bind": container_bind_path, - "mode": directory.mode + "mode": directory.mode, } for host_file in self.host_files: container_bind_path = self._normalize_path(host_file.container_path) - self.volumes[host_file.host_path] = {"bind": container_bind_path, "mode": host_file.mode} + self.volumes[host_file.host_path] = { + "bind": container_bind_path, + "mode": host_file.mode, + } try: existing_container = self.client.containers.get(self.container_name) - logging.warning(f"Found existing container '{self.container_name}'. Removing it first.") + logger.warning( + f"Found existing container '{self.container_name}'. Removing it first." + ) existing_container.remove(force=True) except docker.errors.NotFound: pass try: - logging.info(f"Creating and starting container '{self.container_name}'...") + logger.info(f"Creating and starting container '{self.container_name}'...") self.container = self.client.containers.run( image=self.image_name, name=self.container_name, @@ -110,14 +124,14 @@ def deploy(self, context: MinifiTestContext | None) -> bool: command=self.command, entrypoint=self.entrypoint, detach=True, - tty=False + tty=False, ) for file in self.files: self._copy_content_to_container(file.content, file.path) except Exception as e: - logging.error(f"Error starting container: {e}") + logger.error(f"Error starting container: {e}") self.clean_up() raise return True @@ -133,9 +147,9 @@ def _copy_content_to_container(self, content: str | bytes, target_path: str): self._run_powershell(f"New-Item -ItemType Directory -Force -Path '{dir_name}'") tar_stream = io.BytesIO() - with tarfile.open(fileobj=tar_stream, mode='w') as tar: + with tarfile.open(fileobj=tar_stream, mode="w") as tar: if isinstance(content, str): - encoded_data = content.encode('utf-8') + encoded_data = content.encode("utf-8") else: encoded_data = content tarinfo = tarfile.TarInfo(name=file_name) @@ -151,7 +165,7 @@ def clean_up(self): try: self._temp_dir.cleanup() except Exception as e: - logging.warning(f"Failed to cleanup temp dir: {e}") + logger.warning(f"Failed to cleanup temp dir: {e}") finally: self._temp_dir = None @@ -161,16 +175,16 @@ def clean_up(self): except docker.errors.NotFound: pass except Exception as e: - logging.warning(f"Failed to remove container: {e}") + logger.warning(f"Failed to remove container: {e}") finally: self.container = None def exec_run(self, command: str | list) -> tuple[int | None, str]: - logging.debug(f"Running command: {command}") + logger.debug(f"Running command: {command}") if self.container: (code, output) = self.container.exec_run(command, detach=False) - decoded_output = output.decode("utf-8", errors='replace') - logging.debug(f"Result {code}, output: {decoded_output}") + decoded_output = output.decode("utf-8", errors="replace") + logger.debug(f"Result {code}, output: {decoded_output}") return code, decoded_output return None, "Container not running." @@ -178,9 +192,17 @@ def _run_powershell(self, ps_script: str) -> tuple[int | None, str]: if not self.container: return None, "Container not running" - encoded_command = base64.b64encode(ps_script.encode('utf_16_le')).decode('utf-8') + encoded_command = base64.b64encode(ps_script.encode("utf_16_le")).decode( + "utf-8" + ) - cmd_parts = ["powershell", "-NonInteractive", "-NoProfile", "-EncodedCommand", encoded_command] + cmd_parts = [ + "powershell", + "-NonInteractive", + "-NoProfile", + "-EncodedCommand", + encoded_command, + ] return self.exec_run(cmd_parts) @@ -198,7 +220,9 @@ def not_empty_dir_exists(self, directory_path: str) -> bool: exit_code, _ = self._run_powershell(ps_script) return exit_code == 0 - def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool: + def directory_contains_file_with_content( + self, directory_path: str, expected_content: str + ) -> bool: if not self.container: return False @@ -215,7 +239,9 @@ def directory_contains_file_with_content(self, directory_path: str, expected_con exit_code, _ = self._run_powershell(ps_script) return exit_code == 0 - def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool: + def directory_contains_file_with_regex( + self, directory_path: str, regex_str: str + ) -> bool: if not self.container: return False @@ -249,16 +275,22 @@ def path_with_content_exists(self, path: str, content: str) -> bool: exit_code, output = self._run_powershell(ps_script) if exit_code != 0: - logging.debug(f"path_with_content_exists failed for {win_path}. Output: {output}") + logger.debug( + f"path_with_content_exists failed for {win_path}. Output: {output}" + ) return exit_code == 0 - def directory_has_single_file_with_content(self, directory_path: str, expected_content: str) -> bool: + def directory_has_single_file_with_content( + self, directory_path: str, expected_content: str + ) -> bool: if not self.container: return False win_path = self._normalize_path(directory_path) - escaped_content = expected_content.strip().replace("'", "''").replace("\n", "\r\n") + escaped_content = ( + expected_content.strip().replace("'", "''").replace("\n", "\r\n") + ) ps_script = ( f"$files = Get-ChildItem -Path '{win_path}' -File -Depth 0; " @@ -269,22 +301,24 @@ def directory_has_single_file_with_content(self, directory_path: str, expected_c exit_code, output = self._run_powershell(ps_script) if exit_code != 0: - logging.debug(f"Check for single file failed in {win_path}. Output: {output}") + logger.debug( + f"Check for single file failed in {win_path}. Output: {output}" + ) return exit_code == 0 def get_logs(self) -> str: - logging.debug("Getting logs from container '%s'", self.container_name) + logger.debug("Getting logs from container '%s'", self.container_name) if not self.container: return "" logs_as_bytes = self.container.logs() - return logs_as_bytes.decode('utf-8', errors='replace') + return logs_as_bytes.decode("utf-8", errors="replace") def log_app_output(self) -> bool: logs = self.get_logs() - logging.info("Logs of container '%s':", self.container_name) + logger.info("Logs of container '%s':", self.container_name) for line in logs.splitlines(): - logging.info(line) + logger.info(line) return False @property @@ -293,7 +327,7 @@ def exited(self) -> bool: return False try: self.container.reload() - return self.container.status == 'exited' + return self.container.status == "exited" except docker.errors.NotFound: self.container = None return False @@ -310,7 +344,7 @@ def get_number_of_files(self, directory_path: str) -> int: exit_code, output = self._run_powershell(ps_script) if exit_code != 0: - logging.error(f"Error counting files in '{win_path}': {output}") + logger.error(f"Error counting files in '{win_path}': {output}") return -1 try: @@ -318,7 +352,9 @@ def get_number_of_files(self, directory_path: str) -> int: except (ValueError, IndexError): return -1 - def verify_file_contents(self, directory_path: str, expected_contents: list[str]) -> bool: + def verify_file_contents( + self, directory_path: str, expected_contents: list[str] + ) -> bool: if not self.container: return False @@ -328,10 +364,12 @@ def verify_file_contents(self, directory_path: str, expected_contents: list[str] exit_code, output = self._run_powershell(ps_list) if exit_code != 0: - logging.error(f"Error listing files in '{win_path}': {output}") + logger.error(f"Error listing files in '{win_path}': {output}") return False - actual_filepaths = [path.strip() for path in output.splitlines() if path.strip()] + actual_filepaths = [ + path.strip() for path in output.splitlines() if path.strip() + ] if len(actual_filepaths) != len(expected_contents): return False @@ -349,7 +387,8 @@ def verify_file_contents(self, directory_path: str, expected_contents: list[str] actual_file_contents.append(content) normalized_expected = [ - s.strip().replace("\r\n", "\n").replace("\r", "\n") for s in expected_contents + s.strip().replace("\r\n", "\n").replace("\r", "\n") + for s in expected_contents ] return sorted(actual_file_contents) == sorted(normalized_expected) @@ -369,12 +408,14 @@ def nonempty_dir_exists(self, directory_path: str) -> bool: exit_code, output = self._run_powershell(ps_script) if exit_code != 0: - logging.error(f"Error running command for nonempty_dir_exists: {output}") + logger.error(f"Error running command for nonempty_dir_exists: {output}") return False return True - def directory_contains_file_with_minimum_size(self, directory_path: str, expected_size: int) -> bool: + def directory_contains_file_with_minimum_size( + self, directory_path: str, expected_size: int + ) -> bool: if not self.container or not self.nonempty_dir_exists(directory_path): return False @@ -387,10 +428,7 @@ def directory_contains_file_with_minimum_size(self, directory_path: str, expecte exit_code, output = self._run_powershell(ps_script) if exit_code != 0: - logging.error(f"Error running command to get file sizes: {output}") + logger.error(f"Error running command to get file sizes: {output}") return False - if output and len(output.strip()) > 0: - return True - - return False + return bool(output and len(output.strip()) > 0) diff --git a/behave_framework/src/minifi_behave/containers/directory.py b/behave_framework/src/minifi_behave/containers/directory.py index 47462ef91e..4628b0413b 100644 --- a/behave_framework/src/minifi_behave/containers/directory.py +++ b/behave_framework/src/minifi_behave/containers/directory.py @@ -15,6 +15,7 @@ # limitations under the License. # + class Directory: def __init__(self, path, files: dict[str, str | bytes] | None = None, mode="rw"): self.path = path diff --git a/behave_framework/src/minifi_behave/containers/docker_image_builder.py b/behave_framework/src/minifi_behave/containers/docker_image_builder.py index 1e5d5e602f..3f933615fb 100644 --- a/behave_framework/src/minifi_behave/containers/docker_image_builder.py +++ b/behave_framework/src/minifi_behave/containers/docker_image_builder.py @@ -19,16 +19,29 @@ import os import tempfile -import docker from docker.models.images import Image +import docker + +logger = logging.getLogger(__name__) + class DockerImageBuilder: - def __init__(self, image_tag: str, dockerfile_content: str | None = None, files_on_context: dict[str, bytes] | None = None, build_context_path: str | None = None): + def __init__( + self, + image_tag: str, + dockerfile_content: str | None = None, + files_on_context: dict[str, bytes] | None = None, + build_context_path: str | None = None, + ): if not dockerfile_content and not build_context_path: - raise ValueError("Either 'dockerfile_content' or 'build_context_path' must be provided.") + raise ValueError( + "Either 'dockerfile_content' or 'build_context_path' must be provided." + ) if dockerfile_content and build_context_path: - raise ValueError("Provide either 'dockerfile_content' or 'build_context_path', not both.") + raise ValueError( + "Provide either 'dockerfile_content' or 'build_context_path', not both." + ) self.image_tag: str = image_tag self.dockerfile_content: str | None = dockerfile_content @@ -43,27 +56,33 @@ def build(self) -> Image: if self.dockerfile_content: self._temp_dir = tempfile.TemporaryDirectory() context_path = self._temp_dir.name - dockerfile_path = os.path.join(context_path, 'Dockerfile') - with open(dockerfile_path, 'w') as f: + dockerfile_path = os.path.join(context_path, "Dockerfile") + with open(dockerfile_path, "w") as f: f.write(self.dockerfile_content) if self.files_on_context: for filename, content in self.files_on_context.items(): file_path = os.path.join(context_path, filename) - with open(file_path, 'wb') as f: + with open(file_path, "wb") as f: f.write(content) - logging.info(f"Building Docker image '{self.image_tag}' from context '{context_path}'...") + logger.info( + f"Building Docker image '{self.image_tag}' from context '{context_path}'..." + ) try: - self.image, build_logs = self.client.images.build(path=context_path, tag=self.image_tag, rm=True, forcerm=True) + self.image, build_logs = self.client.images.build( + path=context_path, tag=self.image_tag, rm=True, forcerm=True + ) for log_line in build_logs: - logging.debug(log_line.get('stream', '').strip()) - logging.info(f"Successfully built image '{self.image_tag}' (ID: {self.image.short_id})") + logger.debug(log_line.get("stream", "").strip()) + logger.info( + f"Successfully built image '{self.image_tag}' (ID: {self.image.short_id})" + ) return self.image except docker.errors.BuildError as e: - logging.error(f"Failed to build image '{self.image_tag}'. Build logs:") + logger.error(f"Failed to build image '{self.image_tag}'. Build logs:") for log_line in e.build_log: - logging.error(log_line.get('stream', '').strip()) + logger.error(log_line.get("stream", "").strip()) raise finally: if self._temp_dir: @@ -71,18 +90,20 @@ def build(self) -> Image: def remove_image(self): if not self.image: - logging.warning(f"No image object to remove for tag '{self.image_tag}'. Trying to find by tag.") + logger.warning( + f"No image object to remove for tag '{self.image_tag}'. Trying to find by tag." + ) try: self.image = self.client.images.get(self.image_tag) except docker.errors.ImageNotFound: - logging.info(f"Image '{self.image_tag}' not found, cleanup not needed.") + logger.info(f"Image '{self.image_tag}' not found, cleanup not needed.") return - logging.info(f"Removing dynamically built image '{self.image_tag}'...") + logger.info(f"Removing dynamically built image '{self.image_tag}'...") try: self.client.images.remove(image=self.image.id, force=True) - logging.info("Image removed successfully.") + logger.info("Image removed successfully.") except docker.errors.ImageNotFound: - logging.info("Image was already removed.") + logger.info("Image was already removed.") except docker.errors.APIError as e: - logging.error(f"Error removing image: {e}") + logger.error(f"Error removing image: {e}") diff --git a/behave_framework/src/minifi_behave/containers/file.py b/behave_framework/src/minifi_behave/containers/file.py index ee50af95e9..6df14a92e9 100644 --- a/behave_framework/src/minifi_behave/containers/file.py +++ b/behave_framework/src/minifi_behave/containers/file.py @@ -15,6 +15,7 @@ # limitations under the License. # + class File: def __init__(self, path, content: str | bytes, mode="rw", permissions=None): self.path = path diff --git a/behave_framework/src/minifi_behave/containers/host_file.py b/behave_framework/src/minifi_behave/containers/host_file.py index 1e046f54c6..f68844fc81 100644 --- a/behave_framework/src/minifi_behave/containers/host_file.py +++ b/behave_framework/src/minifi_behave/containers/host_file.py @@ -15,6 +15,7 @@ # limitations under the License. # + class HostFile: def __init__(self, path, host_path, mode="ro"): self.container_path = path diff --git a/behave_framework/src/minifi_behave/containers/http_proxy_container.py b/behave_framework/src/minifi_behave/containers/http_proxy_container.py index 4063363e9b..6d16c372ba 100644 --- a/behave_framework/src/minifi_behave/containers/http_proxy_container.py +++ b/behave_framework/src/minifi_behave/containers/http_proxy_container.py @@ -19,15 +19,16 @@ from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.helpers import wait_for_condition, retry_check -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key from minifi_behave.containers.file import File +from minifi_behave.core.helpers import retry_check, wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert class HttpProxy(LinuxContainer): def __init__(self, test_context: MinifiTestContext): - dockerfile = dedent("""\ + dockerfile = dedent( + """\ FROM ubuntu:24.04 RUN apt -y update && apt install -y squid-openssl apache2-utils RUN htpasswd -b -c /etc/squid/.squid_users {proxy_username} {proxy_password} @@ -43,20 +44,40 @@ def __init__(self, test_context: MinifiTestContext): echo 'max_filedescriptors 1024' >> /etc/squid/squid.conf && \ echo 'https_port {proxy_ssl_port} tls-cert=/etc/squid/certs/squid-cert.pem tls-key=/etc/squid/certs/squid-key.pem' >> /etc/squid/squid.conf ENTRYPOINT ["/bin/sh", "-c", "squid -N -f /etc/squid/squid.conf & tail -F --pid=$! /var/log/squid/cache.log /var/log/squid/access.log"] - """.format(proxy_username='admin', proxy_password='test101', - proxy_port='3128', proxy_ssl_port='3129')) + """.format( + proxy_username="admin", + proxy_password="test101", + proxy_port="3128", + proxy_ssl_port="3129", + ) + ) builder = DockerImageBuilder( - image_tag="minifi-http-proxy:latest", - dockerfile_content=dockerfile + image_tag="minifi-http-proxy:latest", dockerfile_content=dockerfile ) builder.build() - super().__init__("minifi-http-proxy:latest", f"http-proxy-{test_context.scenario_id}", test_context.network) - squid_cert, squid_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + super().__init__( + "minifi-http-proxy:latest", + f"http-proxy-{test_context.scenario_id}", + test_context.network, + ) + squid_cert, squid_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) - self.files.append(File("/etc/squid/certs/squid-cert.pem", dump_cert(squid_cert), permissions=0o666)) - self.files.append(File("/etc/squid/certs/squid-key.pem", dump_key(squid_key), permissions=0o666)) + self.files.append( + File( + "/etc/squid/certs/squid-cert.pem", + dump_cert(squid_cert), + permissions=0o666, + ) + ) + self.files.append( + File( + "/etc/squid/certs/squid-key.pem", dump_key(squid_key), permissions=0o666 + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -65,14 +86,22 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context + context=context, ) @retry_check(10, 1) def check_http_proxy_access(self, url): (code, output) = self.exec_run(["cat", "/var/log/squid/access.log"]) - return code == 0 and url.lower() in output.lower() \ - and ((output.count("TCP_DENIED") != 0 - and output.count("TCP_MISS") >= output.count("TCP_DENIED")) - or output.count("TCP_DENIED") == 0 and "TCP_MISS" in output - or output.count("TCP_TUNNEL") > 0) + return ( + code == 0 + and url.lower() in output.lower() + and ( + ( + output.count("TCP_DENIED") != 0 + and output.count("TCP_MISS") >= output.count("TCP_DENIED") + ) + or output.count("TCP_DENIED") == 0 + and "TCP_MISS" in output + or output.count("TCP_TUNNEL") > 0 + ) + ) diff --git a/behave_framework/src/minifi_behave/containers/minifi_controller.py b/behave_framework/src/minifi_behave/containers/minifi_controller.py index 756cbc389c..b6bd6f086e 100644 --- a/behave_framework/src/minifi_behave/containers/minifi_controller.py +++ b/behave_framework/src/minifi_behave/containers/minifi_controller.py @@ -14,8 +14,11 @@ # limitations under the License. import logging + from minifi_behave.core.helpers import retry_check +logger = logging.getLogger(__name__) + class MinifiController: def __init__(self, minifi_container, config_path, bin_path): @@ -27,34 +30,52 @@ def set_controller_socket_properties(self): self.minifi_container.properties["controller.socket.enable"] = "true" self.minifi_container.properties["controller.socket.host"] = "localhost" self.minifi_container.properties["controller.socket.port"] = "9998" - self.minifi_container.properties["controller.socket.local.any.interface"] = "false" + self.minifi_container.properties["controller.socket.local.any.interface"] = ( + "false" + ) def update_flow_config_through_controller(self): - self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--updateflow", "/tmp/resources/minifi-controller/config.yml"]) + self.minifi_container.exec_run( + [ + f"{self.bin_path}/minifi-controller", + "--updateflow", + "/tmp/resources/minifi-controller/config.yml", + ] + ) def updated_config_is_persisted(self) -> bool: exit_code, output = self.minifi_container.exec_run(["cat", self.config_path]) if exit_code != 0: - logging.error(f"Failed to read MiNiFi config file to check if updated config is persisted: {exit_code} {output}") + logger.error( + f"Failed to read MiNiFi config file to check if updated config is persisted: {exit_code} {output}" + ) return False return "2f2a3b47-f5ba-49f6-82b5-bc1c86b96f38" in output def stop_component_through_controller(self, component: str): - self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--stop", component]) + self.minifi_container.exec_run( + [f"{self.bin_path}/minifi-controller", "--stop", component] + ) def start_component_through_controller(self, component: str): - self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--start", component]) + self.minifi_container.exec_run( + [f"{self.bin_path}/minifi-controller", "--start", component] + ) @retry_check(max_tries=10, retry_interval_seconds=1) def is_component_running(self, component: str) -> bool: - (code, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--list", "components"]) + (code, output) = self.minifi_container.exec_run( + [f"{self.bin_path}/minifi-controller", "--list", "components"] + ) return code == 0 and component + ", running: true" in output def get_connections(self): - (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--list", "connections"]) + (_, output) = self.minifi_container.exec_run( + [f"{self.bin_path}/minifi-controller", "--list", "connections"] + ) connections = [] - for line in output.split('\n'): - if not line.startswith('[') and not line.startswith('Connection Names'): + for line in output.split("\n"): + if not line.startswith("[") and not line.startswith("Connection Names"): connections.append(line) return connections @@ -63,32 +84,40 @@ def connection_found_through_controller(self, connection: str) -> bool: return connection in self.get_connections() def get_full_connection_count(self) -> int: - (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--getfull"]) - for line in output.split('\n'): + (_, output) = self.minifi_container.exec_run( + [f"{self.bin_path}/minifi-controller", "--getfull"] + ) + for line in output.split("\n"): if "are full" in line: - return int(line.split(' ')[0]) + return int(line.split(" ")[0]) return -1 def get_connection_size(self, connection: str): - (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--getsize", connection]) - for line in output.split('\n'): + (_, output) = self.minifi_container.exec_run( + [f"{self.bin_path}/minifi-controller", "--getsize", connection] + ) + for line in output.split("\n"): if "Size/Max of " + connection in line: - size_and_max = line.split(connection)[1].split('/') + size_and_max = line.split(connection)[1].split("/") return (int(size_and_max[0].strip()), int(size_and_max[1].strip())) return (-1, -1) def get_manifest(self) -> str: - (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--manifest"]) + (_, output) = self.minifi_container.exec_run( + [f"{self.bin_path}/minifi-controller", "--manifest"] + ) manifest = "" - for line in output.split('\n'): - if not line.startswith('['): + for line in output.split("\n"): + if not line.startswith("["): manifest += line return manifest def create_debug_bundle(self) -> bool: - (code, _) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--debug", "/tmp"]) + (code, _) = self.minifi_container.exec_run( + [f"{self.bin_path}/minifi-controller", "--debug", "/tmp"] + ) if code != 0: - logging.error("Minifi controller debug command failed with code: %d", code) + logger.error("Minifi controller debug command failed with code: %d", code) return False (code, _) = self.minifi_container.exec_run(["test", "-f", "/tmp/debug.tar.gz"]) diff --git a/behave_framework/src/minifi_behave/containers/minifi_linux_container.py b/behave_framework/src/minifi_behave/containers/minifi_linux_container.py index 4658a82379..b5245b0c6c 100644 --- a/behave_framework/src/minifi_behave/containers/minifi_linux_container.py +++ b/behave_framework/src/minifi_behave/containers/minifi_linux_container.py @@ -17,24 +17,31 @@ import logging import os +from pathlib import Path + from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, \ - dump_cert, dump_key +from minifi_behave.core.ssl_utils import ( + dump_cert, + dump_key, + make_cert_without_extended_usage, + make_client_cert, + make_server_cert, +) from minifi_behave.minifi.minifi_flow_definition import MinifiFlowDefinition -from pathlib import Path from .container_linux import LinuxContainer from .minifi_controller import MinifiController from .minifi_protocol import MinifiProtocol +logger = logging.getLogger(__name__) CA_CERT_PATHS = [ "/usr/local/share/certs/ca-root-nss.crt", "/etc/ssl/certs/ca-certificates.crt", - "/etc/pki/tls/certs/ca-bundle.crt" + "/etc/pki/tls/certs/ca-bundle.crt", ] @@ -44,7 +51,9 @@ def __init__(self): self.bin_path = "/opt/minifi/minifi-current/bin" self.extension_pattern = "../extensions/*" self.minifi_python_path = "/opt/minifi/minifi-current/minifi-python" - self.minifi_python_examples_path = "/opt/minifi/minifi-current/minifi-python-examples" + self.minifi_python_examples_path = ( + "/opt/minifi/minifi-current/minifi-python-examples" + ) class FHSDeployment: @@ -53,14 +62,23 @@ def __init__(self): self.bin_path = "/usr/bin" self.extension_pattern = "/usr/lib64/nifi-minifi-cpp/extensions/*" self.minifi_python_path = "/var/lib/nifi-minifi-cpp/minifi-python" - self.minifi_python_examples_path = "/usr/share/doc/nifi-minifi-cpp/pythonprocessor-examples" + self.minifi_python_examples_path = ( + "/usr/share/doc/nifi-minifi-cpp/pythonprocessor-examples" + ) class MinifiLinuxContainer(LinuxContainer, MinifiProtocol): - def __init__(self, container_name: str, test_context: MinifiTestContext, - deployment: NormalDeployment | FHSDeployment): - super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", - test_context.network) + def __init__( + self, + container_name: str, + test_context: MinifiTestContext, + deployment: NormalDeployment | FHSDeployment, + ): + super().__init__( + test_context.minifi_container_image, + f"{container_name}-{test_context.scenario_id}", + test_context.network, + ) self.flow_definition = MinifiFlowDefinition() self.properties: dict[str, str] = {} self.log_properties: dict[str, str] = {} @@ -68,54 +86,105 @@ def __init__(self, container_name: str, test_context: MinifiTestContext, self.deploy_timeout_seconds = 20 self.deployment_type = deployment - self.controller = MinifiController(self, f"{self.deployment_type.conf_path}/config.yml", - self.deployment_type.bin_path) - - minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, - ca_cert=test_context.root_ca_cert, - ca_key=test_context.root_ca_key) - self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert))) + self.controller = MinifiController( + self, + f"{self.deployment_type.conf_path}/config.yml", + self.deployment_type.bin_path, + ) + + minifi_client_cert, minifi_client_key = make_cert_without_extended_usage( + common_name=self.container_name, + ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key, + ) + self.files.append( + File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert)) + ) if test_context.override_default_ca_cert_files: for ca_cert_path in CA_CERT_PATHS: - self.files.append(File(ca_cert_path, dump_cert(test_context.root_ca_cert))) - self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert))) - self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key))) + self.files.append( + File(ca_cert_path, dump_cert(test_context.root_ca_cert)) + ) self.files.append( - File("/tmp/resources/minifi_merged_cert.crt", dump_cert(minifi_client_cert) + dump_key(minifi_client_key))) - - clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, - ca_key=test_context.root_ca_key) - self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert))) - self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key))) - - minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", - ca_cert=test_context.root_ca_cert, - ca_key=test_context.root_ca_key) + File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert)) + ) + self.files.append( + File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key)) + ) + self.files.append( + File( + "/tmp/resources/minifi_merged_cert.crt", + dump_cert(minifi_client_cert) + dump_key(minifi_client_key), + ) + ) + + clientuser_cert, clientuser_key = make_client_cert( + "clientuser", + ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key, + ) self.files.append( - File("/tmp/resources/minifi_server.crt", dump_cert(cert=minifi_server_cert) + dump_key(minifi_server_key))) + File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert)) + ) + self.files.append( + File("/tmp/resources/clientuser.key", dump_key(clientuser_key)) + ) + + minifi_server_cert, minifi_server_key = make_server_cert( + common_name=f"server-{test_context.scenario_id}", + ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key, + ) + self.files.append( + File( + "/tmp/resources/minifi_server.crt", + dump_cert(cert=minifi_server_cert) + dump_key(minifi_server_key), + ) + ) self._fill_default_properties() self._fill_default_log_properties() def deploy(self, context: MinifiTestContext | None) -> bool: flow_config = self.flow_definition.to_yaml() - logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}") - self.files.append(File(f"{self.deployment_type.conf_path}/config.yml", flow_config)) + logger.info( + f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}" + ) + self.files.append( + File(f"{self.deployment_type.conf_path}/config.yml", flow_config) + ) self.files.append( - File(f"{self.deployment_type.conf_path}/minifi.properties", self._get_properties_file_content())) + File( + f"{self.deployment_type.conf_path}/minifi.properties", + self._get_properties_file_content(), + ) + ) self.files.append( - File(f"{self.deployment_type.conf_path}/minifi-log.properties", self._get_log_properties_file_content())) - resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller" + File( + f"{self.deployment_type.conf_path}/minifi-log.properties", + self._get_log_properties_file_content(), + ) + ) + resource_dir = ( + Path(__file__).resolve().parent / "resources" / "minifi-controller" + ) self.host_files.append( - HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml"))) + HostFile( + "/tmp/resources/minifi-controller/config.yml", + os.path.join(resource_dir, "config.yml"), + ) + ) if not super().deploy(context): return False finished_str = "MiNiFi started" - return wait_for_condition(condition=lambda: finished_str in self.get_logs(), - timeout_seconds=self.deploy_timeout_seconds, bail_condition=lambda: self.exited, - context=context) + return wait_for_condition( + condition=lambda: finished_str in self.get_logs(), + timeout_seconds=self.deploy_timeout_seconds, + bail_condition=lambda: self.exited, + context=context, + ) def set_deploy_timeout_seconds(self, timeout_seconds: int): self.deploy_timeout_seconds = timeout_seconds @@ -127,13 +196,17 @@ def set_log_property(self, key: str, value: str): self.log_properties[key] = value def _fill_default_properties(self): - self.properties["nifi.flow.configuration.file"] = f"{self.deployment_type.conf_path}/config.yml" + self.properties["nifi.flow.configuration.file"] = ( + f"{self.deployment_type.conf_path}/config.yml" + ) self.properties["nifi.extension.path"] = self.deployment_type.extension_pattern self.properties["nifi.administrative.yield.duration"] = "1 sec" self.properties["nifi.bored.yield.duration"] = "100 millis" self.properties["nifi.openssl.fips.support.enable"] = "false" self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository" - self.properties["nifi.python.processor.dir"] = self.deployment_type.minifi_python_path + self.properties["nifi.python.processor.dir"] = ( + self.deployment_type.minifi_python_path + ) def _fill_default_log_properties(self): self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v" @@ -151,5 +224,5 @@ def _get_log_properties_file_content(self): return "\n".join(lines) def add_example_python_processors(self): - run_minifi_cmd = f'{self.deployment_type.bin_path}/minifi' + run_minifi_cmd = f"{self.deployment_type.bin_path}/minifi" self.command = f'sh -c "cp -r {self.deployment_type.minifi_python_examples_path} {self.deployment_type.minifi_python_path}/examples && {run_minifi_cmd}"' diff --git a/behave_framework/src/minifi_behave/containers/minifi_protocol.py b/behave_framework/src/minifi_behave/containers/minifi_protocol.py index 1293d8a0e3..0dee498b2f 100644 --- a/behave_framework/src/minifi_behave/containers/minifi_protocol.py +++ b/behave_framework/src/minifi_behave/containers/minifi_protocol.py @@ -21,14 +21,11 @@ class MinifiProtocol(Protocol): flow_definition: FlowDefinition - def set_property(self, key: str, value: str): - ... + def set_property(self, key: str, value: str): ... - def set_log_property(self, key: str, value: str): - ... + def set_log_property(self, key: str, value: str): ... - def set_deploy_timeout_seconds(self, timeout_seconds: int): - ... + def set_deploy_timeout_seconds(self, timeout_seconds: int): ... def enable_openssl_fips_mode(minifi: MinifiProtocol): @@ -36,18 +33,30 @@ def enable_openssl_fips_mode(minifi: MinifiProtocol): def enable_log_metrics_publisher(minifi: MinifiProtocol): - minifi.set_property("nifi.metrics.publisher.LogMetricsPublisher.metrics", "RepositoryMetrics") - minifi.set_property("nifi.metrics.publisher.LogMetricsPublisher.logging.interval", "1s") + minifi.set_property( + "nifi.metrics.publisher.LogMetricsPublisher.metrics", "RepositoryMetrics" + ) + minifi.set_property( + "nifi.metrics.publisher.LogMetricsPublisher.logging.interval", "1s" + ) minifi.set_property("nifi.metrics.publisher.class", "LogMetricsPublisher") def conf_c2_flow_url(minifi: MinifiProtocol, scenario_id: str): - minifi.set_property("nifi.c2.flow.url", - f"http://minifi-c2-server-{scenario_id}:10090/c2/config?class=minifi-test-class") + minifi.set_property( + "nifi.c2.flow.url", + f"http://minifi-c2-server-{scenario_id}:10090/c2/config?class=minifi-test-class", + ) def set_up_ssl_properties(minifi: MinifiProtocol): minifi.set_property("nifi.remote.input.secure", "true") - minifi.set_property("nifi.security.client.certificate", "/tmp/resources/minifi_client.crt") - minifi.set_property("nifi.security.client.private.key", "/tmp/resources/minifi_client.key") - minifi.set_property("nifi.security.client.ca.certificate", "/tmp/resources/root_ca.crt") + minifi.set_property( + "nifi.security.client.certificate", "/tmp/resources/minifi_client.crt" + ) + minifi.set_property( + "nifi.security.client.private.key", "/tmp/resources/minifi_client.key" + ) + minifi.set_property( + "nifi.security.client.ca.certificate", "/tmp/resources/root_ca.crt" + ) diff --git a/behave_framework/src/minifi_behave/containers/minifi_win_container.py b/behave_framework/src/minifi_behave/containers/minifi_win_container.py index 9ece439b47..4c77723d8c 100644 --- a/behave_framework/src/minifi_behave/containers/minifi_win_container.py +++ b/behave_framework/src/minifi_behave/containers/minifi_win_container.py @@ -14,17 +14,25 @@ # limitations under the License. +import logging + +from minifi_behave.containers.directory import Directory from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.minifi.minifi_flow_definition import MinifiFlowDefinition -from minifi_behave.containers.directory import Directory + from .container_windows import WindowsContainer from .minifi_protocol import MinifiProtocol -import logging + +logger = logging.getLogger(__name__) class MinifiWindowsContainer(WindowsContainer, MinifiProtocol): def __init__(self, container_name: str, test_context: MinifiTestContext): - super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network) + super().__init__( + test_context.minifi_container_image, + f"{container_name}-{test_context.scenario_id}", + test_context.network, + ) self.flow_config_str: str = "" self.flow_definition = MinifiFlowDefinition() self.properties: dict[str, str] = {} @@ -34,11 +42,13 @@ def __init__(self, container_name: str, test_context: MinifiTestContext): self._fill_default_log_properties() def deploy(self, context: MinifiTestContext | None) -> bool: - logging.info(self.flow_definition.to_yaml()) + logger.info(self.flow_definition.to_yaml()) conf_dir = Directory("\\Program Files\\ApacheNiFiMiNiFi\\nifi-minifi-cpp\\conf") conf_dir.add_file("config.yml", self.flow_definition.to_yaml()) conf_dir.add_file("minifi.properties", self._get_properties_file_content()) - conf_dir.add_file("minifi-log.properties", self._get_log_properties_file_content()) + conf_dir.add_file( + "minifi-log.properties", self._get_log_properties_file_content() + ) self.dirs.append(conf_dir) return super().deploy(context) diff --git a/behave_framework/src/minifi_behave/containers/nifi_container.py b/behave_framework/src/minifi_behave/containers/nifi_container.py index 8e36c4a1b6..36e710305c 100644 --- a/behave_framework/src/minifi_behave/containers/nifi_container.py +++ b/behave_framework/src/minifi_behave/containers/nifi_container.py @@ -13,82 +13,113 @@ # See the License for the specific language governing permissions and # limitations under the License. -import io import gzip +import io import logging import os from pathlib import Path -from minifi_behave.containers.file import File from minifi_behave.containers.container_linux import LinuxContainer +from minifi_behave.containers.file import File +from minifi_behave.containers.host_file import HostFile from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert from minifi_behave.minifi.nifi_flow_definition import NifiFlowDefinition -from minifi_behave.containers.host_file import HostFile -from minifi_behave.core.ssl_utils import make_server_cert -from minifi_behave.core.ssl_utils import dump_cert, dump_key +logger = logging.getLogger(__name__) class NifiContainer(LinuxContainer): - def __init__(self, test_context: MinifiTestContext, command: list[str] | None = None, use_ssl: bool = False): + def __init__( + self, + test_context: MinifiTestContext, + command: list[str] | None = None, + use_ssl: bool = False, + ): self.flow_definition = NifiFlowDefinition() name = f"nifi-{test_context.scenario_id}" if use_ssl: - entry_command = (r"/scripts/convert_cert_to_jks.sh /tmp/resources /tmp/resources/nifi_client.key /tmp/resources/nifi_client.crt /tmp/resources/root_ca.crt &&" - r"sed -i -e 's/^\(nifi.remote.input.host\)=.*/\1={name}/' " - r"-e 's/^\(nifi.remote.input.secure\)=.*/\1=true/' " - r"-e 's/^\(nifi.sensitive.props.key\)=.*/\1=secret_key_12345/' " - r"-e 's/^\(nifi.web.https.port\)=.*/\1=8443/' " - r"-e 's/^\(nifi.web.https.host\)=.*/\1={name}/' " - r"-e 's/^\(nifi.security.keystore\)=.*/\1=\/tmp\/resources\/keystore.jks/' " - r"-e 's/^\(nifi.security.keystoreType\)=.*/\1=jks/' " - r"-e 's/^\(nifi.security.keystorePasswd\)=.*/\1=passw0rd1!/' " - r"-e 's/^\(nifi.security.keyPasswd\)=.*/#\1=passw0rd1!/' " - r"-e 's/^\(nifi.security.truststore\)=.*/\1=\/tmp\/resources\/truststore.jks/' " - r"-e 's/^\(nifi.security.truststoreType\)=.*/\1=jks/' " - r"-e 's/^\(nifi.security.truststorePasswd\)=.*/\1=passw0rd1!/' " - r"-e 's/^\(nifi.remote.input.socket.port\)=.*/\1=10443/' /opt/nifi/nifi-current/conf/nifi.properties && " - r"cp /tmp/nifi_config/flow.json.gz /opt/nifi/nifi-current/conf && /opt/nifi/nifi-current/bin/nifi.sh run & " - r"nifi_pid=$! &&" - r"tail -F --pid=${{nifi_pid}} /opt/nifi/nifi-current/logs/nifi-app.log").format(name=name) + entry_command = ( + r"/scripts/convert_cert_to_jks.sh /tmp/resources /tmp/resources/nifi_client.key /tmp/resources/nifi_client.crt /tmp/resources/root_ca.crt &&" + rf"sed -i -e 's/^\(nifi.remote.input.host\)=.*/\1={name}/' " + r"-e 's/^\(nifi.remote.input.secure\)=.*/\1=true/' " + r"-e 's/^\(nifi.sensitive.props.key\)=.*/\1=secret_key_12345/' " + r"-e 's/^\(nifi.web.https.port\)=.*/\1=8443/' " + rf"-e 's/^\(nifi.web.https.host\)=.*/\1={name}/' " + r"-e 's/^\(nifi.security.keystore\)=.*/\1=\/tmp\/resources\/keystore.jks/' " + r"-e 's/^\(nifi.security.keystoreType\)=.*/\1=jks/' " + r"-e 's/^\(nifi.security.keystorePasswd\)=.*/\1=passw0rd1!/' " + r"-e 's/^\(nifi.security.keyPasswd\)=.*/#\1=passw0rd1!/' " + r"-e 's/^\(nifi.security.truststore\)=.*/\1=\/tmp\/resources\/truststore.jks/' " + r"-e 's/^\(nifi.security.truststoreType\)=.*/\1=jks/' " + r"-e 's/^\(nifi.security.truststorePasswd\)=.*/\1=passw0rd1!/' " + r"-e 's/^\(nifi.remote.input.socket.port\)=.*/\1=10443/' /opt/nifi/nifi-current/conf/nifi.properties && " + r"cp /tmp/nifi_config/flow.json.gz /opt/nifi/nifi-current/conf && /opt/nifi/nifi-current/bin/nifi.sh run & " + r"nifi_pid=$! &&" + r"tail -F --pid=${nifi_pid} /opt/nifi/nifi-current/logs/nifi-app.log" + ) else: - entry_command = (r"sed -i -e 's/^\(nifi.remote.input.host\)=.*/\1={name}/' " - r"-e 's/^\(nifi.sensitive.props.key\)=.*/\1=secret_key_12345/' " - r"-e 's/^\(nifi.remote.input.secure\)=.*/\1=false/' " - r"-e 's/^\(nifi.web.http.port\)=.*/\1=8080/' " - r"-e 's/^\(nifi.web.https.port\)=.*/\1=/' " - r"-e 's/^\(nifi.web.https.host\)=.*/\1=/' " - r"-e 's/^\(nifi.web.http.host\)=.*/\1={name}/' " - r"-e 's/^\(nifi.security.keystore\)=.*/\1=/' " - r"-e 's/^\(nifi.security.keystoreType\)=.*/\1=/' " - r"-e 's/^\(nifi.security.keystorePasswd\)=.*/\1=/' " - r"-e 's/^\(nifi.security.keyPasswd\)=.*/\1=/' " - r"-e 's/^\(nifi.security.truststore\)=.*/\1=/' " - r"-e 's/^\(nifi.security.truststoreType\)=.*/\1=/' " - r"-e 's/^\(nifi.security.truststorePasswd\)=.*/\1=/' " - r"-e 's/^\(nifi.remote.input.socket.port\)=.*/\1=10000/' /opt/nifi/nifi-current/conf/nifi.properties && " - r"cp /tmp/nifi_config/flow.json.gz /opt/nifi/nifi-current/conf && /opt/nifi/nifi-current/bin/nifi.sh run & " - r"nifi_pid=$! &&" - r"tail -F --pid=${{nifi_pid}} /opt/nifi/nifi-current/logs/nifi-app.log").format(name=name) + entry_command = ( + rf"sed -i -e 's/^\(nifi.remote.input.host\)=.*/\1={name}/' " + r"-e 's/^\(nifi.sensitive.props.key\)=.*/\1=secret_key_12345/' " + r"-e 's/^\(nifi.remote.input.secure\)=.*/\1=false/' " + r"-e 's/^\(nifi.web.http.port\)=.*/\1=8080/' " + r"-e 's/^\(nifi.web.https.port\)=.*/\1=/' " + r"-e 's/^\(nifi.web.https.host\)=.*/\1=/' " + rf"-e 's/^\(nifi.web.http.host\)=.*/\1={name}/' " + r"-e 's/^\(nifi.security.keystore\)=.*/\1=/' " + r"-e 's/^\(nifi.security.keystoreType\)=.*/\1=/' " + r"-e 's/^\(nifi.security.keystorePasswd\)=.*/\1=/' " + r"-e 's/^\(nifi.security.keyPasswd\)=.*/\1=/' " + r"-e 's/^\(nifi.security.truststore\)=.*/\1=/' " + r"-e 's/^\(nifi.security.truststoreType\)=.*/\1=/' " + r"-e 's/^\(nifi.security.truststorePasswd\)=.*/\1=/' " + r"-e 's/^\(nifi.remote.input.socket.port\)=.*/\1=10000/' /opt/nifi/nifi-current/conf/nifi.properties && " + r"cp /tmp/nifi_config/flow.json.gz /opt/nifi/nifi-current/conf && /opt/nifi/nifi-current/bin/nifi.sh run & " + r"nifi_pid=$! &&" + r"tail -F --pid=${nifi_pid} /opt/nifi/nifi-current/logs/nifi-app.log" + ) if not command: command = ["/bin/sh", "-c", entry_command] - super().__init__("apache/nifi:" + NifiFlowDefinition.NIFI_VERSION, name, test_context.network, entrypoint=command) + super().__init__( + "apache/nifi:" + NifiFlowDefinition.NIFI_VERSION, + name, + test_context.network, + entrypoint=command, + ) resource_dir = Path(__file__).resolve().parent / "resources" / "nifi" - self.host_files.append(HostFile("/scripts/convert_cert_to_jks.sh", os.path.join(resource_dir, "convert_cert_to_jks.sh"))) + self.host_files.append( + HostFile( + "/scripts/convert_cert_to_jks.sh", + os.path.join(resource_dir, "convert_cert_to_jks.sh"), + ) + ) - nifi_client_cert, nifi_client_key = make_server_cert(common_name=f"nifi-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) - self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert))) - self.files.append(File("/tmp/resources/nifi_client.crt", dump_cert(nifi_client_cert))) - self.files.append(File("/tmp/resources/nifi_client.key", dump_key(nifi_client_key))) + nifi_client_cert, nifi_client_key = make_server_cert( + common_name=f"nifi-{test_context.scenario_id}", + ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key, + ) + self.files.append( + File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert)) + ) + self.files.append( + File("/tmp/resources/nifi_client.crt", dump_cert(nifi_client_cert)) + ) + self.files.append( + File("/tmp/resources/nifi_client.key", dump_key(nifi_client_key)) + ) def deploy(self, context: MinifiTestContext | None) -> bool: flow_config = self.flow_definition.to_json() - logging.info(f"Deploying NiFi container '{self.container_name}' with flow configuration:\n{flow_config}") + logger.info( + f"Deploying NiFi container '{self.container_name}' with flow configuration:\n{flow_config}" + ) buffer = io.BytesIO() - with gzip.GzipFile(fileobj=buffer, mode='wb') as gz_file: + with gzip.GzipFile(fileobj=buffer, mode="wb") as gz_file: gz_file.write(flow_config.encode()) gzipped_bytes = buffer.getvalue() @@ -100,4 +131,5 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=300, bail_condition=lambda: self.exited, - context=context) + context=context, + ) diff --git a/behave_framework/src/minifi_behave/core/helpers.py b/behave_framework/src/minifi_behave/core/helpers.py index 128aa4e017..715b78b54c 100644 --- a/behave_framework/src/minifi_behave/core/helpers.py +++ b/behave_framework/src/minifi_behave/core/helpers.py @@ -17,14 +17,17 @@ from __future__ import annotations +import functools import logging import time -import functools from collections.abc import Callable -import docker from minifi_behave.core.minifi_test_context import MinifiTestContext +import docker + +logger = logging.getLogger(__name__) + def log_due_to_failure(context: MinifiTestContext | None): if context is not None: @@ -32,19 +35,25 @@ def log_due_to_failure(context: MinifiTestContext | None): container.log_app_output() -def check_condition_after_wait(condition: Callable[[], bool], context: MinifiTestContext | None, wait_time: int) -> bool: +def check_condition_after_wait( + condition: Callable[[], bool], context: MinifiTestContext | None, wait_time: int +) -> bool: time.sleep(wait_time) if not condition(): - logging.warning("Condition not met after wait") + logger.warning("Condition not met after wait") log_due_to_failure(context) return False return True -def wait_for_condition(condition: Callable[[], bool], timeout_seconds: float, bail_condition: Callable[[], bool], - context: MinifiTestContext | None) -> bool: +def wait_for_condition( + condition: Callable[[], bool], + timeout_seconds: float, + bail_condition: Callable[[], bool], + context: MinifiTestContext | None, +) -> bool: if bail_condition(): - logging.warning("Bail condition evaluated to 'True', aborting wait.") + logger.warning("Bail condition evaluated to 'True', aborting wait.") log_due_to_failure(context) return False start_time = time.monotonic() @@ -53,7 +62,7 @@ def wait_for_condition(condition: Callable[[], bool], timeout_seconds: float, ba if condition(): return True if bail_condition(): - logging.warning("Bail condition evaluated to 'True', aborting wait.") + logger.warning("Bail condition evaluated to 'True', aborting wait.") log_due_to_failure(context) return False remaining_time = timeout_seconds - (time.monotonic() - start_time) @@ -61,23 +70,27 @@ def wait_for_condition(condition: Callable[[], bool], timeout_seconds: float, ba if sleep_time > 0: time.sleep(sleep_time) except Exception as ex: - logging.warning("Exception while waiting for condition: %s", ex) + logger.warning("Exception while waiting for condition: %s", ex) log_due_to_failure(context) return False - logging.warning("Timed out after %d seconds while waiting for condition", timeout_seconds) + logger.warning( + f"Timed out after {timeout_seconds} seconds while waiting for condition" + ) log_due_to_failure(context) return False def run_cmd_in_docker_image(image_name: str, cmd: str | list, network: str) -> str: client = docker.from_env() - output = client.containers.run(image=image_name, - command=cmd, - remove=True, - stdout=True, - stderr=True, - network=network, - detach=False) + output = client.containers.run( + image=image_name, + command=cmd, + remove=True, + stdout=True, + stderr=True, + network=network, + detach=False, + ) return output.decode("utf-8") @@ -90,6 +103,7 @@ def retry_check(max_tries: int = 5, retry_interval_seconds: int = 1): Decorator for retrying a checker function that returns a boolean. The decorated function is called repeatedly until it returns True or the maximum number of attempts is reached. The maximum number of attempts and the interval between attempts in seconds can be configured. """ + def retry_check_func(func): @functools.wraps(func) def retry_wrapper(*args, **kwargs): @@ -99,5 +113,7 @@ def retry_wrapper(*args, **kwargs): if i < max_tries - 1: time.sleep(retry_interval_seconds) return False + return retry_wrapper + return retry_check_func diff --git a/behave_framework/src/minifi_behave/core/hooks.py b/behave_framework/src/minifi_behave/core/hooks.py index bec809f1ce..ff0a33b0d0 100644 --- a/behave_framework/src/minifi_behave/core/hooks.py +++ b/behave_framework/src/minifi_behave/core/hooks.py @@ -14,24 +14,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os -import docker import types -import logging from pathlib import Path from behave.model import Scenario from behave.runner import Context - -from minifi_behave.core.ssl_utils import make_self_signed_cert from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import make_self_signed_cert + +import docker + +logger = logging.getLogger(__name__) def get_minifi_container_image(): - if 'MINIFI_TAG_PREFIX' in os.environ and 'MINIFI_VERSION' in os.environ: - minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] - minifi_version = os.environ['MINIFI_VERSION'] - return 'apacheminificpp:' + minifi_tag_prefix + minifi_version + if "MINIFI_TAG_PREFIX" in os.environ and "MINIFI_VERSION" in os.environ: + minifi_tag_prefix = os.environ["MINIFI_TAG_PREFIX"] + minifi_version = os.environ["MINIFI_VERSION"] + return "apacheminificpp:" + minifi_tag_prefix + minifi_version return "apacheminificpp:behave" @@ -40,13 +42,18 @@ def inject_scenario_id(context: MinifiTestContext, step): step.name = step.name.replace("${scenario_id}", context.scenario_id) if getattr(step, "table", None): for row in step.table: - row.cells = [cell.replace("${scenario_id}", context.scenario_id) if "${scenario_id}" in cell else cell for cell in row.cells] + row.cells = [ + cell.replace("${scenario_id}", context.scenario_id) + if "${scenario_id}" in cell + else cell + for cell in row.cells + ] if hasattr(step, "text") and step.text and "${scenario_id}" in step.text: step.text = step.text.replace("${scenario_id}", context.scenario_id) def common_before_scenario(context: Context, scenario: Scenario): - if "SUPPORTS_WINDOWS" not in scenario.effective_tags and os.name == 'nt': + if "SUPPORTS_WINDOWS" not in scenario.effective_tags and os.name == "nt": scenario.skip("No windows support") return @@ -55,21 +62,24 @@ def common_before_scenario(context: Context, scenario: Scenario): method_map = { "get_or_create_minifi_container": MinifiTestContext.get_or_create_minifi_container, - "get_or_create_default_minifi_container": MinifiTestContext.get_or_create_default_minifi_container + "get_or_create_default_minifi_container": MinifiTestContext.get_or_create_default_minifi_container, } for attr, method in method_map.items(): if not hasattr(context, attr): setattr(context, attr, types.MethodType(method, context)) - logging.info("Running scenario: %s", scenario) - context.scenario_id = scenario.filename.rsplit("/", 1)[1].split(".")[0] + "-" + str( - scenario.parent.scenarios.index(scenario)) + logger.info("Running scenario: %s", scenario) + context.scenario_id = ( + scenario.filename.rsplit("/", 1)[1].split(".")[0] + + "-" + + str(scenario.parent.scenarios.index(scenario)) + ) network_name = f"{context.scenario_id}-net" docker_client = docker.client.from_env() try: existing_network = docker_client.networks.get(network_name) - logging.warning(f"Found existing network '{network_name}'. Removing it first.") + logger.warning(f"Found existing network '{network_name}'. Removing it first.") existing_network.remove() except docker.errors.NotFound: pass # No existing network found, which is good. @@ -98,8 +108,8 @@ def common_after_scenario(context: MinifiTestContext, scenario: Scenario): with open(scenario_info_path, "w") as f: f.write(header) - if hasattr(context, 'containers'): + if hasattr(context, "containers"): for container in context.containers.values(): container.clean_up() - if hasattr(context, 'network'): + if hasattr(context, "network"): context.network.remove() diff --git a/behave_framework/src/minifi_behave/core/minifi_test_context.py b/behave_framework/src/minifi_behave/core/minifi_test_context.py index 59a1e04f2c..1d52b16be3 100644 --- a/behave_framework/src/minifi_behave/core/minifi_test_context.py +++ b/behave_framework/src/minifi_behave/core/minifi_test_context.py @@ -18,15 +18,15 @@ from __future__ import annotations import os -import docker + from behave.runner import Context from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.x509 import Certificate from docker.models.networks import Network - from minifi_behave.containers.container_protocol import ContainerProtocol from minifi_behave.containers.minifi_protocol import MinifiProtocol +import docker DEFAULT_MINIFI_CONTAINER_NAME = "minifi-primary" @@ -47,15 +47,32 @@ class MinifiTestContext(Context): def get_or_create_minifi_container(self, container_name: str) -> MinifiContainer: if container_name not in self.containers: - if os.name == 'nt': - from minifi_behave.containers.minifi_win_container import MinifiWindowsContainer + if os.name == "nt": + from minifi_behave.containers.minifi_win_container import ( + MinifiWindowsContainer, + ) + minifi_container = MinifiWindowsContainer(container_name, self) - elif 'MINIFI_INSTALLATION_TYPE=FHS' in str(docker.from_env().images.get(self.minifi_container_image).history()): - from minifi_behave.containers.minifi_linux_container import MinifiLinuxContainer, FHSDeployment - minifi_container = MinifiLinuxContainer(container_name, self, FHSDeployment()) + elif "MINIFI_INSTALLATION_TYPE=FHS" in str( + docker.from_env().images.get(self.minifi_container_image).history() + ): + from minifi_behave.containers.minifi_linux_container import ( + FHSDeployment, + MinifiLinuxContainer, + ) + + minifi_container = MinifiLinuxContainer( + container_name, self, FHSDeployment() + ) else: - from minifi_behave.containers.minifi_linux_container import MinifiLinuxContainer, NormalDeployment - minifi_container = MinifiLinuxContainer(container_name, self, NormalDeployment()) + from minifi_behave.containers.minifi_linux_container import ( + MinifiLinuxContainer, + NormalDeployment, + ) + + minifi_container = MinifiLinuxContainer( + container_name, self, NormalDeployment() + ) self.containers[container_name] = minifi_container return self.containers[container_name] diff --git a/behave_framework/src/minifi_behave/core/ssl_utils.py b/behave_framework/src/minifi_behave/core/ssl_utils.py index 3fc3c1de52..1e16f38611 100644 --- a/behave_framework/src/minifi_behave/core/ssl_utils.py +++ b/behave_framework/src/minifi_behave/core/ssl_utils.py @@ -16,12 +16,11 @@ import datetime from cryptography import x509 -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.x509 import Certificate, ExtendedKeyUsage -from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID def gen_cert() -> tuple[Certificate, RSAPrivateKey]: @@ -41,7 +40,9 @@ def gen_cert() -> tuple[Certificate, RSAPrivateKey]: .public_key(key.public_key()) .serial_number(x509.random_serial_number()) .not_valid_before(datetime.datetime.now(datetime.timezone.utc)) - .not_valid_after(datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365)) + .not_valid_after( + datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365) + ) .sign(key, hashes.SHA256()) ) @@ -64,8 +65,12 @@ def make_self_signed_cert(common_name: str) -> tuple[Certificate, RSAPrivateKey] .public_key(key.public_key()) .serial_number(x509.random_serial_number()) .not_valid_before(datetime.datetime.now(datetime.timezone.utc)) - .not_valid_after(datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=3650)) - .add_extension(x509.SubjectKeyIdentifier.from_public_key(key.public_key()), critical=False) + .not_valid_after( + datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=3650) + ) + .add_extension( + x509.SubjectKeyIdentifier.from_public_key(key.public_key()), critical=False + ) .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) .sign(key, hashes.SHA256()) ) @@ -94,7 +99,9 @@ def _make_cert( .public_key(key.public_key()) .serial_number(x509.random_serial_number()) .not_valid_before(datetime.datetime.now(datetime.timezone.utc)) - .not_valid_after(datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=3650)) + .not_valid_after( + datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=3650) + ) .add_extension( x509.BasicConstraints(ca=False, path_length=None), critical=True, @@ -110,13 +117,17 @@ def _make_cert( ) if extended_key_usage: - builder = builder.add_extension(x509.ExtendedKeyUsage(extended_key_usage), critical=False) + builder = builder.add_extension( + x509.ExtendedKeyUsage(extended_key_usage), critical=False + ) cert = builder.sign(ca_key, hashes.SHA256()) return cert, key -def make_client_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey) -> tuple[Certificate, RSAPrivateKey]: +def make_client_cert( + common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey +) -> tuple[Certificate, RSAPrivateKey]: return _make_cert( common_name, ca_cert, @@ -125,7 +136,9 @@ def make_client_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateK ) -def make_server_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey) -> tuple[Certificate, RSAPrivateKey]: +def make_server_cert( + common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey +) -> tuple[Certificate, RSAPrivateKey]: return _make_cert( common_name, ca_cert, @@ -134,7 +147,9 @@ def make_server_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateK ) -def make_cert_without_extended_usage(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey) -> tuple[Certificate, RSAPrivateKey]: +def make_cert_without_extended_usage( + common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey +) -> tuple[Certificate, RSAPrivateKey]: return _make_cert(common_name, ca_cert, ca_key, None) diff --git a/behave_framework/src/minifi_behave/minifi/controller_service.py b/behave_framework/src/minifi_behave/minifi/controller_service.py index 58a9136ae2..2af194e1e3 100644 --- a/behave_framework/src/minifi_behave/minifi/controller_service.py +++ b/behave_framework/src/minifi_behave/minifi/controller_service.py @@ -30,6 +30,11 @@ def add_property(self, property_name: str, property_value: str): self.properties[property_name] = property_value def to_yaml_dict(self) -> dict: - data = {'name': self.name, 'id': self.id, 'class': self.class_name, 'Properties': self.properties} + data = { + "name": self.name, + "id": self.id, + "class": self.class_name, + "Properties": self.properties, + } return data diff --git a/behave_framework/src/minifi_behave/minifi/flow_definition.py b/behave_framework/src/minifi_behave/minifi/flow_definition.py index a636cf353a..829b1f84eb 100644 --- a/behave_framework/src/minifi_behave/minifi/flow_definition.py +++ b/behave_framework/src/minifi_behave/minifi/flow_definition.py @@ -20,11 +20,11 @@ from .connection import Connection from .controller_service import ControllerService from .funnel import Funnel +from .input_port import InputPort +from .output_port import OutputPort from .parameter_context import ParameterContext from .processor import Processor from .remote_process_group import RemoteProcessGroup -from .input_port import InputPort -from .output_port import OutputPort class FlowDefinition(ABC): @@ -46,39 +46,59 @@ def add_remote_process_group(self, address: str, name: str, protocol: str = "RAW rpg = RemoteProcessGroup(name, address, protocol) self.remote_process_groups.append(rpg) - def add_input_port_to_rpg(self, rpg_name: str, port_name: str, use_compression: bool = False): - rpg = next((rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None) + def add_input_port_to_rpg( + self, rpg_name: str, port_name: str, use_compression: bool = False + ): + rpg = next( + (rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None + ) if rpg: rpg.add_input_port(port_name, use_compression) else: raise ValueError(f"RemoteProcessGroup with name '{rpg_name}' not found.") - def add_output_port_to_rpg(self, rpg_name: str, port_name: str, use_compression: bool = False): - rpg = next((rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None) + def add_output_port_to_rpg( + self, rpg_name: str, port_name: str, use_compression: bool = False + ): + rpg = next( + (rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None + ) if rpg: rpg.add_output_port(port_name, use_compression) else: raise ValueError(f"RemoteProcessGroup with name '{rpg_name}' not found.") def get_input_port_id_of_rpg(self, rpg_name: str, rpg_port_name: str) -> str: - rpg = next((rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None) + rpg = next( + (rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None + ) if rpg: - port = next((port for port in rpg.input_ports if port.name == rpg_port_name), None) + port = next( + (port for port in rpg.input_ports if port.name == rpg_port_name), None + ) if port: return port.id else: - raise ValueError(f"InputPort with name '{rpg_port_name}' not found in RPG '{rpg_name}'.") + raise ValueError( + f"InputPort with name '{rpg_port_name}' not found in RPG '{rpg_name}'." + ) else: raise ValueError(f"RemoteProcessGroup with name '{rpg_name}' not found.") def get_output_port_id_of_rpg(self, rpg_name: str, rpg_port_name: str) -> str: - rpg = next((rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None) + rpg = next( + (rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None + ) if rpg: - port = next((port for port in rpg.output_ports if port.name == rpg_port_name), None) + port = next( + (port for port in rpg.output_ports if port.name == rpg_port_name), None + ) if port: return port.id else: - raise ValueError(f"OutputPort with name '{rpg_port_name}' not found in RPG '{rpg_name}'.") + raise ValueError( + f"OutputPort with name '{rpg_port_name}' not found in RPG '{rpg_name}'." + ) else: raise ValueError(f"RemoteProcessGroup with name '{rpg_name}' not found.") @@ -89,17 +109,38 @@ def add_output_port(self, output_port_id: str, output_port_name: str): self.output_ports.append(OutputPort(output_port_name, output_port_id)) def get_processor(self, processor_name: str) -> Processor | None: - return next((proc for proc in self.processors if proc.name == processor_name), None) - - def get_controller_service(self, controller_service_name: str) -> ControllerService | None: - return next((controller for controller in self.controller_services if controller.name == controller_service_name), None) - - def get_parameter_context(self, parameter_context_name: str) -> ParameterContext | None: - return next((parameter_context for parameter_context in self.parameter_contexts if - parameter_context.name == parameter_context_name), None) + return next( + (proc for proc in self.processors if proc.name == processor_name), None + ) + + def get_controller_service( + self, controller_service_name: str + ) -> ControllerService | None: + return next( + ( + controller + for controller in self.controller_services + if controller.name == controller_service_name + ), + None, + ) + + def get_parameter_context( + self, parameter_context_name: str + ) -> ParameterContext | None: + return next( + ( + parameter_context + for parameter_context in self.parameter_contexts + if parameter_context.name == parameter_context_name + ), + None, + ) def get_remote_process_group(self, rpg_name: str) -> RemoteProcessGroup | None: - return next((rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None) + return next( + (rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None + ) def add_funnel(self, funnel: Funnel): self.funnels.append(funnel) diff --git a/behave_framework/src/minifi_behave/minifi/funnel.py b/behave_framework/src/minifi_behave/minifi/funnel.py index 998b453375..c38322e5a9 100644 --- a/behave_framework/src/minifi_behave/minifi/funnel.py +++ b/behave_framework/src/minifi_behave/minifi/funnel.py @@ -28,4 +28,7 @@ def __repr__(self): def to_yaml_dict(self) -> dict: # Funnels have a simpler representation in the MiNiFi YAML - return {'id': self.id, 'name': self.name, } + return { + "id": self.id, + "name": self.name, + } diff --git a/behave_framework/src/minifi_behave/minifi/input_port.py b/behave_framework/src/minifi_behave/minifi/input_port.py index 1af6d20972..3f7e51a5f1 100644 --- a/behave_framework/src/minifi_behave/minifi/input_port.py +++ b/behave_framework/src/minifi_behave/minifi/input_port.py @@ -15,6 +15,7 @@ # limitations under the License. # + class InputPort: def __init__(self, name: str, id: str): self.id: str = id @@ -26,16 +27,13 @@ def to_json_dict(self): "instanceIdentifier": self.id, "name": self.name, "comments": "", - "position": { - "x": 0, - "y": 0 - }, + "position": {"x": 0, "y": 0}, "type": "INPUT_PORT", "concurrentlySchedulableTaskCount": 1, "scheduledState": "RUNNING", "allowRemoteAccess": True, "portFunction": "STANDARD", "componentType": "INPUT_PORT", - "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8" + "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8", } return data diff --git a/behave_framework/src/minifi_behave/minifi/minifi_flow_definition.py b/behave_framework/src/minifi_behave/minifi/minifi_flow_definition.py index e01a91880a..67f77fde4c 100644 --- a/behave_framework/src/minifi_behave/minifi/minifi_flow_definition.py +++ b/behave_framework/src/minifi_behave/minifi/minifi_flow_definition.py @@ -31,23 +31,42 @@ def to_yaml(self) -> str: # This is crucial for finding the source/destination IDs for connections processors_by_name = {p.name: p for p in self.processors} funnels_by_name = {f.name: f for f in self.funnels} - remote_input_ports_by_name = {port.name: port for rpg in self.remote_process_groups for port in rpg.input_ports} - remote_output_ports_by_name = {port.name: port for rpg in self.remote_process_groups for port in rpg.output_ports} + remote_input_ports_by_name = { + port.name: port + for rpg in self.remote_process_groups + for port in rpg.input_ports + } + remote_output_ports_by_name = { + port.name: port + for rpg in self.remote_process_groups + for port in rpg.output_ports + } - connectables_by_name = {**processors_by_name, **funnels_by_name, **remote_input_ports_by_name, **remote_output_ports_by_name} + connectables_by_name = { + **processors_by_name, + **funnels_by_name, + **remote_input_ports_by_name, + **remote_output_ports_by_name, + } if len(self.parameter_contexts) > 0: parameter_context_name = self.parameter_contexts[0].name else: - parameter_context_name = '' + parameter_context_name = "" # Build the final dictionary structure - config = {'MiNiFi Config Version': 3, 'Flow Controller': {'name': self.flow_name}, - 'Parameter Contexts': [p.to_yaml_dict() for p in self.parameter_contexts], - 'Processors': [p.to_yaml_dict() for p in self.processors], - 'Funnels': [f.to_yaml_dict() for f in self.funnels], 'Connections': [], - 'Controller Services': [c.to_yaml_dict() for c in self.controller_services], - 'Remote Processing Groups': [rpg.to_yaml_dict() for rpg in self.remote_process_groups], - 'Parameter Context Name': parameter_context_name} + config = { + "MiNiFi Config Version": 3, + "Flow Controller": {"name": self.flow_name}, + "Parameter Contexts": [p.to_yaml_dict() for p in self.parameter_contexts], + "Processors": [p.to_yaml_dict() for p in self.processors], + "Funnels": [f.to_yaml_dict() for f in self.funnels], + "Connections": [], + "Controller Services": [c.to_yaml_dict() for c in self.controller_services], + "Remote Processing Groups": [ + rpg.to_yaml_dict() for rpg in self.remote_process_groups + ], + "Parameter Context Name": parameter_context_name, + } # Build the connections list by looking up processor IDs for conn in self.connections: @@ -56,11 +75,18 @@ def to_yaml(self) -> str: if not source_proc or not dest_proc: raise ValueError( - f"Could not find processors for connection from '{conn.source_name}' to '{conn.target_name}'") + f"Could not find processors for connection from '{conn.source_name}' to '{conn.target_name}'" + ) - config['Connections'].append( - {'name': f"{conn.source_name}/{conn.source_relationship}/{conn.target_name}", 'id': conn.id, - 'source id': source_proc.id, 'source relationship name': conn.source_relationship, - 'destination id': dest_proc.id, "drop empty": conn.drop_empty_flowfiles}) + config["Connections"].append( + { + "name": f"{conn.source_name}/{conn.source_relationship}/{conn.target_name}", + "id": conn.id, + "source id": source_proc.id, + "source relationship name": conn.source_relationship, + "destination id": dest_proc.id, + "drop empty": conn.drop_empty_flowfiles, + } + ) return yaml.dump(config, sort_keys=False, indent=2, width=120) diff --git a/behave_framework/src/minifi_behave/minifi/nifi_flow_definition.py b/behave_framework/src/minifi_behave/minifi/nifi_flow_definition.py index a41e248bcf..a957be23a9 100644 --- a/behave_framework/src/minifi_behave/minifi/nifi_flow_definition.py +++ b/behave_framework/src/minifi_behave/minifi/nifi_flow_definition.py @@ -17,21 +17,19 @@ import json import uuid + from .flow_definition import FlowDefinition class NifiFlowDefinition(FlowDefinition): - NIFI_VERSION: str = '2.7.2' + NIFI_VERSION: str = "2.7.2" def __init__(self, flow_name: str = "NiFi Flow"): super().__init__(flow_name) def to_json(self) -> str: config = { - "encodingVersion": { - "majorVersion": 2, - "minorVersion": 0 - }, + "encodingVersion": {"majorVersion": 2, "minorVersion": 0}, "maxTimerDrivenThreadCount": 10, "maxEventDrivenThreadCount": 1, "registries": [], @@ -45,10 +43,7 @@ def to_json(self) -> str: "instanceIdentifier": str(uuid.uuid4()), "name": "NiFi Flow", "comments": "", - "position": { - "x": 0, - "y": 0 - }, + "position": {"x": 0, "y": 0}, "processGroups": [], "remoteProcessGroups": [], "processors": [], @@ -67,49 +62,54 @@ def to_json(self) -> str: "statelessFlowTimeout": "1 min", "flowFileConcurrency": "UNBOUNDED", "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", - "componentType": "PROCESS_GROUP" - } + "componentType": "PROCESS_GROUP", + }, } processors_by_name = {p.name: p for p in self.processors} processors_node = config["rootGroup"]["processors"] for proc in self.processors: - processors_node.append({ - "identifier": str(proc.id), - "instanceIdentifier": str(proc.id), - "name": proc.name, - "comments": "", - "position": { - "x": 0, - "y": 0 - }, - "type": 'org.apache.nifi.processors.standard.' + proc.class_name, - "bundle": { - "group": "org.apache.nifi", - "artifact": "nifi-standard-nar", - "version": self.NIFI_VERSION - }, - "properties": {key: value for key, value in proc.properties.items() if key}, - "propertyDescriptors": {}, - "style": {}, - "schedulingPeriod": "0 sec" if proc.scheduling_strategy == "EVENT_DRIVEN" else proc.scheduling_period, - "schedulingStrategy": "TIMER_DRIVEN", - "executionNode": "ALL", - "penaltyDuration": "5 sec", - "yieldDuration": "1 sec", - "bulletinLevel": "WARN", - "runDurationMillis": "0", - "concurrentlySchedulableTaskCount": proc.max_concurrent_tasks if proc.max_concurrent_tasks is not None else 1, - "autoTerminatedRelationships": proc.auto_terminated_relationships, - "scheduledState": "RUNNING", - "retryCount": 10, - "retriedRelationships": [], - "backoffMechanism": "PENALIZE_FLOWFILE", - "maxBackoffPeriod": "10 mins", - "componentType": "PROCESSOR", - "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8" - }) + processors_node.append( + { + "identifier": str(proc.id), + "instanceIdentifier": str(proc.id), + "name": proc.name, + "comments": "", + "position": {"x": 0, "y": 0}, + "type": "org.apache.nifi.processors.standard." + proc.class_name, + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": self.NIFI_VERSION, + }, + "properties": { + key: value for key, value in proc.properties.items() if key + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec" + if proc.scheduling_strategy == "EVENT_DRIVEN" + else proc.scheduling_period, + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "5 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": "0", + "concurrentlySchedulableTaskCount": proc.max_concurrent_tasks + if proc.max_concurrent_tasks is not None + else 1, + "autoTerminatedRelationships": proc.auto_terminated_relationships, + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8", + } + ) connections_node = config["rootGroup"]["connections"] @@ -131,44 +131,51 @@ def to_json(self) -> str: dest_type = "OUTPUT_PORT" if not source_proc or not dest_proc: raise ValueError( - f"Could not find processors for connection from '{conn.source_name}' to '{conn.target_name}'") + f"Could not find processors for connection from '{conn.source_name}' to '{conn.target_name}'" + ) - connections_node.append({ - "identifier": conn.id, - "instanceIdentifier": conn.id, - "name": f"{conn.source_name}/{conn.source_relationship}/{conn.target_name}", - "source": { - "id": source_proc.id, - "type": source_type, - "groupId": "9802c873-3322-3b60-a71d-732d02bd60f8", - "name": conn.source_name, - "comments": "", - "instanceIdentifier": source_proc.id - }, - "destination": { - "id": dest_proc.id, - "type": dest_type, - "groupId": "9802c873-3322-3b60-a71d-732d02bd60f8", - "name": dest_proc.name, - "comments": "", - "instanceIdentifier": dest_proc.id - }, - "labelIndex": 1, - "zIndex": 0, - "selectedRelationships": [conn.source_relationship], - "backPressureObjectThreshold": 10, - "backPressureDataSizeThreshold": "50 B", - "flowFileExpiration": "0 sec", - "prioritizers": [], - "bends": [], - "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", - "partitioningAttribute": "", - "loadBalanceCompression": "DO_NOT_COMPRESS", - "componentType": "CONNECTION", - "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8" - }) + connections_node.append( + { + "identifier": conn.id, + "instanceIdentifier": conn.id, + "name": f"{conn.source_name}/{conn.source_relationship}/{conn.target_name}", + "source": { + "id": source_proc.id, + "type": source_type, + "groupId": "9802c873-3322-3b60-a71d-732d02bd60f8", + "name": conn.source_name, + "comments": "", + "instanceIdentifier": source_proc.id, + }, + "destination": { + "id": dest_proc.id, + "type": dest_type, + "groupId": "9802c873-3322-3b60-a71d-732d02bd60f8", + "name": dest_proc.name, + "comments": "", + "instanceIdentifier": dest_proc.id, + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [conn.source_relationship], + "backPressureObjectThreshold": 10, + "backPressureDataSizeThreshold": "50 B", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "partitioningAttribute": "", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8", + } + ) - config["rootGroup"]["inputPorts"] = [input_port.to_json_dict() for input_port in self.input_ports] - config["rootGroup"]["outputPorts"] = [output_port.to_json_dict() for output_port in self.output_ports] + config["rootGroup"]["inputPorts"] = [ + input_port.to_json_dict() for input_port in self.input_ports + ] + config["rootGroup"]["outputPorts"] = [ + output_port.to_json_dict() for output_port in self.output_ports + ] return json.dumps(config) diff --git a/behave_framework/src/minifi_behave/minifi/output_port.py b/behave_framework/src/minifi_behave/minifi/output_port.py index ca2ce57272..e3aa174a07 100644 --- a/behave_framework/src/minifi_behave/minifi/output_port.py +++ b/behave_framework/src/minifi_behave/minifi/output_port.py @@ -15,6 +15,7 @@ # limitations under the License. # + class OutputPort: def __init__(self, name: str, id: str): self.id: str = id @@ -26,16 +27,13 @@ def to_json_dict(self): "instanceIdentifier": self.id, "name": self.name, "comments": "", - "position": { - "x": 0, - "y": 0 - }, + "position": {"x": 0, "y": 0}, "type": "OUTPUT_PORT", "concurrentlySchedulableTaskCount": 1, "scheduledState": "RUNNING", "allowRemoteAccess": True, "portFunction": "STANDARD", "componentType": "OUTPUT_PORT", - "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8" + "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8", } return data diff --git a/behave_framework/src/minifi_behave/minifi/parameter.py b/behave_framework/src/minifi_behave/minifi/parameter.py index 5f681430a7..e55b30372f 100644 --- a/behave_framework/src/minifi_behave/minifi/parameter.py +++ b/behave_framework/src/minifi_behave/minifi/parameter.py @@ -15,6 +15,7 @@ # limitations under the License. # + class Parameter: def __init__(self, name: str, value: str, is_sensitive: bool): self.name: str = name @@ -23,6 +24,10 @@ def __init__(self, name: str, value: str, is_sensitive: bool): self.is_sensitive: bool = is_sensitive def to_yaml_dict(self): - data = {'name': self.name, 'value': self.value, 'description': self.description, - 'sensitive': self.is_sensitive, } + data = { + "name": self.name, + "value": self.value, + "description": self.description, + "sensitive": self.is_sensitive, + } return data diff --git a/behave_framework/src/minifi_behave/minifi/parameter_context.py b/behave_framework/src/minifi_behave/minifi/parameter_context.py index 58cc9161eb..55c6bad9ba 100644 --- a/behave_framework/src/minifi_behave/minifi/parameter_context.py +++ b/behave_framework/src/minifi_behave/minifi/parameter_context.py @@ -28,7 +28,7 @@ def __init__(self, name: str): def to_yaml_dict(self) -> dict: return { - 'id': self.id, - 'name': self.name, - 'Parameters': [p.to_yaml_dict() for p in self.parameters], + "id": self.id, + "name": self.name, + "Parameters": [p.to_yaml_dict() for p in self.parameters], } diff --git a/behave_framework/src/minifi_behave/minifi/processor.py b/behave_framework/src/minifi_behave/minifi/processor.py index 193b8deaf4..3361ab92f7 100644 --- a/behave_framework/src/minifi_behave/minifi/processor.py +++ b/behave_framework/src/minifi_behave/minifi/processor.py @@ -19,8 +19,14 @@ class Processor: - def __init__(self, class_name: str, proc_name: str, scheduling_strategy: str = "TIMER_DRIVEN", - scheduling_period: str = "1 sec", penalization_period: str = "1 sec"): + def __init__( + self, + class_name: str, + proc_name: str, + scheduling_strategy: str = "TIMER_DRIVEN", + scheduling_period: str = "1 sec", + penalization_period: str = "1 sec", + ): self.class_name = class_name self.id: str = str(uuid.uuid4()) self.name = proc_name @@ -46,18 +52,20 @@ def __repr__(self): def to_yaml_dict(self) -> dict: data = { - 'name': self.name, - 'id': self.id, - 'class': self.class_name, - 'scheduling strategy': self.scheduling_strategy, - 'scheduling period': self.scheduling_period, - 'penalization period': self.penalization_period, + "name": self.name, + "id": self.id, + "class": self.class_name, + "scheduling strategy": self.scheduling_strategy, + "scheduling period": self.scheduling_period, + "penalization period": self.penalization_period, } if self.auto_terminated_relationships: - data['auto-terminated relationships list'] = self.auto_terminated_relationships + data["auto-terminated relationships list"] = ( + self.auto_terminated_relationships + ) if self.max_concurrent_tasks is not None: - data['max concurrent tasks'] = self.max_concurrent_tasks + data["max concurrent tasks"] = self.max_concurrent_tasks # The YAML format capitalizes 'Properties' - data['Properties'] = self.properties + data["Properties"] = self.properties return data diff --git a/behave_framework/src/minifi_behave/minifi/remote_port.py b/behave_framework/src/minifi_behave/minifi/remote_port.py index 5e0d7ef677..7f456a6ba7 100644 --- a/behave_framework/src/minifi_behave/minifi/remote_port.py +++ b/behave_framework/src/minifi_behave/minifi/remote_port.py @@ -28,6 +28,11 @@ def add_property(self, property_name: str, property_value: str): self.properties[property_name] = property_value def to_yaml_dict(self): - data = {'id': self.id, 'name': self.name, 'use compression': self.use_compression, 'max concurrent tasks': 1, - 'Properties': self.properties} + data = { + "id": self.id, + "name": self.name, + "use compression": self.use_compression, + "max concurrent tasks": 1, + "Properties": self.properties, + } return data diff --git a/behave_framework/src/minifi_behave/minifi/remote_process_group.py b/behave_framework/src/minifi_behave/minifi/remote_process_group.py index ae8b697535..0298264bfc 100644 --- a/behave_framework/src/minifi_behave/minifi/remote_process_group.py +++ b/behave_framework/src/minifi_behave/minifi/remote_process_group.py @@ -15,6 +15,7 @@ # limitations under the License. # import uuid + from .remote_port import RemotePort @@ -38,7 +39,14 @@ def get_input_port(self, port_name: str) -> RemotePort | None: return next((port for port in self.input_ports if port.name == port_name), None) def to_yaml_dict(self): - data = {'id': self.id, 'name': self.name, 'timeout': '30 sec', 'transport protocol': self.protocol, - 'url': self.address, 'yield period': '3 sec', 'Input Ports': [port.to_yaml_dict() for port in self.input_ports], - 'Output Ports': [port.to_yaml_dict() for port in self.output_ports]} + data = { + "id": self.id, + "name": self.name, + "timeout": "30 sec", + "transport protocol": self.protocol, + "url": self.address, + "yield period": "3 sec", + "Input Ports": [port.to_yaml_dict() for port in self.input_ports], + "Output Ports": [port.to_yaml_dict() for port in self.output_ports], + } return data diff --git a/behave_framework/src/minifi_behave/steps/checking_steps.py b/behave_framework/src/minifi_behave/steps/checking_steps.py index 418e875c93..344eb199a6 100644 --- a/behave_framework/src/minifi_behave/steps/checking_steps.py +++ b/behave_framework/src/minifi_behave/steps/checking_steps.py @@ -14,267 +14,560 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import time import re +import time import humanfriendly -from behave import then, step - +from behave import step, then from minifi_behave.containers.http_proxy_container import HttpProxy -from minifi_behave.core.helpers import wait_for_condition, check_condition_after_wait, log_due_to_failure -from minifi_behave.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext - - -@then('a file with the content "{content}" is placed on the path "{path}" in less than {duration}') -def verify_file_content_on_path(context: MinifiTestContext, content: str, path: str, duration: str): +from minifi_behave.core.helpers import ( + check_condition_after_wait, + log_due_to_failure, + wait_for_condition, +) +from minifi_behave.core.minifi_test_context import ( + DEFAULT_MINIFI_CONTAINER_NAME, + MinifiTestContext, +) + + +@then( + 'a file with the content "{content}" is placed on the path "{path}" in less than {duration}' +) +def verify_file_content_on_path( + context: MinifiTestContext, content: str, path: str, duration: str +): new_content = content.replace("\\n", "\n") timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].path_with_content_exists(path, new_content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, context=context) - - -@then('in the "{container_name}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') -def verify_single_file_content_in_container_directory(context: MinifiTestContext, container_name: str, content: str, directory: str, duration: str): + condition=lambda: context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].path_with_content_exists(path, new_content), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'in the "{container_name}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}' +) +def verify_single_file_content_in_container_directory( + context: MinifiTestContext, + container_name: str, + content: str, + directory: str, + duration: str, +): new_content = content.replace("\\n", "\n") timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[container_name].directory_has_single_file_with_content(directory, new_content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[container_name].exited, context=context) - - -@then('a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') -@then("a single file with the content '{content}' is placed in the '{directory}' directory in less than {duration}") -def verify_single_file_content_in_directory(context: MinifiTestContext, content: str, directory: str, duration: str): - context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') - - -@then('in the "{container_name}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') -def verify_at_least_one_file_content_in_container_directory(context: MinifiTestContext, container_name: str, content: str, directory: str, duration: str): + condition=lambda: context.containers[ + container_name + ].directory_has_single_file_with_content(directory, new_content), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[container_name].exited, + context=context, + ) + + +@then( + 'a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}' +) +@then( + "a single file with the content '{content}' is placed in the '{directory}' directory in less than {duration}" +) +def verify_single_file_content_in_directory( + context: MinifiTestContext, content: str, directory: str, duration: str +): + context.execute_steps( + f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}' + ) + + +@then( + 'in the "{container_name}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}' +) +def verify_at_least_one_file_content_in_container_directory( + context: MinifiTestContext, + container_name: str, + content: str, + directory: str, + duration: str, +): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[container_name].directory_contains_file_with_content(directory, content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[container_name].exited, context=context) - - -@then('at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') -@then("at least one file with the content '{content}' is placed in the '{directory}' directory in less than {duration}") -def verify_at_least_one_file_content_in_directory(context: MinifiTestContext, content: str, directory: str, duration: str): - context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') - - -@then("the logs of the '{container}' container do not contain the following message: '{message}' after {duration}") -@then('the logs of the "{container}" container do not contain the following message: "{message}" after {duration}') -def verify_container_logs_do_not_contain_message(context: MinifiTestContext, container: str, message: str, duration: str): + condition=lambda: context.containers[ + container_name + ].directory_contains_file_with_content(directory, content), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[container_name].exited, + context=context, + ) + + +@then( + 'at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}' +) +@then( + "at least one file with the content '{content}' is placed in the '{directory}' directory in less than {duration}" +) +def verify_at_least_one_file_content_in_directory( + context: MinifiTestContext, content: str, directory: str, duration: str +): + context.execute_steps( + f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}' + ) + + +@then( + "the logs of the '{container}' container do not contain the following message: '{message}' after {duration}" +) +@then( + 'the logs of the "{container}" container do not contain the following message: "{message}" after {duration}' +) +def verify_container_logs_do_not_contain_message( + context: MinifiTestContext, container: str, message: str, duration: str +): duration_seconds = humanfriendly.parse_timespan(duration) time.sleep(duration_seconds) - assert message not in context.containers[container].get_logs() or log_due_to_failure(context) + assert message not in context.containers[ + container + ].get_logs() or log_due_to_failure(context) -@then('the Minifi logs do not contain the following message: "{message}" after {duration}') -def verify_minifi_logs_do_not_contain_message(context: MinifiTestContext, message: str, duration: str): - context.execute_steps(f'then the logs of the "{DEFAULT_MINIFI_CONTAINER_NAME}" container do not contain the following message: "{message}" after {duration}') +@then( + 'the Minifi logs do not contain the following message: "{message}" after {duration}' +) +def verify_minifi_logs_do_not_contain_message( + context: MinifiTestContext, message: str, duration: str +): + context.execute_steps( + f'then the logs of the "{DEFAULT_MINIFI_CONTAINER_NAME}" container do not contain the following message: "{message}" after {duration}' + ) @then("the Minifi logs do not contain errors") def verify_minifi_logs_do_not_contain_errors(context: MinifiTestContext): - assert "[error]" not in context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs() or log_due_to_failure(context) + assert "[error]" not in context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].get_logs() or log_due_to_failure(context) @then("the Minifi logs do not contain warnings") def verify_minifi_logs_do_not_contain_warnings(context: MinifiTestContext): - assert "[warning]" not in context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs() or log_due_to_failure(context) - - -@then("the logs of the '{container}' container contain the following message: '{message}' in less than {duration}") -@then('the logs of the "{container}" container contain the following message: "{message}" in less than {duration}') -def verify_container_logs_contain_message(context: MinifiTestContext, container: str, message: str, duration: str): + assert "[warning]" not in context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].get_logs() or log_due_to_failure(context) + + +@then( + "the logs of the '{container}' container contain the following message: '{message}' in less than {duration}" +) +@then( + 'the logs of the "{container}" container contain the following message: "{message}" in less than {duration}' +) +def verify_container_logs_contain_message( + context: MinifiTestContext, container: str, message: str, duration: str +): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: message in context.containers[container].get_logs(), - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers[container].exited, - context=context) - - -@then("the Minifi logs contain the following message: '{message}' in less than {duration}") -@then('the Minifi logs contain the following message: "{message}" in less than {duration}') -def verify_minifi_logs_contain_message(context: MinifiTestContext, message: str, duration: str): - context.execute_steps(f'then the logs of the "{DEFAULT_MINIFI_CONTAINER_NAME}" container contain the following message: "{message}" in less than {duration}') + assert wait_for_condition( + condition=lambda: message in context.containers[container].get_logs(), + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers[container].exited, + context=context, + ) + + +@then( + "the Minifi logs contain the following message: '{message}' in less than {duration}" +) +@then( + 'the Minifi logs contain the following message: "{message}" in less than {duration}' +) +def verify_minifi_logs_contain_message( + context: MinifiTestContext, message: str, duration: str +): + context.execute_steps( + f'then the logs of the "{DEFAULT_MINIFI_CONTAINER_NAME}" container contain the following message: "{message}" in less than {duration}' + ) -@then("the Minifi logs contain the following message: \"{log_message}\" {count:d} times after {duration}") -def verify_minifi_logs_contain_message_multiple_times(context, log_message, count, duration): +@then( + 'the Minifi logs contain the following message: "{log_message}" {count:d} times after {duration}' +) +def verify_minifi_logs_contain_message_multiple_times( + context, log_message, count, duration +): duration_seconds = humanfriendly.parse_timespan(duration) time.sleep(duration_seconds) - assert context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs().count(log_message) == count or log_due_to_failure(context) + assert context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs().count( + log_message + ) == count or log_due_to_failure(context) -@then("the Minifi logs match the following regex: \"{regex}\" in less than {duration}") +@then('the Minifi logs match the following regex: "{regex}" in less than {duration}') def verify_minifi_logs_match_regex(context, regex, duration): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: re.search(regex, context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs()), - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) + assert wait_for_condition( + condition=lambda: re.search( + regex, context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs() + ), + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) @step('no errors were generated on the http-proxy regarding "{url}"') def verify_no_errors_on_http_proxy(context: MinifiTestContext, url: str): - http_proxy_container = next(container for container in context.containers.values() if isinstance(container, HttpProxy)) - assert http_proxy_container.check_http_proxy_access(url) or log_due_to_failure(context) - - -@then('in the "{container}" container no files are placed in the "{directory}" directory in {duration} of running time') + http_proxy_container = next( + container + for container in context.containers.values() + if isinstance(container, HttpProxy) + ) + assert http_proxy_container.check_http_proxy_access(url) or log_due_to_failure( + context + ) + + +@then( + 'in the "{container}" container no files are placed in the "{directory}" directory in {duration} of running time' +) def verify_no_files_in_container_directory(context, container, directory, duration): duration_seconds = humanfriendly.parse_timespan(duration) - assert check_condition_after_wait(condition=lambda: context.containers[container].get_number_of_files(directory) == 0, - context=context, wait_time=duration_seconds) - - -@then('no files are placed in the "{directory}" directory in {duration} of running time') + assert check_condition_after_wait( + condition=lambda: ( + context.containers[container].get_number_of_files(directory) == 0 + ), + context=context, + wait_time=duration_seconds, + ) + + +@then( + 'no files are placed in the "{directory}" directory in {duration} of running time' +) def verify_no_files_in_directory(context, directory, duration): - context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container no files are placed in the "{directory}" directory in {duration} of running time') + context.execute_steps( + f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container no files are placed in the "{directory}" directory in {duration} of running time' + ) @then('{num:d} files are placed in the "{directory}" directory in less than {duration}') @then('{num:d} file is placed in the "{directory}" directory in less than {duration}') -def verify_number_of_files_in_directory(context: MinifiTestContext, num: int, directory: str, duration: str): +def verify_number_of_files_in_directory( + context: MinifiTestContext, num: int, directory: str, duration: str +): duration_seconds = humanfriendly.parse_timespan(duration) if num == 0: - context.execute_steps(f'then no files are placed in the "{directory}" directory in {duration} of running time') + context.execute_steps( + f'then no files are placed in the "{directory}" directory in {duration} of running time' + ) return - assert wait_for_condition(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) == num, - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) - - -@then('at least {num:d} files are placed in the "{directory}" directory in less than {duration}') -@then('at least {num:d} file is placed in the "{directory}" directory in less than {duration}') -def verify_at_least_number_of_files_in_directory(context: MinifiTestContext, num: int, directory: str, duration: str): + assert wait_for_condition( + condition=lambda: ( + context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files( + directory + ) + == num + ), + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'at least {num:d} files are placed in the "{directory}" directory in less than {duration}' +) +@then( + 'at least {num:d} file is placed in the "{directory}" directory in less than {duration}' +) +def verify_at_least_number_of_files_in_directory( + context: MinifiTestContext, num: int, directory: str, duration: str +): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) >= num, - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) - - -@then('at least one file in "{directory}" content match the following regex: "{regex_str}" in less than {duration}') -@then('the content of at least one file in the "{directory}" directory matches the \'{regex_str}\' regex in less than {duration}') -def verify_file_content_matches_regex_in_directory(context: MinifiTestContext, directory: str, regex_str: str, duration: str): + assert wait_for_condition( + condition=lambda: ( + context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files( + directory + ) + >= num + ), + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'at least one file in "{directory}" content match the following regex: "{regex_str}" in less than {duration}' +) +@then( + "the content of at least one file in the \"{directory}\" directory matches the '{regex_str}' regex in less than {duration}" +) +def verify_file_content_matches_regex_in_directory( + context: MinifiTestContext, directory: str, regex_str: str, duration: str +): duration_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_regex(directory, regex_str), - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, context=context) - - -@then('files with contents "{content_one}" and "{content_two}" are placed in the "{directory}" directory in less than {timeout}') -@then("files with contents '{content_one}' and '{content_two}' are placed in the '{directory}' directory in less than {timeout}") -def verify_files_with_two_contents_in_directory(context: MinifiTestContext, directory: str, timeout: str, content_one: str, content_two: str): + condition=lambda: context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].directory_contains_file_with_regex(directory, regex_str), + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'files with contents "{content_one}" and "{content_two}" are placed in the "{directory}" directory in less than {timeout}' +) +@then( + "files with contents '{content_one}' and '{content_two}' are placed in the '{directory}' directory in less than {timeout}" +) +def verify_files_with_two_contents_in_directory( + context: MinifiTestContext, + directory: str, + timeout: str, + content_one: str, + content_two: str, +): timeout_seconds = humanfriendly.parse_timespan(timeout) c1 = content_one.replace("\\n", "\n") c2 = content_two.replace("\\n", "\n") contents_arr = [c1, c2] - assert wait_for_condition(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_file_contents(directory, contents_arr), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) - - -@then('files with contents "{contents}" are placed in the "{directory}" directory in less than {timeout}') -def verify_files_with_contents_in_directory(context: MinifiTestContext, directory: str, timeout: str, contents: str): + assert wait_for_condition( + condition=lambda: context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].verify_file_contents(directory, contents_arr), + timeout_seconds=timeout_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'files with contents "{contents}" are placed in the "{directory}" directory in less than {timeout}' +) +def verify_files_with_contents_in_directory( + context: MinifiTestContext, directory: str, timeout: str, contents: str +): timeout_seconds = humanfriendly.parse_timespan(timeout) new_contents = contents.replace("\\n", "\n") contents_arr = new_contents.split(",") - assert wait_for_condition(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_file_contents(directory, contents_arr), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) - - -@then('files with at least these contents "{contents}" are placed in the "{directory}" directory in less than {timeout}') -def verify_files_with_at_least_contents_in_directory(context: MinifiTestContext, directory: str, timeout: str, contents: str): + assert wait_for_condition( + condition=lambda: context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].verify_file_contents(directory, contents_arr), + timeout_seconds=timeout_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'files with at least these contents "{contents}" are placed in the "{directory}" directory in less than {timeout}' +) +def verify_files_with_at_least_contents_in_directory( + context: MinifiTestContext, directory: str, timeout: str, contents: str +): timeout_seconds = humanfriendly.parse_timespan(timeout) new_contents = contents.replace("\\n", "\n") contents_arr = new_contents.split(",") - assert wait_for_condition(condition=lambda: all([context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_content(directory, content) for content in contents_arr]), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) - - -@then("a file with the JSON content \"{content}\" is placed in the \"{directory}\" directory in less than {duration}") -@then("a file with the JSON content '{content}' is placed in the '{directory}' directory in less than {duration}") -def verify_file_with_json_content_in_directory(context: MinifiTestContext, content: str, directory: str, duration: str): + assert wait_for_condition( + condition=lambda: all( + context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].directory_contains_file_with_content(directory, content) + for content in contents_arr + ), + timeout_seconds=timeout_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'a file with the JSON content "{content}" is placed in the "{directory}" directory in less than {duration}' +) +@then( + "a file with the JSON content '{content}' is placed in the '{directory}' directory in less than {duration}" +) +def verify_file_with_json_content_in_directory( + context: MinifiTestContext, content: str, directory: str, duration: str +): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_path_with_json_content(directory, content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, context=context) - - -@then('MiNiFi\'s memory usage does not increase by more than {max_increase} after {duration}') -def verify_minifi_memory_usage_increase(context: MinifiTestContext, max_increase: str, duration: str): + condition=lambda: context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].verify_path_with_json_content(directory, content), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + "MiNiFi's memory usage does not increase by more than {max_increase} after {duration}" +) +def verify_minifi_memory_usage_increase( + context: MinifiTestContext, max_increase: str, duration: str +): time_in_seconds = humanfriendly.parse_timespan(duration) max_increase_in_bytes = humanfriendly.parse_size(max_increase) - initial_memory_usage = context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_memory_usage() + initial_memory_usage = context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].get_memory_usage() time.sleep(time_in_seconds) - final_memory_usage = context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_memory_usage() + final_memory_usage = context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].get_memory_usage() assert final_memory_usage - initial_memory_usage <= max_increase_in_bytes -@then("at least one file with the JSON content \"{content}\" is placed in the \"{directory}\" directory in less than {duration}") -@then("at least one file with the JSON content '{content}' is placed in the '{directory}' directory in less than {duration}") -def verify_at_least_one_file_with_json_content_in_directory(context: MinifiTestContext, content: str, directory: str, duration: str): +@then( + 'at least one file with the JSON content "{content}" is placed in the "{directory}" directory in less than {duration}' +) +@then( + "at least one file with the JSON content '{content}' is placed in the '{directory}' directory in less than {duration}" +) +def verify_at_least_one_file_with_json_content_in_directory( + context: MinifiTestContext, content: str, directory: str, duration: str +): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_json_content(directory, content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, context=context) - - -@then('after a wait of {duration}, at least {lower_bound:d} and at most {upper_bound:d} files are produced and placed in the "{directory}" directory') -def verify_file_count_bounds_in_directory(context: MinifiTestContext, lower_bound: int, upper_bound: int, duration: str, directory: str): + condition=lambda: context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].directory_contains_file_with_json_content(directory, content), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'after a wait of {duration}, at least {lower_bound:d} and at most {upper_bound:d} files are produced and placed in the "{directory}" directory' +) +def verify_file_count_bounds_in_directory( + context: MinifiTestContext, + lower_bound: int, + upper_bound: int, + duration: str, + directory: str, +): duration_seconds = humanfriendly.parse_timespan(duration) - assert check_condition_after_wait(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) >= lower_bound - and context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) <= upper_bound, - context=context, wait_time=duration_seconds) - - -@then('exactly these files are in the "{directory}" directory in less than {duration}: "{contents}"') -def verify_exact_files_in_directory(context: MinifiTestContext, directory: str, duration: str, contents: str): + assert check_condition_after_wait( + condition=lambda: ( + context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files( + directory + ) + >= lower_bound + and context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files( + directory + ) + <= upper_bound + ), + context=context, + wait_time=duration_seconds, + ) + + +@then( + 'exactly these files are in the "{directory}" directory in less than {duration}: "{contents}"' +) +def verify_exact_files_in_directory( + context: MinifiTestContext, directory: str, duration: str, contents: str +): if not contents: - context.execute_steps(f'then no files are placed in the "{directory}" directory in {duration} of running time') + context.execute_steps( + f'then no files are placed in the "{directory}" directory in {duration} of running time' + ) return contents_arr = contents.split(",") timeout_in_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_file_contents(directory, contents_arr), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: False, - context=context) - - -@then('exactly these files are in the "{directory}" directory in less than {duration}: ""') -def verify_no_files_in_directory(context, directory, duration): - context.execute_steps(f'then no files are placed in the "{directory}" directory in {duration} of running time') + assert wait_for_condition( + condition=lambda: context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].verify_file_contents(directory, contents_arr), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: False, + context=context, + ) + + +@then( + 'exactly these files are in the "{directory}" directory in less than {duration}: ""' +) +def verify_no_files_in_directory_exact(context, directory, duration): + context.execute_steps( + f'then no files are placed in the "{directory}" directory in {duration} of running time' + ) -@then("at least one empty file is placed in the \"{directory}\" directory in less than {duration}") -def verify_at_least_one_empty_file_in_directory(context: MinifiTestContext, directory: str, duration: str): +@then( + 'at least one empty file is placed in the "{directory}" directory in less than {duration}' +) +def verify_at_least_one_empty_file_in_directory( + context: MinifiTestContext, directory: str, duration: str +): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_empty_file(directory), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, context=context) - - -@then("in the \"{container_name}\" container at least one empty file is placed in the \"{directory}\" directory in less than {duration}") -def verify_at_least_one_empty_file_in_container_directory(context: MinifiTestContext, container_name: str, directory: str, duration: str): + condition=lambda: context.containers[ + DEFAULT_MINIFI_CONTAINER_NAME + ].directory_contains_empty_file(directory), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'in the "{container_name}" container at least one empty file is placed in the "{directory}" directory in less than {duration}' +) +def verify_at_least_one_empty_file_in_container_directory( + context: MinifiTestContext, container_name: str, directory: str, duration: str +): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[container_name].directory_contains_empty_file(directory), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[container_name].exited, context=context) - - -@then("in the \"{container_name}\" container at least one file with minimum size of \"{size}\" is placed in the \"{directory}\" directory in less than {duration}") -def verify_file_with_minimum_size_in_container_directory(context: MinifiTestContext, container_name: str, directory: str, size: str, duration: str): + condition=lambda: context.containers[ + container_name + ].directory_contains_empty_file(directory), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[container_name].exited, + context=context, + ) + + +@then( + 'in the "{container_name}" container at least one file with minimum size of "{size}" is placed in the "{directory}" directory in less than {duration}' +) +def verify_file_with_minimum_size_in_container_directory( + context: MinifiTestContext, + container_name: str, + directory: str, + size: str, + duration: str, +): timeout_in_seconds = humanfriendly.parse_timespan(duration) size_in_bytes = humanfriendly.parse_size(size) assert wait_for_condition( - condition=lambda: context.containers[container_name].directory_contains_file_with_minimum_size(directory, size_in_bytes), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[container_name].exited, context=context) - - -@then("at least one file with minimum size of \"{size}\" is placed in the \"{directory}\" directory in less than {duration}") -def verify_file_with_minimum_size_in_directory(context: MinifiTestContext, directory: str, size: str, duration: str): + condition=lambda: context.containers[ + container_name + ].directory_contains_file_with_minimum_size(directory, size_in_bytes), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[container_name].exited, + context=context, + ) + + +@then( + 'at least one file with minimum size of "{size}" is placed in the "{directory}" directory in less than {duration}' +) +def verify_file_with_minimum_size_in_directory( + context: MinifiTestContext, directory: str, size: str, duration: str +): context.execute_steps( - f'Then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with minimum size of "{size}" is placed in the "{directory}" directory in less than {duration}') + f'Then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with minimum size of "{size}" is placed in the "{directory}" directory in less than {duration}' + ) diff --git a/behave_framework/src/minifi_behave/steps/configuration_steps.py b/behave_framework/src/minifi_behave/steps/configuration_steps.py index d2e8659c3c..99b1f8d64f 100644 --- a/behave_framework/src/minifi_behave/steps/configuration_steps.py +++ b/behave_framework/src/minifi_behave/steps/configuration_steps.py @@ -15,18 +15,23 @@ # limitations under the License. # -from behave import step, given - +from behave import given, step +from minifi_behave.containers.minifi_protocol import ( + conf_c2_flow_url, + enable_log_metrics_publisher, + enable_openssl_fips_mode, + set_up_ssl_properties, +) from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.containers.minifi_protocol import enable_openssl_fips_mode -from minifi_behave.containers.minifi_protocol import set_up_ssl_properties -from minifi_behave.containers.minifi_protocol import enable_log_metrics_publisher -from minifi_behave.containers.minifi_protocol import conf_c2_flow_url @step('MiNiFi configuration "{config_key}" is set to "{config_value}"') -def set_minifi_config_property(context: MinifiTestContext, config_key: str, config_value: str): - context.get_or_create_default_minifi_container().set_property(config_key, config_value) +def set_minifi_config_property( + context: MinifiTestContext, config_key: str, config_value: str +): + context.get_or_create_default_minifi_container().set_property( + config_key, config_value + ) @step("log metrics publisher is enabled in MiNiFi") @@ -35,8 +40,12 @@ def enable_minifi_log_metrics_publisher(context: MinifiTestContext): @step('log property "{log_property_key}" is set to "{log_property_value}"') -def set_minifi_log_property(context: MinifiTestContext, log_property_key: str, log_property_value: str): - context.get_or_create_default_minifi_container().set_log_property(log_property_key, log_property_value) +def set_minifi_log_property( + context: MinifiTestContext, log_property_key: str, log_property_value: str +): + context.get_or_create_default_minifi_container().set_log_property( + log_property_key, log_property_value + ) @given("OpenSSL FIPS mode is enabled in MiNiFi") @@ -46,7 +55,9 @@ def enable_minifi_openssl_fips_mode(context: MinifiTestContext): @given("the C2 flow URL property is configured") def configure_c2_flow_url(context: MinifiTestContext): - conf_c2_flow_url(context.get_or_create_default_minifi_container(), context.scenario_id) + conf_c2_flow_url( + context.get_or_create_default_minifi_container(), context.scenario_id + ) @given("SSL properties are set in MiNiFi") diff --git a/behave_framework/src/minifi_behave/steps/core_steps.py b/behave_framework/src/minifi_behave/steps/core_steps.py index 2c569637bb..c68349c694 100644 --- a/behave_framework/src/minifi_behave/steps/core_steps.py +++ b/behave_framework/src/minifi_behave/steps/core_steps.py @@ -16,20 +16,24 @@ # import logging +import os import random import string -import os import time import uuid import humanfriendly -from behave import when, step, given - -from minifi_behave.containers.http_proxy_container import HttpProxy -from minifi_behave.containers.nifi_container import NifiContainer +from behave import given, step, when from minifi_behave.containers.directory import Directory from minifi_behave.containers.file import File -from minifi_behave.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext +from minifi_behave.containers.http_proxy_container import HttpProxy +from minifi_behave.containers.nifi_container import NifiContainer +from minifi_behave.core.minifi_test_context import ( + DEFAULT_MINIFI_CONTAINER_NAME, + MinifiTestContext, +) + +logger = logging.getLogger(__name__) @when("both instances start up") @@ -37,19 +41,23 @@ def start_all_instances(context: MinifiTestContext): for container in context.containers.values(): assert container.deploy(context) - logging.debug("All instances started up") + logger.debug("All instances started up") @when("the MiNiFi instance starts up") def start_minifi_instance(context: MinifiTestContext): assert context.get_or_create_default_minifi_container().deploy(context) - logging.debug("MiNiFi instance started up") + logger.debug("MiNiFi instance started up") @step('a directory at "{directory}" has a file with the size "{size}"') -def create_file_with_size_in_directory(context: MinifiTestContext, directory: str, size: str): +def create_file_with_size_in_directory( + context: MinifiTestContext, directory: str, size: str +): size = humanfriendly.parse_size(size) - content = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) + content = "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(size) + ) dirs = context.get_or_create_default_minifi_container().dirs if directory in dirs: dirs[directory].files[str(uuid.uuid4())] = content @@ -59,7 +67,13 @@ def create_file_with_size_in_directory(context: MinifiTestContext, directory: st dirs.append(new_dir) -def __add_directory_with_file_to_container(context: MinifiTestContext, directory: str, file_name: str, content: str | bytes, container_name: str): +def __add_directory_with_file_to_container( + context: MinifiTestContext, + directory: str, + file_name: str, + content: str | bytes, + container_name: str, +): dirs = context.get_or_create_minifi_container(container_name).dirs if isinstance(content, str): content = content.replace("\\n", "\n") @@ -71,76 +85,138 @@ def __add_directory_with_file_to_container(context: MinifiTestContext, directory dirs.append(new_dir) -@step('a directory at "{directory}" has a file with the content "{content}" in the "{flow_name}" flow') -@step("a directory at '{directory}' has a file with the content '{content}' in the '{flow_name}' flow") -def create_file_with_content_in_directory_for_flow(context: MinifiTestContext, directory: str, content: str, flow_name: str): - __add_directory_with_file_to_container(context, directory, str(uuid.uuid4()), content, flow_name) +@step( + 'a directory at "{directory}" has a file with the content "{content}" in the "{flow_name}" flow' +) +@step( + "a directory at '{directory}' has a file with the content '{content}' in the '{flow_name}' flow" +) +def create_file_with_content_in_directory_for_flow( + context: MinifiTestContext, directory: str, content: str, flow_name: str +): + __add_directory_with_file_to_container( + context, directory, str(uuid.uuid4()), content, flow_name + ) @step('a directory at "{directory}" has a file with the content "{content}"') @step("a directory at '{directory}' has a file with the content '{content}'") -def create_file_with_content_in_directory(context: MinifiTestContext, directory: str, content: str): - context.execute_steps(f'given a directory at "{directory}" has a file with the content "{content}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') +def create_file_with_content_in_directory( + context: MinifiTestContext, directory: str, content: str +): + context.execute_steps( + f'given a directory at "{directory}" has a file with the content "{content}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @step('a directory at "{directory}" has a file with the content from "{path}"') @step("a directory at '{directory}' has a file with the content from '{path}'") -def create_file_with_content_from_path_in_directory(context: MinifiTestContext, directory: str, path: str): - assert context.resource_dir is not None, "Cannot copy file if resource_dir is not set for the context" +def create_file_with_content_from_path_in_directory( + context: MinifiTestContext, directory: str, path: str +): + assert context.resource_dir is not None, ( + "Cannot copy file if resource_dir is not set for the context" + ) content = None with open(context.resource_dir / path, "rb") as f: content = f.read() - __add_directory_with_file_to_container(context, directory, str(uuid.uuid4()), content, DEFAULT_MINIFI_CONTAINER_NAME) - - -@step('a directory at "{directory}" has a file "{file_name}" with the content "{content}"') -def create_file_with_name_and_content_in_directory(context: MinifiTestContext, directory: str, file_name: str, content: str): - __add_directory_with_file_to_container(context, directory, file_name, content, DEFAULT_MINIFI_CONTAINER_NAME) - - -@step('a file with filename "{file_name}" and content "{content}" is present in "{path}"') -def file_with_name_and_content_present_in_path(context: MinifiTestContext, file_name: str, content: str, path: str): + __add_directory_with_file_to_container( + context, directory, str(uuid.uuid4()), content, DEFAULT_MINIFI_CONTAINER_NAME + ) + + +@step( + 'a directory at "{directory}" has a file "{file_name}" with the content "{content}"' +) +def create_file_with_name_and_content_in_directory( + context: MinifiTestContext, directory: str, file_name: str, content: str +): + __add_directory_with_file_to_container( + context, directory, file_name, content, DEFAULT_MINIFI_CONTAINER_NAME + ) + + +@step( + 'a file with filename "{file_name}" and content "{content}" is present in "{path}"' +) +def file_with_name_and_content_present_in_path( + context: MinifiTestContext, file_name: str, content: str, path: str +): new_content = content.replace("\\n", "\n") - context.get_or_create_default_minifi_container().files.append(File(os.path.join(path, file_name), new_content)) + context.get_or_create_default_minifi_container().files.append( + File(os.path.join(path, file_name), new_content) + ) -@given('a file with the content "{content}" is present in "{path}" in the "{container_name}" flow') -def file_with_content_present_in_path_for_flow(context: MinifiTestContext, content: str, path: str, container_name: str): +@given( + 'a file with the content "{content}" is present in "{path}" in the "{container_name}" flow' +) +def file_with_content_present_in_path_for_flow( + context: MinifiTestContext, content: str, path: str, container_name: str +): new_content = content.replace("\\n", "\n") - context.get_or_create_minifi_container(container_name).files.append(File(os.path.join(path, str(uuid.uuid4())), new_content)) + context.get_or_create_minifi_container(container_name).files.append( + File(os.path.join(path, str(uuid.uuid4())), new_content) + ) @given('a file with the content "{content}" is present in "{path}"') @given("a file with the content '{content}' is present in '{path}'") -def file_with_content_present_in_path(context: MinifiTestContext, content: str, path: str): - context.execute_steps(f"given a file with the content \"{content}\" is present in \"{path}\" in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow") - - -@when('a file with the content "{content}" is placed in "{path}" in the "{container_name}" flow') -def place_file_with_content_in_path_for_flow(context: MinifiTestContext, content: str, path: str, container_name: str): +def file_with_content_present_in_path( + context: MinifiTestContext, content: str, path: str +): + context.execute_steps( + f'given a file with the content "{content}" is present in "{path}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) + + +@when( + 'a file with the content "{content}" is placed in "{path}" in the "{container_name}" flow' +) +def place_file_with_content_in_path_for_flow( + context: MinifiTestContext, content: str, path: str, container_name: str +): new_content = content.replace("\\n", "\n") context.containers[container_name].add_file_to_running_container(new_content, path) @when('a file with the content "{content}" is placed in "{path}"') -def place_file_with_content_in_path(context: MinifiTestContext, content: str, path: str): - context.execute_steps(f"when a file with the content \"{content}\" is placed in \"{path}\" in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow") +def place_file_with_content_in_path( + context: MinifiTestContext, content: str, path: str +): + context.execute_steps( + f'when a file with the content "{content}" is placed in "{path}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) -@given("an empty file is present in \"{path}\"") +@given('an empty file is present in "{path}"') def empty_file_present_in_path(context, path): - context.get_or_create_default_minifi_container().files.append(File(os.path.join(path, str(uuid.uuid4())), "")) + context.get_or_create_default_minifi_container().files.append( + File(os.path.join(path, str(uuid.uuid4())), "") + ) -@given('a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container "{container_name}"') -def bind_host_resource_file_to_container_path_for_container(context: MinifiTestContext, filename: str, container_path: str, container_name: str): +@given( + 'a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container "{container_name}"' +) +def bind_host_resource_file_to_container_path_for_container( + context: MinifiTestContext, filename: str, container_path: str, container_name: str +): path = os.path.join(context.resource_dir, filename) - context.get_or_create_minifi_container(container_name).add_host_file(path, container_path) + context.get_or_create_minifi_container(container_name).add_host_file( + path, container_path + ) -@given('a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container') -def bind_host_resource_file_to_container_path(context: MinifiTestContext, filename: str, container_path: str): - context.execute_steps(f"given a host resource file \"{filename}\" is bound to the \"{container_path}\" path in the MiNiFi container \"{DEFAULT_MINIFI_CONTAINER_NAME}\"") +@given( + 'a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container' +) +def bind_host_resource_file_to_container_path( + context: MinifiTestContext, filename: str, container_path: str +): + context.execute_steps( + f'given a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container "{DEFAULT_MINIFI_CONTAINER_NAME}"' + ) @step("after {duration} have passed") @@ -155,7 +231,7 @@ def stop_minifi(context: MinifiTestContext): context.get_or_create_default_minifi_container().stop() -@when("the \"{container_name}\" flow is stopped") +@when('the "{container_name}" flow is stopped') def stop_flow(context: MinifiTestContext, container_name: str): context.get_or_create_minifi_container(container_name).stop() @@ -165,17 +241,17 @@ def restart_minifi(context: MinifiTestContext): context.get_or_create_default_minifi_container().restart() -@when("the \"{container_name}\" flow is restarted") +@when('the "{container_name}" flow is restarted') def restart_flow(context: MinifiTestContext, container_name: str): context.get_or_create_minifi_container(container_name).restart() -@when("the \"{container_name}\" flow is started") +@when('the "{container_name}" flow is started') def start_flow(context: MinifiTestContext, container_name: str): context.get_or_create_minifi_container(container_name).start() -@when("the \"{container_name}\" flow is killed") +@when('the "{container_name}" flow is killed') def kill_flow(context: MinifiTestContext, container_name: str): context.get_or_create_minifi_container(container_name).kill() @@ -195,11 +271,16 @@ def setup_nifi_container_with_ssl(context: MinifiTestContext): context.containers["nifi"] = NifiContainer(context, use_ssl=True) -@when('NiFi is started') +@when("NiFi is started") def start_nifi(context): - assert context.containers["nifi"].deploy(context) or context.containers["nifi"].log_app_output() + assert ( + context.containers["nifi"].deploy(context) + or context.containers["nifi"].log_app_output() + ) @step("the MiNiFi deployment timeout is set to {timeout_seconds:d} seconds") def set_minifi_deployment_timeout(context: MinifiTestContext, timeout_seconds: int): - context.get_or_create_default_minifi_container().set_deploy_timeout_seconds(timeout_seconds) + context.get_or_create_default_minifi_container().set_deploy_timeout_seconds( + timeout_seconds + ) diff --git a/behave_framework/src/minifi_behave/steps/flow_building_steps.py b/behave_framework/src/minifi_behave/steps/flow_building_steps.py index b880bbfbc0..1e6338ddc8 100644 --- a/behave_framework/src/minifi_behave/steps/flow_building_steps.py +++ b/behave_framework/src/minifi_behave/steps/flow_building_steps.py @@ -17,9 +17,12 @@ import logging import uuid -from behave import given, step -from minifi_behave.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext +from behave import given, step +from minifi_behave.core.minifi_test_context import ( + DEFAULT_MINIFI_CONTAINER_NAME, + MinifiTestContext, +) from minifi_behave.minifi.connection import Connection from minifi_behave.minifi.controller_service import ControllerService from minifi_behave.minifi.funnel import Funnel @@ -27,6 +30,8 @@ from minifi_behave.minifi.parameter_context import ParameterContext from minifi_behave.minifi.processor import Processor +logger = logging.getLogger(__name__) + @given("a MiNiFi CPP server with yaml config") def minifi_cpp_server_with_yaml_config(context: MinifiTestContext): @@ -35,56 +40,110 @@ def minifi_cpp_server_with_yaml_config(context: MinifiTestContext): @given("a transient MiNiFi flow with a LogOnDestructionProcessor processor") def transient_flow_with_logondestructionprocessor(context: MinifiTestContext): - context.get_or_create_default_minifi_container().command = ["/bin/sh", "-c", "timeout 10s ./bin/minifi.sh run && sleep 100"] + context.get_or_create_default_minifi_container().command = [ + "/bin/sh", + "-c", + "timeout 10s ./bin/minifi.sh run && sleep 100", + ] context.get_or_create_default_minifi_container().flow_definition.add_processor( - Processor("LogOnDestructionProcessor", "LogOnDestructionProcessor")) + Processor("LogOnDestructionProcessor", "LogOnDestructionProcessor") + ) @given( - 'a {processor_type} processor with the name "{processor_name}" and the "{property_name}" property set to "{property_value}"') -def processor_with_name_and_property(context: MinifiTestContext, processor_type: str, processor_name: str, property_name: str, - property_value: str): + 'a {processor_type} processor with the name "{processor_name}" and the "{property_name}" property set to "{property_value}"' +) +def processor_with_name_and_property( + context: MinifiTestContext, + processor_type: str, + processor_name: str, + property_name: str, + property_value: str, +): processor = Processor(processor_type, processor_name) processor.add_property(property_name, property_value) - context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) + context.get_or_create_default_minifi_container().flow_definition.add_processor( + processor + ) -@step('a {processor_type} processor with the "{property_name}" property set to "{property_value}"') -def processor_with_property(context: MinifiTestContext, processor_type: str, property_name: str, property_value: str): +@step( + 'a {processor_type} processor with the "{property_name}" property set to "{property_value}"' +) +def processor_with_property( + context: MinifiTestContext, + processor_type: str, + property_name: str, + property_value: str, +): context.execute_steps( - f'Given a {processor_type} processor with the name "{processor_type}" and the "{property_name}" property set to "{property_value}"') + f'Given a {processor_type} processor with the name "{processor_type}" and the "{property_name}" property set to "{property_value}"' + ) -@step('a {processor_type} processor with the "{property_name}" property set to "{property_value}" in the "{minifi_container_name}" flow') -def processor_with_property_in_minifi_flow(context: MinifiTestContext, processor_type: str, property_name: str, property_value: str, minifi_container_name: str): +@step( + 'a {processor_type} processor with the "{property_name}" property set to "{property_value}" in the "{minifi_container_name}" flow' +) +def processor_with_property_in_minifi_flow( + context: MinifiTestContext, + processor_type: str, + property_name: str, + property_value: str, + minifi_container_name: str, +): processor = Processor(processor_type, processor_type) processor.add_property(property_name, property_value) - context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor) + context.get_or_create_minifi_container( + minifi_container_name + ).flow_definition.add_processor(processor) -@step('a {processor_type} processor with the "{property_name}" property set to "{property_value}" in the NiFi flow') -def processor_with_property_in_nifi_flow(context: MinifiTestContext, processor_type: str, property_name: str, property_value: str): +@step( + 'a {processor_type} processor with the "{property_name}" property set to "{property_value}" in the NiFi flow' +) +def processor_with_property_in_nifi_flow( + context: MinifiTestContext, + processor_type: str, + property_name: str, + property_value: str, +): processor = Processor(processor_type, processor_type) processor.add_property(property_name, property_value) context.containers["nifi"].flow_definition.add_processor(processor) -@given('a {processor_type} processor with the name "{processor_name}" in the "{minifi_container_name}" flow') -def processor_with_name_in_minifi_flow(context: MinifiTestContext, processor_type: str, processor_name: str, minifi_container_name: str): +@given( + 'a {processor_type} processor with the name "{processor_name}" in the "{minifi_container_name}" flow' +) +def processor_with_name_in_minifi_flow( + context: MinifiTestContext, + processor_type: str, + processor_name: str, + minifi_container_name: str, +): processor = Processor(processor_type, processor_name) - context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor) + context.get_or_create_minifi_container( + minifi_container_name + ).flow_definition.add_processor(processor) @given('a {processor_type} processor with the name "{processor_name}"') -def processor_with_name(context: MinifiTestContext, processor_type: str, processor_name: str): +def processor_with_name( + context: MinifiTestContext, processor_type: str, processor_name: str +): context.execute_steps( - f'given a {processor_type} processor with the name "{processor_name}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + f'given a {processor_type} processor with the name "{processor_name}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) -@given("a {processor_type} processor in the \"{minifi_container_name}\" flow") -def processor_in_minifi_flow(context: MinifiTestContext, processor_type: str, minifi_container_name: str): +@given('a {processor_type} processor in the "{minifi_container_name}" flow') +def processor_in_minifi_flow( + context: MinifiTestContext, processor_type: str, minifi_container_name: str +): processor = Processor(processor_type, processor_type) - context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor) + context.get_or_create_minifi_container( + minifi_container_name + ).flow_definition.add_processor(processor) @given("a {processor_type} processor in the NiFi flow") @@ -95,31 +154,62 @@ def processor_in_nifi_flow(context: MinifiTestContext, processor_type: str): @given("a {processor_type} processor") def processor_setup(context: MinifiTestContext, processor_type: str): - context.execute_steps(f'given a {processor_type} processor in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + context.execute_steps( + f'given a {processor_type} processor in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) -@given('the "{property_name}" property of the "{port_name}" port in the "{rpg_name}" remote process group is set to "{property_value}"') -def set_property_of_port_in_rpg(context: MinifiTestContext, property_name: str, port_name: str, rpg_name: str, property_value: str): - rpg = context.get_or_create_minifi_container(DEFAULT_MINIFI_CONTAINER_NAME).flow_definition.get_remote_process_group(rpg_name) +@given( + 'the "{property_name}" property of the "{port_name}" port in the "{rpg_name}" remote process group is set to "{property_value}"' +) +def set_property_of_port_in_rpg( + context: MinifiTestContext, + property_name: str, + port_name: str, + rpg_name: str, + property_value: str, +): + rpg = context.get_or_create_minifi_container( + DEFAULT_MINIFI_CONTAINER_NAME + ).flow_definition.get_remote_process_group(rpg_name) if not rpg: raise ValueError(f"Remote Process Group with name {rpg_name} not found") port = rpg.get_input_port(port_name) if not port: - raise ValueError(f"Remote input port with name {port_name} not found in RPG {rpg_name}") + raise ValueError( + f"Remote input port with name {port_name} not found in RPG {rpg_name}" + ) port.add_property(property_name, property_value) -@given('the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the "{minifi_container_name}" flow') -def set_property_of_processor_in_minifi_flow(context: MinifiTestContext, property_name: str, processor_name: str, property_value: str, minifi_container_name: str): - processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name) +@given( + 'the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the "{minifi_container_name}" flow' +) +def set_property_of_processor_in_minifi_flow( + context: MinifiTestContext, + property_name: str, + processor_name: str, + property_value: str, + minifi_container_name: str, +): + processor = context.get_or_create_minifi_container( + minifi_container_name + ).flow_definition.get_processor(processor_name) if property_value == "(not set)": processor.remove_property(property_name) else: processor.add_property(property_name, property_value) -@given('the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the NiFi flow') -def set_property_of_processor_in_nifi_flow(context: MinifiTestContext, property_name: str, processor_name: str, property_value: str): +@given( + 'the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the NiFi flow' +) +def set_property_of_processor_in_nifi_flow( + context: MinifiTestContext, + property_name: str, + processor_name: str, + property_value: str, +): processor = context.containers["nifi"].flow_definition.get_processor(processor_name) if property_value == "(not set)": processor.remove_property(property_name) @@ -127,15 +217,32 @@ def set_property_of_processor_in_nifi_flow(context: MinifiTestContext, property_ processor.add_property(property_name, property_value) -@given('the "{property_name}" property of the {processor_name} processor is set to "{property_value}"') -def set_property_of_processor(context: MinifiTestContext, property_name: str, processor_name: str, property_value: str): +@given( + 'the "{property_name}" property of the {processor_name} processor is set to "{property_value}"' +) +def set_property_of_processor( + context: MinifiTestContext, + property_name: str, + processor_name: str, + property_value: str, +): context.execute_steps( - f'given the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + f'given the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) -@step('the "{property_name}" property of the {controller_name} controller service is set to "{property_value}"') -def set_property_of_controller_service(context: MinifiTestContext, property_name: str, controller_name: str, property_value: str): - controller_service = context.get_or_create_default_minifi_container().flow_definition.get_controller_service(controller_name) +@step( + 'the "{property_name}" property of the {controller_name} controller service is set to "{property_value}"' +) +def set_property_of_controller_service( + context: MinifiTestContext, + property_name: str, + controller_name: str, + property_value: str, +): + controller_service = context.get_or_create_default_minifi_container().flow_definition.get_controller_service( + controller_name + ) if property_value == "(not set)": controller_service.remove_property(property_name) else: @@ -144,121 +251,244 @@ def set_property_of_controller_service(context: MinifiTestContext, property_name @step('a Funnel with the name "{funnel_name}" is set up') def setup_funnel(context: MinifiTestContext, funnel_name: str): - context.get_or_create_default_minifi_container().flow_definition.add_funnel(Funnel(funnel_name)) + context.get_or_create_default_minifi_container().flow_definition.add_funnel( + Funnel(funnel_name) + ) -@step('in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}') -@step('in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} node is connected to the {target}') -def connect_relationship_in_minifi_flow(context: MinifiTestContext, relationship_name: str, source: str, target: str, minifi_container_name: str): - connection = Connection(source_name=source, source_relationship=relationship_name, target_name=target) - context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_connection(connection) +@step( + 'in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}' +) +@step( + 'in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} node is connected to the {target}' +) +def connect_relationship_in_minifi_flow( + context: MinifiTestContext, + relationship_name: str, + source: str, + target: str, + minifi_container_name: str, +): + connection = Connection( + source_name=source, source_relationship=relationship_name, target_name=target + ) + context.get_or_create_minifi_container( + minifi_container_name + ).flow_definition.add_connection(connection) -@step('in the NiFi flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}') -def connect_relationship_in_nifi_flow(context: MinifiTestContext, relationship_name: str, source: str, target: str): - connection = Connection(source_name=source, source_relationship=relationship_name, target_name=target) +@step( + 'in the NiFi flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}' +) +def connect_relationship_in_nifi_flow( + context: MinifiTestContext, relationship_name: str, source: str, target: str +): + connection = Connection( + source_name=source, source_relationship=relationship_name, target_name=target + ) context.containers["nifi"].flow_definition.add_connection(connection) -@step('the "{relationship_name}" relationship of the {source} processor is connected to the {target}') -@step('the "{relationship_name}" relationship of the {source} node is connected to the {target}') -def connect_relationship(context: MinifiTestContext, relationship_name: str, source: str, target: str): - context.execute_steps(f'given in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}') +@step( + 'the "{relationship_name}" relationship of the {source} processor is connected to the {target}' +) +@step( + 'the "{relationship_name}" relationship of the {source} node is connected to the {target}' +) +def connect_relationship( + context: MinifiTestContext, relationship_name: str, source: str, target: str +): + context.execute_steps( + f'given in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}' + ) -@step("the output port \"{port_name}\" is connected to the {destination_name} processor") -def connect_output_port_to_processor(context: MinifiTestContext, port_name: str, destination_name: str): - connection = Connection(source_name=port_name, source_relationship="undefined", target_name=destination_name) - context.get_or_create_minifi_container(DEFAULT_MINIFI_CONTAINER_NAME).flow_definition.add_connection(connection) +@step('the output port "{port_name}" is connected to the {destination_name} processor') +def connect_output_port_to_processor( + context: MinifiTestContext, port_name: str, destination_name: str +): + connection = Connection( + source_name=port_name, + source_relationship="undefined", + target_name=destination_name, + ) + context.get_or_create_minifi_container( + DEFAULT_MINIFI_CONTAINER_NAME + ).flow_definition.add_connection(connection) @step('the Funnel with the name "{funnel_name}" is connected to the {target}') def connect_funnel_to_target(context: MinifiTestContext, funnel_name: str, target: str): - connection = Connection(source_name=funnel_name, source_relationship="success", target_name=target) - context.get_or_create_default_minifi_container().flow_definition.add_connection(connection) + connection = Connection( + source_name=funnel_name, source_relationship="success", target_name=target + ) + context.get_or_create_default_minifi_container().flow_definition.add_connection( + connection + ) -@step("{processor_name}'s {relationship} relationship is auto-terminated in the \"{minifi_container_name}\" flow") -def auto_terminate_relationship_in_minifi_flow(context: MinifiTestContext, processor_name: str, relationship: str, minifi_container_name: str): - context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name).auto_terminated_relationships.append( - relationship) +@step( + '{processor_name}\'s {relationship} relationship is auto-terminated in the "{minifi_container_name}" flow' +) +def auto_terminate_relationship_in_minifi_flow( + context: MinifiTestContext, + processor_name: str, + relationship: str, + minifi_container_name: str, +): + context.get_or_create_minifi_container( + minifi_container_name + ).flow_definition.get_processor( + processor_name + ).auto_terminated_relationships.append(relationship) -@step("{processor_name}'s {relationship} relationship is auto-terminated in the NiFi flow") -def auto_terminate_relationship_in_nifi_flow(context: MinifiTestContext, processor_name: str, relationship: str): - context.containers["nifi"].flow_definition.get_processor(processor_name).auto_terminated_relationships.append(relationship) +@step( + "{processor_name}'s {relationship} relationship is auto-terminated in the NiFi flow" +) +def auto_terminate_relationship_in_nifi_flow( + context: MinifiTestContext, processor_name: str, relationship: str +): + context.containers["nifi"].flow_definition.get_processor( + processor_name + ).auto_terminated_relationships.append(relationship) @step("{processor_name}'s {relationship} relationship is auto-terminated") -@step("the \"{relationship}\" relationship of the {processor_name} processor is auto-terminated") -def auto_terminate_relationship(context: MinifiTestContext, processor_name: str, relationship: str): - context.execute_steps(f'given {processor_name}\'s {relationship} relationship is auto-terminated in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') +@step( + 'the "{relationship}" relationship of the {processor_name} processor is auto-terminated' +) +def auto_terminate_relationship( + context: MinifiTestContext, processor_name: str, relationship: str +): + context.execute_steps( + f'given {processor_name}\'s {relationship} relationship is auto-terminated in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @given("a transient MiNiFi flow is set up") def setup_transient_minifi_flow(context: MinifiTestContext): - context.get_or_create_default_minifi_container().command = ["/bin/sh", "-c", "timeout 10s ./bin/minifi.sh run && sleep 100"] + context.get_or_create_default_minifi_container().command = [ + "/bin/sh", + "-c", + "timeout 10s ./bin/minifi.sh run && sleep 100", + ] -@step('the scheduling period of the {processor_name} processor is set to "{duration_str}" in the "{minifi_container_name}" flow') -def set_scheduling_period_in_minifi_flow(context: MinifiTestContext, processor_name: str, duration_str: str, minifi_container_name: str): - context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name).scheduling_period = duration_str +@step( + 'the scheduling period of the {processor_name} processor is set to "{duration_str}" in the "{minifi_container_name}" flow' +) +def set_scheduling_period_in_minifi_flow( + context: MinifiTestContext, + processor_name: str, + duration_str: str, + minifi_container_name: str, +): + context.get_or_create_minifi_container( + minifi_container_name + ).flow_definition.get_processor(processor_name).scheduling_period = duration_str -@step('the scheduling period of the {processor_name} processor is set to "{duration_str}" in the NiFi flow') -def set_scheduling_period_in_nifi_flow(context: MinifiTestContext, processor_name: str, duration_str: str): - context.containers["nifi"].flow_definition.get_processor(processor_name).scheduling_period = duration_str +@step( + 'the scheduling period of the {processor_name} processor is set to "{duration_str}" in the NiFi flow' +) +def set_scheduling_period_in_nifi_flow( + context: MinifiTestContext, processor_name: str, duration_str: str +): + context.containers["nifi"].flow_definition.get_processor( + processor_name + ).scheduling_period = duration_str -@step('the scheduling period of the {processor_name} processor is set to "{duration_str}"') -def set_scheduling_period(context: MinifiTestContext, processor_name: str, duration_str: str): - context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name).scheduling_period = duration_str +@step( + 'the scheduling period of the {processor_name} processor is set to "{duration_str}"' +) +def set_scheduling_period( + context: MinifiTestContext, processor_name: str, duration_str: str +): + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name + ).scheduling_period = duration_str @given("parameter context name is set to '{context_name}'") def add_parameter_context(context: MinifiTestContext, context_name: str): - context.get_or_create_default_minifi_container().flow_definition.parameter_contexts.append(ParameterContext(context_name)) + context.get_or_create_default_minifi_container().flow_definition.parameter_contexts.append( + ParameterContext(context_name) + ) @step( - "a non-sensitive parameter in the flow config called '{parameter_name}' with the value '{parameter_value}' in the parameter context '{context_name}'") -def add_non_sensitive_parameter(context: MinifiTestContext, parameter_name: str, parameter_value: str, context_name: str): - parameter_context = context.get_or_create_default_minifi_container().flow_definition.get_parameter_context(context_name) - parameter_context.parameters.append(Parameter(parameter_name, parameter_value, False)) - - -@given("these processor properties are set in the \"{minifi_container_name}\" flow") -def set_processor_properties_in_minifi_flow(context: MinifiTestContext, minifi_container_name: str): + "a non-sensitive parameter in the flow config called '{parameter_name}' with the value '{parameter_value}' in the parameter context '{context_name}'" +) +def add_non_sensitive_parameter( + context: MinifiTestContext, + parameter_name: str, + parameter_value: str, + context_name: str, +): + parameter_context = context.get_or_create_default_minifi_container().flow_definition.get_parameter_context( + context_name + ) + parameter_context.parameters.append( + Parameter(parameter_name, parameter_value, False) + ) + + +@given('these processor properties are set in the "{minifi_container_name}" flow') +def set_processor_properties_in_minifi_flow( + context: MinifiTestContext, minifi_container_name: str +): for row in context.table: - processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(row["processor name"]) + processor = context.get_or_create_minifi_container( + minifi_container_name + ).flow_definition.get_processor(row["processor name"]) processor.add_property(row["property name"], row["property value"]) @given("these processor properties are set") def set_processor_properties(context: MinifiTestContext): for row in context.table: - processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(row["processor name"]) + processor = context.get_or_create_default_minifi_container().flow_definition.get_processor( + row["processor name"] + ) processor.add_property(row["property name"], row["property value"]) -@given("a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the \"{container_name}\" flow") +@given( + 'a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the "{container_name}" flow' +) def step_impl(context: MinifiTestContext, proxy_type: str, container_name: str): - controller_service = ControllerService(class_name="ProxyConfigurationService", service_name="ProxyConfigurationService") + controller_service = ControllerService( + class_name="ProxyConfigurationService", service_name="ProxyConfigurationService" + ) prefix = proxy_type.lower() if proxy_type.lower() != "direct" else "http" - controller_service.add_property("Proxy Server Host", f"{prefix}://http-proxy-{context.scenario_id}") + controller_service.add_property( + "Proxy Server Host", f"{prefix}://http-proxy-{context.scenario_id}" + ) controller_service.add_property("Proxy User Name", "admin") controller_service.add_property("Proxy User Password", "test101") - controller_service.add_property("Proxy Type", "DIRECT" if proxy_type.lower() == "direct" else "HTTP") + controller_service.add_property( + "Proxy Type", "DIRECT" if proxy_type.lower() == "direct" else "HTTP" + ) if proxy_type.lower() == "http": controller_service.add_property("Proxy Server Port", "3128") else: controller_service.add_property("Proxy Server Port", "3129") - context.get_or_create_minifi_container(container_name).flow_definition.controller_services.append(controller_service) + context.get_or_create_minifi_container( + container_name + ).flow_definition.controller_services.append(controller_service) -@given("a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration") -def step_impl(context: MinifiTestContext, proxy_type: str): - context.execute_steps(f"given a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow") +@given( + "a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration" +) +def set_up_proxy_configuration_service_in_default_flow( + context: MinifiTestContext, proxy_type: str +): + context.execute_steps( + f'given a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @step("the processors are connected up as described here") @@ -269,27 +499,45 @@ def connect_processors_from_table(context: MinifiTestContext): relationship = row["relationship name"] if dest_proc_name == "auto-terminated": context.get_or_create_default_minifi_container().flow_definition.get_processor( - source_proc_name).auto_terminated_relationships.append(relationship) + source_proc_name + ).auto_terminated_relationships.append(relationship) else: - connection = Connection(source_name=row["source name"], source_relationship=relationship, - target_name=row["destination name"]) - context.get_or_create_default_minifi_container().flow_definition.add_connection(connection) - - -@step("{processor_name} is EVENT_DRIVEN in the \"{minifi_container_name}\" flow") -def set_processor_event_driven_in_minifi_flow(context: MinifiTestContext, processor_name: str, minifi_container_name: str): - processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name) + connection = Connection( + source_name=row["source name"], + source_relationship=relationship, + target_name=row["destination name"], + ) + context.get_or_create_default_minifi_container().flow_definition.add_connection( + connection + ) + + +@step('{processor_name} is EVENT_DRIVEN in the "{minifi_container_name}" flow') +def set_processor_event_driven_in_minifi_flow( + context: MinifiTestContext, processor_name: str, minifi_container_name: str +): + processor = context.get_or_create_minifi_container( + minifi_container_name + ).flow_definition.get_processor(processor_name) processor.scheduling_strategy = "EVENT_DRIVEN" @step("{processor_name} is EVENT_DRIVEN") def set_processor_event_driven(context: MinifiTestContext, processor_name: str): - context.execute_steps(f'given {processor_name} is EVENT_DRIVEN in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + context.execute_steps( + f'given {processor_name} is EVENT_DRIVEN in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @step("{processor_name} is TIMER_DRIVEN with {scheduling_period} scheduling period") -def set_processor_timer_driven(context: MinifiTestContext, processor_name: str, scheduling_period: int): - processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) +def set_processor_timer_driven( + context: MinifiTestContext, processor_name: str, scheduling_period: int +): + processor = ( + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name + ) + ) processor.scheduling_strategy = "TIMER_DRIVEN" processor.scheduling_period = scheduling_period @@ -297,63 +545,128 @@ def set_processor_timer_driven(context: MinifiTestContext, processor_name: str, @given("a {service_name} controller service is set up") @given("an {service_name} controller service is set up") def setup_controller_service(context: MinifiTestContext, service_name: str): - controller_service = ControllerService(class_name=service_name, service_name=service_name) - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) + controller_service = ControllerService( + class_name=service_name, service_name=service_name + ) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + controller_service + ) -@given('a {class_name} controller service named "{service_name}" is set up and the "{property_name}" property set to "{property_value}"') -def setup_controller_service_with_property(context: MinifiTestContext, class_name: str, service_name: str, property_name: str, property_value: str): - controller_service = ControllerService(class_name=class_name, service_name=service_name) +@given( + 'a {class_name} controller service named "{service_name}" is set up and the "{property_name}" property set to "{property_value}"' +) +def setup_named_controller_service_with_property( + context: MinifiTestContext, + class_name: str, + service_name: str, + property_name: str, + property_value: str, +): + controller_service = ControllerService( + class_name=class_name, service_name=service_name + ) controller_service.add_property(property_name, property_value) - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + controller_service + ) -@given('a {service_name} controller service is set up and the "{property_name}" property set to "{property_value}"') -def setup_controller_service_with_property(context: MinifiTestContext, service_name: str, property_name: str, property_value: str): - controller_service = ControllerService(class_name=service_name, service_name=service_name) +@given( + 'a {service_name} controller service is set up and the "{property_name}" property set to "{property_value}"' +) +def setup_controller_service_with_property( + context: MinifiTestContext, + service_name: str, + property_name: str, + property_value: str, +): + controller_service = ControllerService( + class_name=service_name, service_name=service_name + ) controller_service.add_property(property_name, property_value) - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + controller_service + ) -@given("the \"{property_name}\" property of the {processor_name} processor is set to match the attribute \"{attribute_key}\" to \"{attribute_value}\"") -def set_processor_property_to_match_attribute(context: MinifiTestContext, property_name: str, processor_name: str, attribute_key: str, attribute_value: str): - processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) +@given( + 'the "{property_name}" property of the {processor_name} processor is set to match the attribute "{attribute_key}" to "{attribute_value}"' +) +def set_processor_property_to_match_attribute( + context: MinifiTestContext, + property_name: str, + processor_name: str, + attribute_key: str, + attribute_value: str, +): + processor = ( + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name + ) + ) if attribute_value == "(not set)": # Ignore filtering processor.add_property(property_name, "true") return filtering = "${" + attribute_key + ":equals('" + attribute_value + "')}" - logging.info("Filter: \"%s\"", filtering) - logging.info("Key: \"%s\", value: \"%s\"", attribute_key, attribute_value) + logger.info('Filter: "%s"', filtering) + logger.info('Key: "%s", value: "%s"', attribute_key, attribute_value) processor.add_property(property_name, filtering) -@given("the max concurrent tasks attribute of the {processor_name} processor is set to {max_concurrent_tasks:d}") -def set_processor_max_concurrent_tasks(context, processor_name: str, max_concurrent_tasks: int): - processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) +@given( + "the max concurrent tasks attribute of the {processor_name} processor is set to {max_concurrent_tasks:d}" +) +def set_processor_max_concurrent_tasks( + context, processor_name: str, max_concurrent_tasks: int +): + processor = ( + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name + ) + ) processor.set_max_concurrent_tasks(max_concurrent_tasks) -@given("the \"{property_name}\" properties of the {processor_name_one} and {processor_name_two} processors are set to the same random UUID") -def set_processors_properties_to_same_uuid(context, property_name, processor_name_one, processor_name_two): +@given( + 'the "{property_name}" properties of the {processor_name_one} and {processor_name_two} processors are set to the same random UUID' +) +def set_processors_properties_to_same_uuid( + context, property_name, processor_name_one, processor_name_two +): uuid_str = str(uuid.uuid4()) - context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name_one).add_property(property_name, uuid_str) - context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name_two).add_property(property_name, uuid_str) + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name_one + ).add_property(property_name, uuid_str) + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name_two + ).add_property(property_name, uuid_str) # TLS -def add_ssl_context_service_for_minifi(context: MinifiTestContext, cert_name: str, use_system_cert_store: bool = False): - ssl_context_service = context.get_or_create_default_minifi_container().flow_definition.get_controller_service("SSLContextService") +def add_ssl_context_service_for_minifi( + context: MinifiTestContext, cert_name: str, use_system_cert_store: bool = False +): + ssl_context_service = context.get_or_create_default_minifi_container().flow_definition.get_controller_service( + "SSLContextService" + ) if ssl_context_service is not None: return - controller_service = ControllerService(class_name="SSLContextService", service_name="SSLContextService") - controller_service.add_property("Client Certificate", f"/tmp/resources/{cert_name}.crt") + controller_service = ControllerService( + class_name="SSLContextService", service_name="SSLContextService" + ) + controller_service.add_property( + "Client Certificate", f"/tmp/resources/{cert_name}.crt" + ) controller_service.add_property("Private Key", f"/tmp/resources/{cert_name}.key") if use_system_cert_store: controller_service.add_property("Use System Cert Store", "true") else: controller_service.add_property("CA Certificate", "/tmp/resources/root_ca.crt") - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + controller_service + ) @given("an ssl context service is set up") @@ -362,62 +675,126 @@ def setup_ssl_context_service(context: MinifiTestContext): @given("an ssl context service is set up for {processor_name}") -@given("an ssl context service with a manual CA cert file is set up for {processor_name}") +@given( + "an ssl context service with a manual CA cert file is set up for {processor_name}" +) def setup_ssl_context_service_for_processor(context, processor_name): add_ssl_context_service_for_minifi(context, "minifi_client") - processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) - processor.add_property('SSL Context Service', 'SSLContextService') + processor = ( + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name + ) + ) + processor.add_property("SSL Context Service", "SSLContextService") -@given("an ssl context service using the system CA cert store is set up for {processor_name}") -def setup_ssl_context_service_system_ca_for_processor(context: MinifiTestContext, processor_name): - add_ssl_context_service_for_minifi(context, "minifi_client", use_system_cert_store=True) - processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) - processor.add_property('SSL Context Service', 'SSLContextService') +@given( + "an ssl context service using the system CA cert store is set up for {processor_name}" +) +def setup_ssl_context_service_system_ca_for_processor( + context: MinifiTestContext, processor_name +): + add_ssl_context_service_for_minifi( + context, "minifi_client", use_system_cert_store=True + ) + processor = ( + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name + ) + ) + processor.add_property("SSL Context Service", "SSLContextService") -@given("a RemoteProcessGroup node with name \"{rpg_name}\" is opened on \"{address}\" with transport protocol set to \"{transport_protocol}\"") -def open_rpg_with_transport_protocol(context: MinifiTestContext, rpg_name: str, address: str, transport_protocol: str): - context.get_or_create_default_minifi_container().flow_definition.add_remote_process_group(address, rpg_name, transport_protocol) +@given( + 'a RemoteProcessGroup node with name "{rpg_name}" is opened on "{address}" with transport protocol set to "{transport_protocol}"' +) +def open_rpg_with_transport_protocol( + context: MinifiTestContext, rpg_name: str, address: str, transport_protocol: str +): + context.get_or_create_default_minifi_container().flow_definition.add_remote_process_group( + address, rpg_name, transport_protocol + ) -@given("a RemoteProcessGroup node with name \"{rpg_name}\" is opened on \"{address}\"") +@given('a RemoteProcessGroup node with name "{rpg_name}" is opened on "{address}"') def open_rpg(context: MinifiTestContext, rpg_name: str, address: str): - context.execute_steps(f"given a RemoteProcessGroup node with name \"{rpg_name}\" is opened on \"{address}\" with transport protocol set to \"RAW\"") + context.execute_steps( + f'given a RemoteProcessGroup node with name "{rpg_name}" is opened on "{address}" with transport protocol set to "RAW"' + ) -@given("an input port with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"") +@given( + 'an input port with name "{port_name}" is created on the RemoteProcessGroup named "{rpg_name}"' +) def create_input_port_on_rpg(context: MinifiTestContext, port_name: str, rpg_name: str): - context.get_or_create_default_minifi_container().flow_definition.add_input_port_to_rpg(rpg_name, port_name) - + context.get_or_create_default_minifi_container().flow_definition.add_input_port_to_rpg( + rpg_name, port_name + ) -@given("an input port using compression with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"") -def create_input_port_with_compression_on_rpg(context: MinifiTestContext, port_name: str, rpg_name: str): - context.get_or_create_default_minifi_container().flow_definition.add_input_port_to_rpg(rpg_name, port_name, use_compression=True) - -@given("an output port with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"") -def create_output_port_on_rpg(context: MinifiTestContext, port_name: str, rpg_name: str): - context.get_or_create_default_minifi_container().flow_definition.add_output_port_to_rpg(rpg_name, port_name) +@given( + 'an input port using compression with name "{port_name}" is created on the RemoteProcessGroup named "{rpg_name}"' +) +def create_input_port_with_compression_on_rpg( + context: MinifiTestContext, port_name: str, rpg_name: str +): + context.get_or_create_default_minifi_container().flow_definition.add_input_port_to_rpg( + rpg_name, port_name, use_compression=True + ) -@given("an output port using compression with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"") -def create_output_port_with_compression_on_rpg(context: MinifiTestContext, port_name: str, rpg_name: str): - context.get_or_create_default_minifi_container().flow_definition.add_output_port_to_rpg(rpg_name, port_name, use_compression=True) +@given( + 'an output port with name "{port_name}" is created on the RemoteProcessGroup named "{rpg_name}"' +) +def create_output_port_on_rpg( + context: MinifiTestContext, port_name: str, rpg_name: str +): + context.get_or_create_default_minifi_container().flow_definition.add_output_port_to_rpg( + rpg_name, port_name + ) -@given("a NiFi flow is receiving data from the RemoteProcessGroup named \"{rpg_name}\" in an input port named \"{input_port_name}\" which has the same id as the port named \"{rpg_port_name}\"") -def nifi_receive_from_rpg_input_port(context: MinifiTestContext, input_port_name: str, rpg_port_name: str, rpg_name: str): - input_port_id = context.get_or_create_default_minifi_container().flow_definition.get_input_port_id_of_rpg(rpg_name, rpg_port_name) - context.containers["nifi"].flow_definition.add_input_port(input_port_id, input_port_name) +@given( + 'an output port using compression with name "{port_name}" is created on the RemoteProcessGroup named "{rpg_name}"' +) +def create_output_port_with_compression_on_rpg( + context: MinifiTestContext, port_name: str, rpg_name: str +): + context.get_or_create_default_minifi_container().flow_definition.add_output_port_to_rpg( + rpg_name, port_name, use_compression=True + ) -@given("a NiFi flow is sending data to an output port named \"{port_name}\" with the id of the port named \"{rpg_port_name}\" from the RemoteProcessGroup named \"{rpg_name}\"") -def nifi_send_to_rpg_output_port(context: MinifiTestContext, port_name: str, rpg_port_name: str, rpg_name: str): - output_port_id = context.get_or_create_default_minifi_container().flow_definition.get_output_port_id_of_rpg(rpg_name, rpg_port_name) - context.containers["nifi"].flow_definition.add_output_port(output_port_id, port_name) +@given( + 'a NiFi flow is receiving data from the RemoteProcessGroup named "{rpg_name}" in an input port named "{input_port_name}" which has the same id as the port named "{rpg_port_name}"' +) +def nifi_receive_from_rpg_input_port( + context: MinifiTestContext, input_port_name: str, rpg_port_name: str, rpg_name: str +): + input_port_id = context.get_or_create_default_minifi_container().flow_definition.get_input_port_id_of_rpg( + rpg_name, rpg_port_name + ) + context.containers["nifi"].flow_definition.add_input_port( + input_port_id, input_port_name + ) -@given("the connection going to {destination} has \"drop empty\" set") +@given( + 'a NiFi flow is sending data to an output port named "{port_name}" with the id of the port named "{rpg_port_name}" from the RemoteProcessGroup named "{rpg_name}"' +) +def nifi_send_to_rpg_output_port( + context: MinifiTestContext, port_name: str, rpg_port_name: str, rpg_name: str +): + output_port_id = context.get_or_create_default_minifi_container().flow_definition.get_output_port_id_of_rpg( + rpg_name, rpg_port_name + ) + context.containers["nifi"].flow_definition.add_output_port( + output_port_id, port_name + ) + + +@given('the connection going to {destination} has "drop empty" set') def set_drop_empty_flag_for_connection(context: MinifiTestContext, destination: str): - context.get_or_create_default_minifi_container().flow_definition.set_drop_empty_for_destination(destination) + context.get_or_create_default_minifi_container().flow_definition.set_drop_empty_for_destination( + destination + ) diff --git a/bootstrap/cli.py b/bootstrap/cli.py index bf2f8cb6b0..505efe8d64 100644 --- a/bootstrap/cli.py +++ b/bootstrap/cli.py @@ -15,16 +15,22 @@ import os -import inquirer +import inquirer from minifi_option import MinifiOptions from package_manager import PackageManager from system_dependency import install_required -def install_dependencies(minifi_options: MinifiOptions, package_manager: PackageManager) -> bool: +def install_dependencies( + minifi_options: MinifiOptions, package_manager: PackageManager +) -> bool: res = install_required(minifi_options, package_manager) - print("Installation went smoothly" if res else "There were some error during installation") + print( + "Installation went smoothly" + if res + else "There were some error during installation" + ) return res @@ -33,28 +39,32 @@ def run_cmake(minifi_options: MinifiOptions, package_manager: PackageManager): os.mkdir(minifi_options.build_dir) cmake_cmd = f"cmake {minifi_options.create_cmake_generator_str()} {minifi_options.create_cmake_options_str()} {minifi_options.source_dir} -B {minifi_options.build_dir}" res = package_manager.run_cmd(cmake_cmd) - print("CMake command run successfully" if res else "CMake command run unsuccessfully") + print( + "CMake command run successfully" if res else "CMake command run unsuccessfully" + ) return res def do_build(minifi_options: MinifiOptions, package_manager: PackageManager): - build_cmd = f"cmake --build {str(minifi_options.build_dir)} {minifi_options.create_cmake_build_flags_str()}" + build_cmd = f"cmake --build {minifi_options.build_dir!s} {minifi_options.create_cmake_build_flags_str()}" res = package_manager.run_cmd(build_cmd) print("Build was successful" if res else "Build was unsuccessful") return res def do_package(minifi_options: MinifiOptions, package_manager: PackageManager): - build_cmd = f"cmake --build {str(minifi_options.build_dir)} --target package {minifi_options.create_cmake_build_flags_str()}" + build_cmd = f"cmake --build {minifi_options.build_dir!s} --target package {minifi_options.create_cmake_build_flags_str()}" return package_manager.run_cmd(build_cmd) def do_docker_build(minifi_options: MinifiOptions, package_manager: PackageManager): - build_cmd = f"cmake --build {str(minifi_options.build_dir)} --target docker" + build_cmd = f"cmake --build {minifi_options.build_dir!s} --target docker" return package_manager.run_cmd(build_cmd) -def do_one_click_build(minifi_options: MinifiOptions, package_manager: PackageManager) -> bool: +def do_one_click_build( + minifi_options: MinifiOptions, package_manager: PackageManager +) -> bool: assert install_dependencies(minifi_options, package_manager) assert run_cmake(minifi_options, package_manager) assert do_build(minifi_options, package_manager) @@ -62,7 +72,9 @@ def do_one_click_build(minifi_options: MinifiOptions, package_manager: PackageMa return True -def do_one_click_configuration(minifi_options: MinifiOptions, package_manager: PackageManager) -> bool: +def do_one_click_configuration( + minifi_options: MinifiOptions, package_manager: PackageManager +) -> bool: assert install_dependencies(minifi_options, package_manager) assert run_cmake(minifi_options, package_manager) return True @@ -93,10 +105,14 @@ def main_menu(minifi_options: MinifiOptions, package_manager: PackageManager): main_menu_prompt = inquirer.prompt(questions) if main_menu_prompt is None: break - done = main_menu_options[main_menu_prompt["sub_menu"]](minifi_options, package_manager) + done = main_menu_options[main_menu_prompt["sub_menu"]]( + minifi_options, package_manager + ) -def build_type_menu(minifi_options: MinifiOptions, _package_manager: PackageManager) -> bool: +def build_type_menu( + minifi_options: MinifiOptions, _package_manager: PackageManager +) -> bool: questions = [ inquirer.List( "build_type", @@ -113,12 +129,13 @@ def build_type_menu(minifi_options: MinifiOptions, _package_manager: PackageMana return False -def build_dir_menu(minifi_options: MinifiOptions, _package_manager: PackageManager) -> bool: +def build_dir_menu( + minifi_options: MinifiOptions, _package_manager: PackageManager +) -> bool: questions = [ - inquirer.Path('build_dir', - message="Build directory", - default=minifi_options.build_dir - ), + inquirer.Path( + "build_dir", message="Build directory", default=minifi_options.build_dir + ), ] answers = inquirer.prompt(questions) if answers is None: @@ -128,15 +145,21 @@ def build_dir_menu(minifi_options: MinifiOptions, _package_manager: PackageManag return False -def extension_options_menu(minifi_options: MinifiOptions, _package_manager: PackageManager) -> bool: +def extension_options_menu( + minifi_options: MinifiOptions, _package_manager: PackageManager +) -> bool: possible_values = [option_name for option_name in minifi_options.extension_options] - selected_values = [option.name for option in minifi_options.extension_options.values() if option.value == "ON"] + selected_values = [ + option.name + for option in minifi_options.extension_options.values() + if option.value == "ON" + ] questions = [ inquirer.Checkbox( "options", message="MiNiFi C++ Extension Options (space to select, enter to confirm)", choices=possible_values, - default=selected_values + default=selected_values, ), ] @@ -153,15 +176,21 @@ def extension_options_menu(minifi_options: MinifiOptions, _package_manager: Pack return False -def build_options_menu(minifi_options: MinifiOptions, _package_manager: PackageManager) -> bool: +def build_options_menu( + minifi_options: MinifiOptions, _package_manager: PackageManager +) -> bool: possible_values = [option_name for option_name in minifi_options.build_options] - selected_values = [option.name for option in minifi_options.build_options.values() if option.value == "ON"] + selected_values = [ + option.name + for option in minifi_options.build_options.values() + if option.value == "ON" + ] questions = [ inquirer.Checkbox( "options", message="MiNiFi C++ Build Options (space to select, enter to confirm)", choices=possible_values, - default=selected_values + default=selected_values, ), ] @@ -178,7 +207,9 @@ def build_options_menu(minifi_options: MinifiOptions, _package_manager: PackageM return False -def step_by_step_menu(minifi_options: MinifiOptions, package_manager: PackageManager) -> bool: +def step_by_step_menu( + minifi_options: MinifiOptions, package_manager: PackageManager +) -> bool: done = False while not done: step_by_step_options = { @@ -195,13 +226,18 @@ def step_by_step_menu(minifi_options: MinifiOptions, package_manager: PackageMan inquirer.List( "selection", message="Step by step menu", - choices=[step_by_step_menu_option_name for step_by_step_menu_option_name in step_by_step_options], + choices=[ + step_by_step_menu_option_name + for step_by_step_menu_option_name in step_by_step_options + ], ), ] step_by_step_prompt = inquirer.prompt(questions) if step_by_step_prompt is None: return True - step_by_step_options[step_by_step_prompt["selection"]](minifi_options, package_manager) - done = step_by_step_prompt['selection'] == 'Back' + step_by_step_options[step_by_step_prompt["selection"]]( + minifi_options, package_manager + ) + done = step_by_step_prompt["selection"] == "Back" return False diff --git a/bootstrap/cmake_parser.py b/bootstrap/cmake_parser.py index 4c9fae362b..9428eeb3b2 100644 --- a/bootstrap/cmake_parser.py +++ b/bootstrap/cmake_parser.py @@ -37,37 +37,47 @@ def create_cmake_option_str(self): return f"-D{self.name}={self.value}" -def create_cmake_cache(cmake_path: str, cmake_options: str, directory: str, package_manager: PackageManager): - cmake_lists_path = os.path.join(directory, 'CMakeLists.txt') +def create_cmake_cache( + cmake_path: str, cmake_options: str, directory: str, package_manager: PackageManager +): + cmake_lists_path = os.path.join(directory, "CMakeLists.txt") - with open(cmake_lists_path, 'w') as cmake_lists_file: - cmake_lists_file.write('cmake_minimum_required(VERSION 3.5)\n') + with open(cmake_lists_path, "w") as cmake_lists_file: + cmake_lists_file.write("cmake_minimum_required(VERSION 3.5)\n") cmake_lists_file.write(f'include("{cmake_path}")\n') if cmake_options is None: - assert package_manager.run_cmd(f'cmake -G Ninja -Wno-dev --log-level=ERROR {directory} -B {directory}') + assert package_manager.run_cmd( + f"cmake -G Ninja -Wno-dev --log-level=ERROR {directory} -B {directory}" + ) else: assert package_manager.run_cmd( - f'cmake -G Ninja -Wno-dev --no-warn-unused-cli --log-level=ERROR {cmake_options} {directory} -B {directory}') - return os.path.join(directory, 'CMakeCache.txt') + f"cmake -G Ninja -Wno-dev --no-warn-unused-cli --log-level=ERROR {cmake_options} {directory} -B {directory}" + ) + return os.path.join(directory, "CMakeCache.txt") def parse_cmake_cache_values(path: str): parsed_variables = {} - with open(path, 'r') as file: + with open(path, "r") as file: contents = file.read() - pattern = r'\/\/(?P[\s\S]*?)\n(?P.+?):(?P\w+?)=(?P.+)\n' + pattern = r"\/\/(?P[\s\S]*?)\n(?P.+?):(?P\w+?)=(?P.+)\n" matches = re.findall(pattern, contents) for match in matches: - cmake_cache_value = CMakeCacheValue(description=match[0].replace("//", "").replace("\n", " "), - name=match[1], - value_type=match[2], value=match[3]) + cmake_cache_value = CMakeCacheValue( + description=match[0].replace("//", "").replace("\n", " "), + name=match[1], + value_type=match[2], + value=match[3], + ) if cmake_cache_value.name.endswith("-STRINGS"): - possible_values_of = cmake_cache_value.name[:-len("-STRINGS")] + possible_values_of = cmake_cache_value.name[: -len("-STRINGS")] if possible_values_of not in parsed_variables: raise ValueError(f"Did not parse {possible_values_of} yet") - parsed_variables[possible_values_of].possible_values = cmake_cache_value.value.split(";") + parsed_variables[ + possible_values_of + ].possible_values = cmake_cache_value.value.split(";") continue parsed_variables[cmake_cache_value.name] = cmake_cache_value diff --git a/bootstrap/main.py b/bootstrap/main.py index af9d1eaf53..508facd40c 100644 --- a/bootstrap/main.py +++ b/bootstrap/main.py @@ -12,29 +12,51 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import tempfile - import argparse import pathlib +import tempfile -from cli import main_menu, do_one_click_build, do_one_click_configuration +from cli import do_one_click_build, do_one_click_configuration, main_menu from minifi_option import parse_minifi_options from package_manager import get_package_manager -if __name__ == '__main__': +if __name__ == "__main__": with tempfile.TemporaryDirectory() as cmake_cache_dir: parser = argparse.ArgumentParser() - parser.add_argument('--noconfirm', action="store_true", default=False, - help="Bypass any and all “Are you sure?” messages.") - parser.add_argument('--minifi-options', default="", help="Overrides the default minifi options during the " - "initial parsing") - parser.add_argument('--cmake-options', default="", help="Appends this to the final cmake command") - parser.add_argument('--skip-compiler-install', action="store_true", default=False, - help="Skips the installation of the default compiler") - parser.add_argument('--noninteractive', action="store_true", default=False, - help="Initiates the one click build") - parser.add_argument('--run-configuration', action="store_true", default=False, - help="Runs configuration") + parser.add_argument( + "--noconfirm", + action="store_true", + default=False, + help="Bypass any and all “Are you sure?” messages.", + ) + parser.add_argument( + "--minifi-options", + default="", + help="Overrides the default minifi options during the initial parsing", + ) + parser.add_argument( + "--cmake-options", + default="", + help="Appends this to the final cmake command", + ) + parser.add_argument( + "--skip-compiler-install", + action="store_true", + default=False, + help="Skips the installation of the default compiler", + ) + parser.add_argument( + "--noninteractive", + action="store_true", + default=False, + help="Initiates the one click build", + ) + parser.add_argument( + "--run-configuration", + action="store_true", + default=False, + help="Runs configuration", + ) args = parser.parse_args() no_confirm = args.noconfirm or args.noninteractive @@ -45,15 +67,26 @@ compiler_override = "" package_manager.ensure_environment() - cmake_options_for_parsing = " ".join(filter(None, [args.minifi_options, compiler_override])) - cmake_options_for_cmake = " ".join(filter(None, [args.cmake_options, compiler_override])) + cmake_options_for_parsing = " ".join( + filter(None, [args.minifi_options, compiler_override]) + ) + cmake_options_for_cmake = " ".join( + filter(None, [args.cmake_options, compiler_override]) + ) - path = pathlib.Path(__file__).parent.resolve() / '..' / "cmake" / "MiNiFiOptions.cmake" + path = ( + pathlib.Path(__file__).parent.resolve() + / ".." + / "cmake" + / "MiNiFiOptions.cmake" + ) - minifi_options = parse_minifi_options(str(path.as_posix()), - cmake_options_for_parsing, - package_manager, - cmake_cache_dir) + minifi_options = parse_minifi_options( + str(path.as_posix()), + cmake_options_for_parsing, + package_manager, + cmake_cache_dir, + ) minifi_options.load_option_state() minifi_options.no_confirm = no_confirm minifi_options.set_cmake_override(cmake_options_for_cmake) diff --git a/bootstrap/minifi_option.py b/bootstrap/minifi_option.py index 3bfc036f35..a68d01640c 100644 --- a/bootstrap/minifi_option.py +++ b/bootstrap/minifi_option.py @@ -13,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict - -import pathlib import json -import platform import os +import pathlib +import platform import cmake_parser from cmake_parser import CMakeCacheValue @@ -26,29 +24,70 @@ class MinifiOptions: - def __init__(self, cache_values: Dict[str, CMakeCacheValue]): + def __init__(self, cache_values: dict[str, CMakeCacheValue]): self.cmake_override = "" - self.build_type = CMakeCacheValue("Specifies the build type on single-configuration generators", - "CMAKE_BUILD_TYPE", "STRING", "Release") - self.build_type.possible_values = ["Release", "Debug", "RelWithDebInfo", "MinSizeRel"] - additional_build_options = ["DOCKER_BUILD_ONLY", "DOCKER_SKIP_TESTS", "DOCKER_CREATE_RPM", "SKIP_TESTS", "PORTABLE"] - self.use_ninja = CMakeCacheValue("Specifies if CMake should use the Ninja generator or the system default", "USE_NINJA", "BOOL", "ON") - self.bool_options = {name: cache_value for name, cache_value in cache_values.items() if - cache_value.value_type == "BOOL" and ("ENABLE" in name or "MINIFI" in name or name in additional_build_options)} - self.build_options = {name: cache_value for name, cache_value in self.bool_options.items() if "MINIFI" in name or name in additional_build_options} + self.build_type = CMakeCacheValue( + "Specifies the build type on single-configuration generators", + "CMAKE_BUILD_TYPE", + "STRING", + "Release", + ) + self.build_type.possible_values = [ + "Release", + "Debug", + "RelWithDebInfo", + "MinSizeRel", + ] + additional_build_options = [ + "DOCKER_BUILD_ONLY", + "DOCKER_SKIP_TESTS", + "DOCKER_CREATE_RPM", + "SKIP_TESTS", + "PORTABLE", + ] + self.use_ninja = CMakeCacheValue( + "Specifies if CMake should use the Ninja generator or the system default", + "USE_NINJA", + "BOOL", + "ON", + ) + self.bool_options = { + name: cache_value + for name, cache_value in cache_values.items() + if cache_value.value_type == "BOOL" + and ( + "ENABLE" in name or "MINIFI" in name or name in additional_build_options + ) + } + self.build_options = { + name: cache_value + for name, cache_value in self.bool_options.items() + if "MINIFI" in name or name in additional_build_options + } self.build_options["USE_NINJA"] = self.use_ninja - self.extension_options = {name: cache_value for name, cache_value in self.bool_options.items() if "ENABLE" in name} - self.multi_choice_options = [cache_value for name, cache_value in cache_values.items() if - cache_value.value_type == "STRING" and cache_value.possible_values is not None] + self.extension_options = { + name: cache_value + for name, cache_value in self.bool_options.items() + if "ENABLE" in name + } + self.multi_choice_options = [ + cache_value + for name, cache_value in cache_values.items() + if cache_value.value_type == "STRING" + and cache_value.possible_values is not None + ] self.build_dir = pathlib.Path(__file__).parent.parent.resolve() / "build" self.source_dir = pathlib.Path(__file__).parent.parent.resolve() self.no_confirm = False def create_cmake_options_str(self) -> str: - cmake_options = [bool_option.create_cmake_option_str() for name, bool_option in self.bool_options.items()] + cmake_options = [ + bool_option.create_cmake_option_str() + for name, bool_option in self.bool_options.items() + ] if self.cmake_override: cmake_options.append(self.cmake_override) - cmake_options.append(f'-DCMAKE_BUILD_TYPE={self.build_type.value}') + cmake_options.append(f"-DCMAKE_BUILD_TYPE={self.build_type.value}") cmake_options_str = " ".join(filter(None, cmake_options)) return cmake_options_str @@ -66,7 +105,10 @@ def create_cmake_build_flags_str(self) -> str: def is_enabled(self, option_name: str) -> bool: if option_name not in self.bool_options: raise ValueError(f"Expected {option_name} to be a minifi option") - if "ENABLE_ALL" in self.bool_options and self.bool_options["ENABLE_ALL"].value == "ON": + if ( + "ENABLE_ALL" in self.bool_options + and self.bool_options["ENABLE_ALL"].value == "ON" + ): return True return self.bool_options[option_name].value == "ON" @@ -74,7 +116,7 @@ def set_cmake_override(self, cmake_override: str): self.cmake_override = cmake_override def save_option_state(self): - options_dict = dict() + options_dict = {} for option_name in self.bool_options: options_dict[option_name] = self.bool_options[option_name].value options_dict[self.use_ninja.name] = self.use_ninja.value @@ -102,6 +144,10 @@ def load_option_state(self): self.build_dir = pathlib.Path(options_dict["build_dir"]) -def parse_minifi_options(path: str, cmake_options: str, package_manager: PackageManager, cmake_cache_dir: str): - cmake_cache_path = cmake_parser.create_cmake_cache(path, cmake_options, cmake_cache_dir, package_manager) +def parse_minifi_options( + path: str, cmake_options: str, package_manager: PackageManager, cmake_cache_dir: str +): + cmake_cache_path = cmake_parser.create_cmake_cache( + path, cmake_options, cmake_cache_dir, package_manager + ) return MinifiOptions(cmake_parser.parse_cmake_cache_values(cmake_cache_path)) diff --git a/bootstrap/package_manager.py b/bootstrap/package_manager.py index 049697c714..ade32d4698 100644 --- a/bootstrap/package_manager.py +++ b/bootstrap/package_manager.py @@ -18,7 +18,6 @@ import subprocess import sys from enum import Enum -from typing import Dict, Set from distro import distro @@ -32,56 +31,68 @@ def _query_yes_no(question: str, no_confirm: bool) -> bool: valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} if no_confirm: - print("Running {} with noconfirm".format(question)) + print(f"Running {question} with noconfirm") return True while True: - print("{} [y/n]".format(question), end=' ', flush=True) + print(f"{question} [y/n]", end=" ", flush=True) choice = input().lower() if choice in valid: return valid[choice] else: - print("Please respond with 'yes' or 'no' " "(or 'y' or 'n').") + print("Please respond with 'yes' or 'no' (or 'y' or 'n').") def _run_command_with_confirm(command: str, no_confirm: bool) -> bool: - if _query_yes_no("Running {}".format(command), no_confirm): + if _query_yes_no(f"Running {command}", no_confirm): return os.system(command) == 0 -class PackageManager(object): +class PackageManager: def __init__(self, no_confirm): self.no_confirm = no_confirm - pass - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - raise Exception("NotImplementedException") + def install(self, dependencies: dict[str, set[str]]) -> bool: + raise RuntimeError("NotImplementedException") def install_compiler(self) -> str: - raise Exception("NotImplementedException") + raise RuntimeError("NotImplementedException") def ensure_environment(self): pass - def _install(self, dependencies: Dict[str, Set[str]], replace_dict: Dict[str, Set[str]], install_cmd: str) -> bool: - dependencies.update({k: v for k, v in replace_dict.items() if k in dependencies}) + def _install( + self, + dependencies: dict[str, set[str]], + replace_dict: dict[str, set[str]], + install_cmd: str, + ) -> bool: + dependencies.update( + {k: v for k, v in replace_dict.items() if k in dependencies} + ) dependencies = self._filter_out_installed_packages(dependencies) - dependencies_str = " ".join(str(value) for value_set in dependencies.values() for value in value_set) + dependencies_str = " ".join( + str(value) for value_set in dependencies.values() for value in value_set + ) if not dependencies_str or dependencies_str.isspace(): return True - return _run_command_with_confirm(f"{install_cmd} {dependencies_str}", self.no_confirm) + return _run_command_with_confirm( + f"{install_cmd} {dependencies_str}", self.no_confirm + ) - def _get_installed_packages(self) -> Set[str]: - raise Exception("NotImplementedException") + def _get_installed_packages(self) -> set[str]: + raise RuntimeError("NotImplementedException") - def _filter_out_installed_packages(self, dependencies: Dict[str, Set[str]]): + def _filter_out_installed_packages(self, dependencies: dict[str, set[str]]): installed_packages = self._get_installed_packages() - filtered_packages = {k: (v - installed_packages) for k, v in dependencies.items()} + filtered_packages = { + k: (v - installed_packages) for k, v in dependencies.items() + } for installed_package in installed_packages: filtered_packages.pop(installed_package, None) return filtered_packages def run_cmd(self, cmd: str) -> bool: - result = subprocess.run(f"{cmd}", shell=True, text=True) + result = subprocess.run(f"{cmd}", shell=True, text=True, check=False) return result.returncode == 0 @@ -89,24 +100,30 @@ class BrewPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - return self._install(dependencies=dependencies, - install_cmd="brew install", - replace_dict={"patch": set()}) + def install(self, dependencies: dict[str, set[str]]) -> bool: + return self._install( + dependencies=dependencies, + install_cmd="brew install", + replace_dict={"patch": set()}, + ) def install_compiler(self) -> str: self.install({"compiler": set()}) return "" - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['brew', 'list'], text=True, capture_output=True, check=True) + def _get_installed_packages(self) -> set[str]: + result = subprocess.run( + ["brew", "list"], text=True, capture_output=True, check=True + ) lines = result.stdout.splitlines() - lines = [line.split('@', 1)[0] for line in lines] + lines = [line.split("@", 1)[0] for line in lines] return set(lines) def run_cmd(self, cmd: str) -> bool: add_m4_to_path_cmd = 'export PATH="$(brew --prefix m4)/bin:$PATH"' - result = subprocess.run(f"{add_m4_to_path_cmd} && {cmd}", shell=True, text=True) + result = subprocess.run( + f"{add_m4_to_path_cmd} && {cmd}", shell=True, text=True, check=False + ) return result.returncode == 0 @@ -114,16 +131,19 @@ class AptPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - return self._install(dependencies=dependencies, - install_cmd="sudo apt install -y", - replace_dict={"libarchive": {"liblzma-dev"}, - "python": {"libpython3-dev"}}) - - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['dpkg', '--get-selections'], text=True, capture_output=True, check=True) - lines = [line.split('\t')[0] for line in result.stdout.splitlines()] - lines = [line.rsplit(':', 1)[0] for line in lines] + def install(self, dependencies: dict[str, set[str]]) -> bool: + return self._install( + dependencies=dependencies, + install_cmd="sudo apt install -y", + replace_dict={"libarchive": {"liblzma-dev"}, "python": {"libpython3-dev"}}, + ) + + def _get_installed_packages(self) -> set[str]: + result = subprocess.run( + ["dpkg", "--get-selections"], text=True, capture_output=True, check=True + ) + lines = [line.split("\t")[0] for line in result.stdout.splitlines()] + lines = [line.rsplit(":", 1)[0] for line in lines] return set(lines) def install_compiler(self) -> str: @@ -136,19 +156,23 @@ def __init__(self, no_confirm, needs_epel): PackageManager.__init__(self, no_confirm) self.needs_epel = needs_epel - def install(self, dependencies: Dict[str, Set[str]]) -> bool: + def install(self, dependencies: dict[str, set[str]]) -> bool: if self.needs_epel: install_cmd = "sudo dnf --enablerepo=crb install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm" else: install_cmd = "sudo dnf install -y" - return self._install(dependencies=dependencies, - install_cmd=install_cmd, - replace_dict={"python": {"python3-devel"}}) - - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['dnf', 'list', 'installed'], text=True, capture_output=True, check=True) - lines = [line.split(' ')[0] for line in result.stdout.splitlines()] - lines = [line.rsplit('.', 1)[0] for line in lines] + return self._install( + dependencies=dependencies, + install_cmd=install_cmd, + replace_dict={"python": {"python3-devel"}}, + ) + + def _get_installed_packages(self) -> set[str]: + result = subprocess.run( + ["dnf", "list", "installed"], text=True, capture_output=True, check=True + ) + lines = [line.split(" ")[0] for line in result.stdout.splitlines()] + lines = [line.rsplit(".", 1)[0] for line in lines] return set(lines) def install_compiler(self) -> str: @@ -160,13 +184,17 @@ class PacmanPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - return self._install(dependencies=dependencies, - install_cmd="sudo pacman --noconfirm -S", - replace_dict={}) - - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['pacman', '-Qq'], text=True, capture_output=True, check=True) + def install(self, dependencies: dict[str, set[str]]) -> bool: + return self._install( + dependencies=dependencies, + install_cmd="sudo pacman --noconfirm -S", + replace_dict={}, + ) + + def _get_installed_packages(self) -> set[str]: + result = subprocess.run( + ["pacman", "-Qq"], text=True, capture_output=True, check=True + ) return set(result.stdout.splitlines()) def install_compiler(self) -> str: @@ -178,21 +206,30 @@ class ZypperPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - return self._install(dependencies=dependencies, - install_cmd="sudo zypper install -y", - replace_dict={"libarchive": {"libarchive-devel"}, - "python": {"python3-devel"}}) - - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['zypper', 'se', '--installed-only'], text=True, capture_output=True, check=True) + def install(self, dependencies: dict[str, set[str]]) -> bool: + return self._install( + dependencies=dependencies, + install_cmd="sudo zypper install -y", + replace_dict={ + "libarchive": {"libarchive-devel"}, + "python": {"python3-devel"}, + }, + ) + + def _get_installed_packages(self) -> set[str]: + result = subprocess.run( + ["zypper", "se", "--installed-only"], + text=True, + capture_output=True, + check=True, + ) lines = result.stdout.splitlines() packages = set() for line in lines: - if line.startswith('S |') or line.startswith('--') or line.startswith(' ') or not line: + if line.startswith(("S |", "--", " ")) or not line: continue - parts = line.split('|') + parts = line.split("|") if len(parts) > 2: package_name = parts[1].strip() packages.add(package_name) @@ -208,14 +245,18 @@ def _get_vs_dev_cmd_path(vs_where_location: VsWhereLocation): if vs_where_location == VsWhereLocation.CHOCO: vs_where_path = "vswhere" else: - vs_where_path = "%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe" + vs_where_path = ( + "%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe" + ) vswhere_results = subprocess.run( f"{vs_where_path} -products * " f"-property installationPath " f"-requires Microsoft.VisualStudio.Component.VC.ATL " f"-version 17", - capture_output=True) + capture_output=True, + check=False, + ) for vswhere_result in vswhere_results.stdout.splitlines(): possible_path = f"{vswhere_result.decode()}\\Common7\\Tools\\VsDevCmd.bat" @@ -251,7 +292,9 @@ def _minifi_setup_env_str(vs_where_location: VsWhereLocation) -> str: def _create_minifi_setup_env_batch(vs_where_location: VsWhereLocation): - with open(pathlib.Path(__file__).parent.resolve() / "build_environment.bat", "w") as f: + with open( + pathlib.Path(__file__).parent.resolve() / "build_environment.bat", "w" + ) as f: f.write(_minifi_setup_env_str(vs_where_location)) @@ -259,26 +302,32 @@ class ChocolateyPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - self._install(dependencies=dependencies, - install_cmd="choco install -y", - replace_dict={"python": set(), - "patch": set(), - "bison": set(), - "flex": set(), - "libarchive": set(), - "automake": set(), - "autoconf": set(), - "libtool": set(), - "make": set(), - "m4": {"gnuwin32-m4"}, - "perl": {"strawberryperl", "NASM"}}) + def install(self, dependencies: dict[str, set[str]]) -> bool: + self._install( + dependencies=dependencies, + install_cmd="choco install -y", + replace_dict={ + "python": set(), + "patch": set(), + "bison": set(), + "flex": set(), + "libarchive": set(), + "automake": set(), + "autoconf": set(), + "libtool": set(), + "make": set(), + "m4": {"gnuwin32-m4"}, + "perl": {"strawberryperl", "NASM"}, + }, + ) return True - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['choco', 'list'], text=True, capture_output=True, check=True) - lines = [line.split(' ')[0] for line in result.stdout.splitlines()] - lines = [line.rsplit('.', 1)[0] for line in lines] + def _get_installed_packages(self) -> set[str]: + result = subprocess.run( + ["choco", "list"], text=True, capture_output=True, check=True + ) + lines = [line.split(" ")[0] for line in result.stdout.splitlines()] + lines = [line.rsplit(".", 1)[0] for line in lines] if os.path.exists("C:\\Program Files\\NASM"): lines.append("NASM") # choco doesnt remember NASM return set(lines) @@ -287,7 +336,9 @@ def _acquire_vswhere(self): installed_packages = self._get_installed_packages() if "vswhere" in installed_packages: return VsWhereLocation.CHOCO - vswhere_default_path = "%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe" + vswhere_default_path = ( + "%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe" + ) if os.path.exists(vswhere_default_path): return VsWhereLocation.DEFAULT self.install({"vswhere": {"vswhere"}}) @@ -298,10 +349,17 @@ def install_compiler(self) -> str: vs_dev_path = _get_vs_dev_cmd_path(vs_where_loc) if not vs_dev_path: self.install( - {"visualstudio2022buildtools": {'visualstudio2022buildtools --package-parameters "--wait --quiet ' - '--add Microsoft.VisualStudio.Workload.VCTools ' - '--add Microsoft.VisualStudio.Component.VC.ATL ' - '--includeRecommended"'}}) + { + "visualstudio2022buildtools": { + ( + 'visualstudio2022buildtools --package-parameters "--wait --quiet ' + "--add Microsoft.VisualStudio.Workload.VCTools " + "--add Microsoft.VisualStudio.Component.VC.ATL " + '--includeRecommended"' + ) + } + } + ) return "" def run_cmd(self, cmd: str) -> bool: diff --git a/bootstrap/system_dependency.py b/bootstrap/system_dependency.py index cc97894f79..728efbca73 100644 --- a/bootstrap/system_dependency.py +++ b/bootstrap/system_dependency.py @@ -15,27 +15,36 @@ from __future__ import annotations -from typing import Dict, Set +import platform from minifi_option import MinifiOptions from package_manager import PackageManager -import platform -def _create_system_dependencies(minifi_options: MinifiOptions) -> Dict[str, Set[str]]: - system_dependencies = {'patch': {'patch'}, 'make': {'make'}, 'perl': {'perl'}, 'git': {'git'}, 'bison': {'bison'}, 'flex': {'flex'}, 'm4': {'m4'}} +def _create_system_dependencies(minifi_options: MinifiOptions) -> dict[str, set[str]]: + system_dependencies = { + "patch": {"patch"}, + "make": {"make"}, + "perl": {"perl"}, + "git": {"git"}, + "bison": {"bison"}, + "flex": {"flex"}, + "m4": {"m4"}, + } if minifi_options.is_enabled("ENABLE_LIBARCHIVE"): - system_dependencies['libarchive'] = {'libarchive'} + system_dependencies["libarchive"] = {"libarchive"} if minifi_options.is_enabled("ENABLE_SQL"): - system_dependencies['automake'] = {'automake'} - system_dependencies['autoconf'] = {'autoconf'} - system_dependencies['libtool'] = {'libtool'} + system_dependencies["automake"] = {"automake"} + system_dependencies["autoconf"] = {"autoconf"} + system_dependencies["libtool"] = {"libtool"} if minifi_options.is_enabled("ENABLE_PYTHON_SCRIPTING"): - system_dependencies['python'] = {'python'} + system_dependencies["python"] = {"python"} if platform.system() == "Windows": - system_dependencies['wixtoolset'] = {'wixtoolset'} + system_dependencies["wixtoolset"] = {"wixtoolset"} return system_dependencies -def install_required(minifi_options: MinifiOptions, package_manager: PackageManager) -> bool: +def install_required( + minifi_options: MinifiOptions, package_manager: PackageManager +) -> bool: return package_manager.install(_create_system_dependencies(minifi_options)) diff --git a/cmake/GenericPython.cmake b/cmake/GenericPython.cmake index a815919030..6843e3b5d0 100644 --- a/cmake/GenericPython.cmake +++ b/cmake/GenericPython.cmake @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -find_package(Python 3.6 REQUIRED COMPONENTS Development Interpreter) +find_package(Python 3.10 REQUIRED COMPONENTS Development Interpreter) if(WIN32) set(Python_LIBRARIES ${Python_LIBRARY_DIRS}/python3.lib) diff --git a/conanfile.py b/conanfile.py index a1047b2f64..289e3ae839 100644 --- a/conanfile.py +++ b/conanfile.py @@ -1,20 +1,79 @@ +import os +import shutil +from typing import ClassVar + from conan import ConanFile +from conan.errors import ConanException from conan.tools.cmake import CMake, CMakeToolchain from conan.tools.files import collect_libs, copy -from conan.errors import ConanException -import os -import shutil required_conan_version = ">=2.0" -shared_requires = ("lz4/1.10.0", "openssl/3.6.2", "libcurl/8.20.0", "civetweb/1.16", "libxml2/2.15.3", "fmt/12.1.0", "spdlog/1.17.0", "catch2/3.14.0", "zlib/1.3.2", "zstd/1.5.7", - "bzip2/1.0.8", "rocksdb/11.1.1@minifi/develop", "libarchive/3.8.7", "lua/5.4.6", "sol2/3.5.0") - -shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "behave_framework/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "core-framework/*", - "docs/*", "encrypt-config/*", "etc/*", "examples/*", "extension-framework/*", "fips/*", "minifi-api/*", "packaging/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", - "README.md", "C2.md", "CONAN.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "METRICS.md", "OPS.md", "PARAMETER_PROVIDERS.md", - "PROCESSORS.md", "SITE_TO_SITE.md", "ThirdParties.md", "Windows.md", "CPPLINT.cfg", "generateVersion.bat", "generateVersion.sh", "run_clang_tidy.sh", "run_flake8.sh", - "run_shellcheck.sh", "versioninfo.rc.in") +shared_requires = ( + "lz4/1.10.0", + "openssl/3.6.2", + "libcurl/8.20.0", + "civetweb/1.16", + "libxml2/2.15.3", + "fmt/12.1.0", + "spdlog/1.17.0", + "catch2/3.14.0", + "zlib/1.3.2", + "zstd/1.5.7", + "bzip2/1.0.8", + "rocksdb/11.1.1@minifi/develop", + "libarchive/3.8.7", + "lua/5.4.6", + "sol2/3.5.0", +) + +shared_sources = ( + "CMakeLists.txt", + "libminifi/*", + "extensions/*", + "minifi_main/*", + "behave_framework/*", + "bin/*", + "bootstrap/*", + "cmake/*", + "conf/*", + "controller/*", + "core-framework/*", + "docs/*", + "encrypt-config/*", + "etc/*", + "examples/*", + "extension-framework/*", + "fips/*", + "minifi-api/*", + "packaging/*", + "thirdparty/*", + "docker/*", + "LICENSE", + "NOTICE", + "README.md", + "C2.md", + "CONAN.md", + "CONFIGURE.md", + "CONTRIBUTING.md", + "CONTROLLERS.md", + "EXPRESSIONS.md", + "Extensions.md", + "METRICS.md", + "OPS.md", + "PARAMETER_PROVIDERS.md", + "PROCESSORS.md", + "SITE_TO_SITE.md", + "ThirdParties.md", + "Windows.md", + "CPPLINT.cfg", + "generateVersion.bat", + "generateVersion.sh", + "run_clang_tidy.sh", + "ruff.toml", + "run_shellcheck.sh", + "versioninfo.rc.in", +) class MiNiFiCppMain(ConanFile): @@ -24,9 +83,9 @@ class MiNiFiCppMain(ConanFile): requires = shared_requires settings = "os", "compiler", "build_type", "arch" generators = "CMakeDeps" - options = {"shared": [True, False], "fPIC": [True, False]} + options: ClassVar[dict] = {"shared": [True, False], "fPIC": [True, False]} - default_options = {"shared": False, "fPIC": True} + default_options: ClassVar[dict] = {"shared": False, "fPIC": True} exports_sources = shared_sources @@ -57,7 +116,7 @@ def build(self): cmake.build() def overwrite_libfile(self, oldfile, newfile): - print("Copying {} to {}".format(oldfile, newfile)) + print(f"Copying {oldfile} to {newfile}") if os.path.exists(oldfile): try: if os.path.exists(newfile): @@ -74,17 +133,47 @@ def package(self): cmake.install() include_dir = os.path.join(self.source_folder) built_dir = os.path.join(self.source_folder, self.folders.build) - copy(self, pattern="*.h*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) - copy(self, pattern="*.i*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) - copy(self, pattern="*.a", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) - copy(self, pattern="*.so*", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) - - minifi_py_ext_oldfile = os.path.join(self.package_folder, "lib", "libminifi-python-script-extension.so") - minifi_py_ext_copynewfile = os.path.join(self.package_folder, "lib", "libminifi_native.so") + copy( + self, + pattern="*.h*", + dst=os.path.join(self.package_folder, "include"), + src=include_dir, + keep_path=True, + ) + copy( + self, + pattern="*.i*", + dst=os.path.join(self.package_folder, "include"), + src=include_dir, + keep_path=True, + ) + copy( + self, + pattern="*.a", + dst=os.path.join(self.package_folder, "lib"), + src=built_dir, + keep_path=False, + ) + copy( + self, + pattern="*.so*", + dst=os.path.join(self.package_folder, "lib"), + src=built_dir, + keep_path=False, + ) + + minifi_py_ext_oldfile = os.path.join( + self.package_folder, "lib", "libminifi-python-script-extension.so" + ) + minifi_py_ext_copynewfile = os.path.join( + self.package_folder, "lib", "libminifi_native.so" + ) self.overwrite_libfile(minifi_py_ext_oldfile, minifi_py_ext_copynewfile) def package_info(self): - self.cpp_info.libs = collect_libs(self, folder=os.path.join(self.package_folder, "lib")) + self.cpp_info.libs = collect_libs( + self, folder=os.path.join(self.package_folder, "lib") + ) self.cpp_info.set_property("cmake_file_name", "minifi-cpp") self.cpp_info.set_property("cmake_target_name", "minifi-cpp::minifi-cpp") self.cpp_info.set_property("pkg_config_name", "minifi-cpp") diff --git a/docker/rockylinux/Dockerfile b/docker/rockylinux/Dockerfile index 019bfe4653..b6c4bdd61b 100644 --- a/docker/rockylinux/Dockerfile +++ b/docker/rockylinux/Dockerfile @@ -43,7 +43,7 @@ COPY . ${MINIFI_BASE_DIR} # ccache is in EPEL RUN dnf -y install epel-release && dnf -y install gcc-toolset-14 gcc-toolset-14-libatomic-devel sudo git which make libarchive ccache ca-certificates perl patch bison flex libtool cmake rpmdevtools && \ if echo "$MINIFI_OPTIONS" | grep -q "MINIFI_RUST=ON"; then dnf -y install rust cargo clang; fi && \ - if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON\|ENABLE_PYTHON_SCRIPTING=ON\|ENABLE_OPC=ON"; then dnf -y --enablerepo=devel install python3-devel; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON\|ENABLE_PYTHON_SCRIPTING=ON\|ENABLE_OPC=ON"; then dnf -y --enablerepo=devel install python3.12-devel; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SFTP=ON" && [ "${DOCKER_SKIP_TESTS}" == "OFF" ]; then dnf -y install java-1.8.0-openjdk maven; fi RUN cd $MINIFI_BASE_DIR && \ @@ -69,4 +69,3 @@ RUN cd $MINIFI_BASE_DIR && \ make -j "$(nproc)" package && \ ../packaging/rpm/check_rpm_contents.sh *.rpm expected-rpm-contents.txt; \ fi - diff --git a/examples/scripts/python/reverse_flow_file_content.py b/examples/scripts/python/reverse_flow_file_content.py index 66e76c0ba9..da9a87c55b 100644 --- a/examples/scripts/python/reverse_flow_file_content.py +++ b/examples/scripts/python/reverse_flow_file_content.py @@ -21,7 +21,7 @@ class ReadCallback: def process(self, input_stream): - self.content = codecs.getreader('utf-8')(input_stream).read() + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) @@ -31,7 +31,7 @@ def __init__(self, content): def process(self, output_stream): reversed_content = self.content[::-1] - output_stream.write(reversed_content.encode('utf-8')) + output_stream.write(reversed_content.encode("utf-8")) return len(reversed_content) @@ -41,5 +41,5 @@ def onTrigger(context, session): read_callback = ReadCallback() session.read(flow_file, read_callback) session.write(flow_file, WriteReverseStringCallback(read_callback.content)) - flow_file.addAttribute('python_timestamp', str(int(time.time()))) + flow_file.addAttribute("python_timestamp", str(int(time.time()))) session.transfer(flow_file, REL_SUCCESS) diff --git a/extensions/aws/tests/features/containers/kinesis_server_container.py b/extensions/aws/tests/features/containers/kinesis_server_container.py index f7767773c1..0d5484c2f8 100644 --- a/extensions/aws/tests/features/containers/kinesis_server_container.py +++ b/extensions/aws/tests/features/containers/kinesis_server_container.py @@ -14,23 +14,31 @@ # limitations under the License. import logging - from pathlib import Path + from minifi_behave.containers.container_linux import LinuxContainer -from minifi_behave.core.helpers import wait_for_condition, retry_check -from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.containers.docker_image_builder import DockerImageBuilder +from minifi_behave.core.helpers import retry_check, wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext + +logger = logging.getLogger(__name__) class KinesisServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): builder = DockerImageBuilder( image_tag="minifi-kinesis-mock:latest", - build_context_path=str(Path(__file__).resolve().parent.parent / "resources" / "kinesis-mock") + build_context_path=str( + Path(__file__).resolve().parent.parent / "resources" / "kinesis-mock" + ), ) builder.build() - super().__init__("minifi-kinesis-mock:latest", f"kinesis-server-{test_context.scenario_id}", test_context.network) + super().__init__( + "minifi-kinesis-mock:latest", + f"kinesis-server-{test_context.scenario_id}", + test_context.network, + ) self.environment.append("INITIALIZE_STREAMS=test_stream:3") self.environment.append("LOG_LEVEL=DEBUG") @@ -41,10 +49,13 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=300, bail_condition=lambda: self.exited, - context=context) + context=context, + ) @retry_check() def check_kinesis_server_record_data(self, record_data): - (code, output) = self.exec_run(["node", "/app/consumer/consumer.js", record_data]) - logging.info(f"Kinesis server returned output: '{output}' with code '{code}'") + (code, output) = self.exec_run( + ["node", "/app/consumer/consumer.js", record_data] + ) + logger.info(f"Kinesis server returned output: '{output}' with code '{code}'") return code == 0 diff --git a/extensions/aws/tests/features/containers/s3_server_container.py b/extensions/aws/tests/features/containers/s3_server_container.py index b2b76ffb12..b1b82ade3f 100644 --- a/extensions/aws/tests/features/containers/s3_server_container.py +++ b/extensions/aws/tests/features/containers/s3_server_container.py @@ -20,12 +20,18 @@ from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class S3ServerContainer(LinuxContainer): IMAGE = "adobe/s3mock:3.12.0" def __init__(self, test_context: MinifiTestContext): - super().__init__(S3ServerContainer.IMAGE, f"s3-server-{test_context.scenario_id}", test_context.network) + super().__init__( + S3ServerContainer.IMAGE, + f"s3-server-{test_context.scenario_id}", + test_context.network, + ) self.environment.append("initialBuckets=test_bucket") def deploy(self, context: MinifiTestContext | None) -> bool: @@ -35,10 +41,22 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context) + context=context, + ) def check_s3_server_object_data(self, test_data): - (code, output) = self.exec_run(["find", "/s3mockroot/test_bucket", "-mindepth", "1", "-maxdepth", "1", "-type", "d"]) + (code, output) = self.exec_run( + [ + "find", + "/s3mockroot/test_bucket", + "-mindepth", + "1", + "-maxdepth", + "1", + "-type", + "d", + ] + ) if code != 0: return False s3_mock_dir = output.strip() @@ -46,7 +64,18 @@ def check_s3_server_object_data(self, test_data): return code == 0 and file_data == test_data def check_s3_server_object_hash(self, expected_file_hash: str): - (code, output) = self.exec_run(["find", "/s3mockroot/test_bucket", "-mindepth", "1", "-maxdepth", "1", "-type", "d"]) + (code, output) = self.exec_run( + [ + "find", + "/s3mockroot/test_bucket", + "-mindepth", + "1", + "-maxdepth", + "1", + "-type", + "d", + ] + ) if code != 0: return False dir_candidates = output.split("\n") @@ -57,19 +86,49 @@ def check_s3_server_object_hash(self, expected_file_hash: str): (code, md5_output) = self.exec_run(["md5sum", s3_mock_dir + "/binaryData"]) if code != 0: return False - file_hash = md5_output.split(' ')[0].strip() + file_hash = md5_output.split(" ")[0].strip() return file_hash == expected_file_hash - def check_s3_server_object_metadata(self, content_type="application/octet-stream", metadata=dict()): - (code, output) = self.exec_run(["find", "/s3mockroot/test_bucket", "-mindepth", "1", "-maxdepth", "1", "-type", "d"]) + def check_s3_server_object_metadata( + self, content_type="application/octet-stream", metadata=None + ): + if metadata is None: + metadata = {} + (code, output) = self.exec_run( + [ + "find", + "/s3mockroot/test_bucket", + "-mindepth", + "1", + "-maxdepth", + "1", + "-type", + "d", + ] + ) if code != 0: return False s3_mock_dir = output.strip() (code, output) = self.exec_run(["cat", s3_mock_dir + "/objectMetadata.json"]) server_metadata = json.loads(output) - return code == 0 and server_metadata["contentType"] == content_type and metadata == server_metadata["userMetadata"] + return ( + code == 0 + and server_metadata["contentType"] == content_type + and metadata == server_metadata["userMetadata"] + ) def is_s3_bucket_empty(self): - (code, output) = self.exec_run(["find", "/s3mockroot/test_bucket", "-mindepth", "1", "-maxdepth", "1", "-type", "d"]) - logging.info(f"is_s3_bucket_empty: {output}") + (code, output) = self.exec_run( + [ + "find", + "/s3mockroot/test_bucket", + "-mindepth", + "1", + "-maxdepth", + "1", + "-type", + "d", + ] + ) + logger.info(f"is_s3_bucket_empty: {output}") return code == 0 and not output.strip() diff --git a/extensions/aws/tests/features/environment.py b/extensions/aws/tests/features/environment.py index 08e04da68f..b68ea0e23f 100644 --- a/extensions/aws/tests/features/environment.py +++ b/extensions/aws/tests/features/environment.py @@ -13,15 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario from containers.s3_server_container import S3ServerContainer +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario +import docker # These hooks are executed by behave before and after each scenario # The common_before_scenario and common_after_scenario must be called for proper setup and tear down + def before_feature(context, feature): if feature.filename.endswith("s3.feature"): docker.from_env().images.pull(S3ServerContainer.IMAGE) diff --git a/extensions/aws/tests/features/steps/steps.py b/extensions/aws/tests/features/steps/steps.py index ab10940464..2a223d76c2 100644 --- a/extensions/aws/tests/features/steps/steps.py +++ b/extensions/aws/tests/features/steps/steps.py @@ -19,38 +19,42 @@ import humanfriendly from behave import step, then - +from containers.kinesis_server_container import KinesisServerContainer +from containers.s3_server_container import S3ServerContainer from minifi_behave.containers.directory import Directory -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from minifi_behave.core.helpers import log_due_to_failure, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.minifi.processor import Processor -from minifi_behave.core.helpers import wait_for_condition, log_due_to_failure +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) -from containers.s3_server_container import S3ServerContainer -from containers.kinesis_server_container import KinesisServerContainer - -@step('a {processor_name} processor set up to communicate with an s3 server') -@step('a {processor_name} processor set up to communicate with the same s3 server') +@step("a {processor_name} processor set up to communicate with an s3 server") +@step("a {processor_name} processor set up to communicate with the same s3 server") def setup_s3_processor(context: MinifiTestContext, processor_name: str): processor = Processor(processor_name, processor_name) - processor.add_property('Object Key', 'test_object_key') - processor.add_property('Bucket', 'test_bucket') - processor.add_property('Access Key', 'test_access_key') - processor.add_property('Secret Key', 'test_secret') - processor.add_property('Endpoint Override URL', f"http://s3-server-{context.scenario_id}:9090") - processor.add_property('Proxy Host', '') - processor.add_property('Proxy Port', '') - processor.add_property('Proxy Username', '') - processor.add_property('Proxy Password', '') - - context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) - - -@step('the s3 server starts up') + processor.add_property("Object Key", "test_object_key") + processor.add_property("Bucket", "test_bucket") + processor.add_property("Access Key", "test_access_key") + processor.add_property("Secret Key", "test_secret") + processor.add_property( + "Endpoint Override URL", f"http://s3-server-{context.scenario_id}:9090" + ) + processor.add_property("Proxy Host", "") + processor.add_property("Proxy Port", "") + processor.add_property("Proxy Username", "") + processor.add_property("Proxy Password", "") + + context.get_or_create_default_minifi_container().flow_definition.add_processor( + processor + ) + + +@step("the s3 server starts up") def start_s3_server(context: MinifiTestContext): context.containers["s3-server"] = S3ServerContainer(context) assert context.containers["s3-server"].deploy(context) @@ -63,8 +67,12 @@ def verify_s3_object_data(context: MinifiTestContext, object_data: str): assert s3_server_container.check_s3_server_object_data(object_data) -@step('the object content type on the s3 server is "{content_type}" and the object metadata matches use metadata') -def verify_s3_object_content_type_and_metadata(context: MinifiTestContext, content_type: str): +@step( + 'the object content type on the s3 server is "{content_type}" and the object metadata matches use metadata' +) +def verify_s3_object_content_type_and_metadata( + context: MinifiTestContext, content_type: str +): s3_server_container = context.containers["s3-server"] assert isinstance(s3_server_container, S3ServerContainer) assert s3_server_container.check_s3_server_object_metadata(content_type) @@ -76,7 +84,10 @@ def verify_s3_bucket_empty(context): assert isinstance(s3_server_container, S3ServerContainer) assert wait_for_condition( condition=lambda: s3_server_container.is_s3_bucket_empty(), - timeout_seconds=10, bail_condition=lambda: s3_server_container.exited, context=context) + timeout_seconds=10, + bail_condition=lambda: s3_server_container.exited, + context=context, + ) @step("the object on the s3 server is present and matches the original hash") @@ -89,14 +100,18 @@ def verify_s3_object_hash_matches(context): def computeMD5hash(my_string): m = hashlib.md5() - m.update(my_string.encode('utf-8')) + m.update(my_string.encode("utf-8")) return m.hexdigest() -@step('there is a 6MB file at the "/tmp/input" directory and we keep track of the hash of that') +@step( + 'there is a 6MB file at the "/tmp/input" directory and we keep track of the hash of that' +) def create_6mb_file_and_track_hash(context): size = humanfriendly.parse_size("6MB") - content = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) + content = "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(size) + ) new_dir = Directory("/tmp/input") new_dir.files["input.txt"] = content context.get_or_create_default_minifi_container().dirs.append(new_dir) @@ -108,6 +123,8 @@ def setup_kinesis_server(context): context.containers["kinesis-server"] = KinesisServerContainer(context) -@then("there is a record on the kinesis server with \"{record_data}\"") +@then('there is a record on the kinesis server with "{record_data}"') def verify_kinesis_record_data(context, record_data): - assert context.containers["kinesis-server"].check_kinesis_server_record_data(record_data) or log_due_to_failure(context) + assert context.containers["kinesis-server"].check_kinesis_server_record_data( + record_data + ) or log_due_to_failure(context) diff --git a/extensions/azure/tests/features/containers/azure_server_container.py b/extensions/azure/tests/features/containers/azure_server_container.py index 7d99ae912e..aafca38469 100644 --- a/extensions/azure/tests/features/containers/azure_server_container.py +++ b/extensions/azure/tests/features/containers/azure_server_container.py @@ -19,30 +19,48 @@ from docker.errors import ContainerError from minifi_behave.containers.container_linux import LinuxContainer -from minifi_behave.core.helpers import run_cmd_in_docker_image -from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.helpers import run_cmd_in_docker_image, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class AzureServerContainer(LinuxContainer): IMAGE = "mcr.microsoft.com/azure-storage/azurite:3.35.0" def __init__(self, test_context: MinifiTestContext): - super().__init__(AzureServerContainer.IMAGE, - f"azure-storage-server-{test_context.scenario_id}", - test_context.network) + super().__init__( + AzureServerContainer.IMAGE, + f"azure-storage-server-{test_context.scenario_id}", + test_context.network, + ) azure_storage_hostname = f"azure-storage-server-{test_context.scenario_id}" - self.azure_connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" \ - f"BlobEndpoint=http://{azure_storage_hostname}:10000/devstoreaccount1;QueueEndpoint=http://{azure_storage_hostname}:10001/devstoreaccount1;" - self.command = ["azurite", "-l", "/data", "--blobHost", "0.0.0.0", "--queueHost", "0.0.0.0", "--tableHost", "0.0.0.0", "--skipApiVersionCheck"] + self.azure_connection_string = ( + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + f"BlobEndpoint=http://{azure_storage_hostname}:10000/devstoreaccount1;QueueEndpoint=http://{azure_storage_hostname}:10001/devstoreaccount1;" + ) + self.command = [ + "azurite", + "-l", + "/data", + "--blobHost", + "0.0.0.0", + "--queueHost", + "0.0.0.0", + "--tableHost", + "0.0.0.0", + "--skipApiVersionCheck", + ] def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) finished_str = "Azurite Queue service is successfully listening at" - return wait_for_condition(condition=lambda: finished_str in self.get_logs(), - timeout_seconds=15, - bail_condition=lambda: self.exited, - context=context) + return wait_for_condition( + condition=lambda: finished_str in self.get_logs(), + timeout_seconds=15, + bail_condition=lambda: self.exited, + context=context, + ) def check_azure_storage_server_data(self, test_data): (code, output) = self.exec_run(["find", "/data/__blobstorage__", "-type", "f"]) @@ -52,49 +70,93 @@ def check_azure_storage_server_data(self, test_data): (code, file_data) = self.exec_run(["cat", data_file]) return code == 0 and test_data in file_data - def add_test_blob(self, blob_name, content="test_data", with_snapshot=False) -> bool: - cmd_create = ["az", "storage", "container", "create", "--name", "test-container", "--connection-string", - self.azure_connection_string] + def add_test_blob( + self, blob_name, content="test_data", with_snapshot=False + ) -> bool: + cmd_create = [ + "az", + "storage", + "container", + "create", + "--name", + "test-container", + "--connection-string", + self.azure_connection_string, + ] try: - run_cmd_in_docker_image("mcr.microsoft.com/azure-cli:2.81.0", cmd_create, self.network.name) + run_cmd_in_docker_image( + "mcr.microsoft.com/azure-cli:2.81.0", cmd_create, self.network.name + ) except ContainerError as e: - logging.error(e) + logger.error(e) return False - cmd_upload = ["az", "storage", "blob", "upload", "--container-name", "test-container", "--name", blob_name, - "--data", content, "--connection-string", self.azure_connection_string] + cmd_upload = [ + "az", + "storage", + "blob", + "upload", + "--container-name", + "test-container", + "--name", + blob_name, + "--data", + content, + "--connection-string", + self.azure_connection_string, + ] try: - run_cmd_in_docker_image("mcr.microsoft.com/azure-cli:2.81.0", cmd_upload, self.network.name) + run_cmd_in_docker_image( + "mcr.microsoft.com/azure-cli:2.81.0", cmd_upload, self.network.name + ) except ContainerError as e: - logging.error(e) + logger.error(e) return False if with_snapshot: - cmd_snapshot = ["az", "storage", "blob", "snapshot", "--container-name", "test-container", "--name", - blob_name, "--connection-string", self.azure_connection_string] + cmd_snapshot = [ + "az", + "storage", + "blob", + "snapshot", + "--container-name", + "test-container", + "--name", + blob_name, + "--connection-string", + self.azure_connection_string, + ] try: - run_cmd_in_docker_image("mcr.microsoft.com/azure-cli:2.81.0", cmd_snapshot, self.network.name) + run_cmd_in_docker_image( + "mcr.microsoft.com/azure-cli:2.81.0", + cmd_snapshot, + self.network.name, + ) except ContainerError as e: - logging.error(e) + logger.error(e) return False return True def __get_blob_and_snapshot_count(self) -> int: - cmd = (f'az storage blob list --container-name "test-container" ' - f'--include deleted --query "length(@)" --output tsv ' - f'--connection-string "{self.azure_connection_string}"') + cmd = ( + f'az storage blob list --container-name "test-container" ' + f'--include deleted --query "length(@)" --output tsv ' + f'--connection-string "{self.azure_connection_string}"' + ) try: - output = run_cmd_in_docker_image("mcr.microsoft.com/azure-cli:2.81.0", cmd, self.network.name) + output = run_cmd_in_docker_image( + "mcr.microsoft.com/azure-cli:2.81.0", cmd, self.network.name + ) except ContainerError as e: - logging.error(e) + logger.error(e) return -1 try: return int(output.strip()) except (ValueError, TypeError): - logging.error(f"{output} Not an int") + logger.error(f"{output} Not an int") return -1 def check_azure_blob_and_snapshot_count(self, blob_and_snapshot_count) -> bool: diff --git a/extensions/azure/tests/features/environment.py b/extensions/azure/tests/features/environment.py index 4684766146..64e98934e9 100644 --- a/extensions/azure/tests/features/environment.py +++ b/extensions/azure/tests/features/environment.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario from containers.azure_server_container import AzureServerContainer +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker # These hooks are executed by behave before and after each scenario # The common_before_scenario and common_after_scenario must be called for proper setup and tear down diff --git a/extensions/azure/tests/features/steps/steps.py b/extensions/azure/tests/features/steps/steps.py index 25409d4275..da568d3edb 100644 --- a/extensions/azure/tests/features/steps/steps.py +++ b/extensions/azure/tests/features/steps/steps.py @@ -17,27 +17,32 @@ import humanfriendly from behave import step, then +from containers.azure_server_container import AzureServerContainer from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.minifi.processor import Processor -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 - -from containers.azure_server_container import AzureServerContainer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @step("a {processor_name} processor set up to communicate with an Azure blob storage") def setup_azure_blob_storage_processor(context: MinifiTestContext, processor_name: str): processor = Processor(processor_name, processor_name) hostname = f"http://azure-storage-server-{context.scenario_id}" - processor.add_property('Container Name', 'test-container') - processor.add_property('Connection String', - f'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={hostname}:10000/devstoreaccount1;QueueEndpoint={hostname}:10001/devstoreaccount1;') - processor.add_property('Blob', 'test-blob') - processor.add_property('Create Container', 'true') - context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) + processor.add_property("Container Name", "test-container") + processor.add_property( + "Connection String", + f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={hostname}:10000/devstoreaccount1;QueueEndpoint={hostname}:10001/devstoreaccount1;", + ) + processor.add_property("Blob", "test-blob") + processor.add_property("Create Container", "true") + context.get_or_create_default_minifi_container().flow_definition.add_processor( + processor + ) @step("an Azure storage server is set up") @@ -53,8 +58,12 @@ def verify_azure_storage_server_data(context: MinifiTestContext, object_data: st assert azure_server_container.check_azure_storage_server_data(object_data) -@step('test blob "{blob_name}" with the content "{data}" is created on Azure blob storage') -def create_test_blob_with_content(context: MinifiTestContext, blob_name: str, data: str): +@step( + 'test blob "{blob_name}" with the content "{data}" is created on Azure blob storage' +) +def create_test_blob_with_content( + context: MinifiTestContext, blob_name: str, data: str +): azure_server_container = context.containers["azure-storage-server"] assert isinstance(azure_server_container, AzureServerContainer) assert azure_server_container.add_test_blob(blob_name, content=data) @@ -83,7 +92,8 @@ def verify_azure_blob_storage_is_empty(context: MinifiTestContext, timeout_str: condition=lambda: azure_server_container.check_azure_blob_storage_is_empty(), timeout_seconds=timeout_in_seconds, bail_condition=lambda: azure_server_container.exited, - context=context) + context=context, + ) @then("the blob and snapshot count becomes 1 in {timeout_str}") @@ -95,4 +105,5 @@ def verify_blob_and_snapshot_count(context: MinifiTestContext, timeout_str: str) condition=lambda: azure_server_container.check_azure_blob_and_snapshot_count(1), timeout_seconds=timeout_in_seconds, bail_condition=lambda: azure_server_container.exited, - context=context) + context=context, + ) diff --git a/extensions/civetweb/tests/features/environment.py b/extensions/civetweb/tests/features/environment.py index 5b3659f585..5c865091c3 100644 --- a/extensions/civetweb/tests/features/environment.py +++ b/extensions/civetweb/tests/features/environment.py @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario def before_scenario(context, scenario): diff --git a/extensions/civetweb/tests/features/steps/steps.py b/extensions/civetweb/tests/features/steps/steps.py index 25759880fa..ff5817598b 100644 --- a/extensions/civetweb/tests/features/steps/steps.py +++ b/extensions/civetweb/tests/features/steps/steps.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) diff --git a/extensions/couchbase/tests/features/containers/couchbase_server_container.py b/extensions/couchbase/tests/features/containers/couchbase_server_container.py index 41c8a89eed..5a03f76630 100644 --- a/extensions/couchbase/tests/features/containers/couchbase_server_container.py +++ b/extensions/couchbase/tests/features/containers/couchbase_server_container.py @@ -15,28 +15,54 @@ import logging -from minifi_behave.core.helpers import wait_for_condition, retry_check -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key +from docker.errors import ContainerError from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.file import File +from minifi_behave.core.helpers import retry_check, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from docker.errors import ContainerError +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert + +logger = logging.getLogger(__name__) class CouchbaseServerContainer(LinuxContainer): IMAGE = "couchbase:enterprise-7.2.5" def __init__(self, test_context: MinifiTestContext): - super().__init__(CouchbaseServerContainer.IMAGE, f"couchbase-server-{test_context.scenario_id}", test_context.network) + super().__init__( + CouchbaseServerContainer.IMAGE, + f"couchbase-server-{test_context.scenario_id}", + test_context.network, + ) - couchbase_cert, couchbase_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + couchbase_cert, couchbase_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", root_ca_content, permissions=0o666)) + self.files.append( + File( + "/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", + root_ca_content, + permissions=0o666, + ) + ) couchbase_cert_content = dump_cert(couchbase_cert) - self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem", couchbase_cert_content, permissions=0o666)) + self.files.append( + File( + "/opt/couchbase/var/lib/couchbase/inbox/chain.pem", + couchbase_cert_content, + permissions=0o666, + ) + ) couchbase_key_content = dump_key(couchbase_key) - self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666)) + self.files.append( + File( + "/opt/couchbase/var/lib/couchbase/inbox/pkey.key", + couchbase_key_content, + permissions=0o666, + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -45,33 +71,95 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context) + context=context, + ) return self.run_post_startup_commands() def run_post_startup_commands(self): commands = [ - ["couchbase-cli", "cluster-init", "-c", "localhost", "--cluster-username", "Administrator", "--cluster-password", "password123", "--services", "data,index,query", - "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"], - ["couchbase-cli", "bucket-create", "-c", "localhost", "--username", "Administrator", "--password", "password123", "--bucket", "test_bucket", "--bucket-type", "couchbase", - "--bucket-ramsize", "1024", "--max-ttl", "36000"], - ["couchbase-cli", "user-manage", "-c", "localhost", "-u", "Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser", "--rbac-password", "password123", - "--roles", "data_reader[test_bucket],data_writer[test_bucket]", "--auth-domain", "local"], - ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable", "prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''], - ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u', 'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json'] + [ + "couchbase-cli", + "cluster-init", + "-c", + "localhost", + "--cluster-username", + "Administrator", + "--cluster-password", + "password123", + "--services", + "data,index,query", + "--cluster-ramsize", + "2048", + "--cluster-index-ramsize", + "256", + ], + [ + "couchbase-cli", + "bucket-create", + "-c", + "localhost", + "--username", + "Administrator", + "--password", + "password123", + "--bucket", + "test_bucket", + "--bucket-type", + "couchbase", + "--bucket-ramsize", + "1024", + "--max-ttl", + "36000", + ], + [ + "couchbase-cli", + "user-manage", + "-c", + "localhost", + "-u", + "Administrator", + "-p", + "password123", + "--set", + "--rbac-username", + "clientuser", + "--rbac-password", + "password123", + "--roles", + "data_reader[test_bucket],data_writer[test_bucket]", + "--auth-domain", + "local", + ], + [ + "bash", + "-c", + 'tee /tmp/auth.json <<< \'{"state": "enable", "prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\'', + ], + [ + "couchbase-cli", + "ssl-manage", + "-c", + "localhost", + "-u", + "Administrator", + "-p", + "password123", + "--set-client-auth", + "/tmp/auth.json", + ], ] if not self._run_couchbase_cli_commands(commands): return False - if not self._load_couchbase_certs(): - return False - - return True + return self._load_couchbase_certs() @retry_check(max_tries=12, retry_interval_seconds=5) def _run_couchbase_cli_command(self, command): (code, output) = self.exec_run(command) if code != 0: - logging.error(f"Failed to run command '{command}', returned error code: {code}, output: '{output}'") + logger.error( + f"Failed to run command '{command}', returned error code: {code}, output: '{output}'" + ) return False return True @@ -80,15 +168,34 @@ def _run_couchbase_cli_commands(self, commands): def _run_python_in_couchbase_helper_docker(self, command: str): try: - self.client.containers.run("minifi-couchbase-helper:latest", ["python", "-c", command], remove=True, stdout=True, stderr=True, network=self.network.name) + self.client.containers.run( + "minifi-couchbase-helper:latest", + ["python", "-c", command], + remove=True, + stdout=True, + stderr=True, + network=self.network.name, + ) return True except ContainerError as e: - stdout = e.stdout.decode("utf-8", errors="replace") if hasattr(e, "stdout") and e.stdout else "" - stderr = e.stderr.decode("utf-8", errors="replace") if hasattr(e, "stderr") and e.stderr else "" - logging.error(f"Python command '{command}' failed in couchbase helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'") + stdout = ( + e.stdout.decode("utf-8", errors="replace") + if hasattr(e, "stdout") and e.stdout + else "" + ) + stderr = ( + e.stderr.decode("utf-8", errors="replace") + if hasattr(e, "stderr") and e.stderr + else "" + ) + logger.error( + f"Python command '{command}' failed in couchbase helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'" + ) return False except Exception as e: - logging.error(f"Unexpected error while running python command '{command}' in couchbase helper docker: '{e}'") + logger.error( + f"Unexpected error while running python command '{command}' in couchbase helper docker: '{e}'" + ) return False @retry_check(max_tries=15, retry_interval_seconds=2) @@ -108,7 +215,9 @@ def _load_couchbase_certs(self): """ return self._run_python_in_couchbase_helper_docker(python_command) - def is_data_present_in_couchbase(self, doc_id: str, bucket_name: str, expected_data: str, expected_data_type: str): + def is_data_present_in_couchbase( + self, doc_id: str, bucket_name: str, expected_data: str, expected_data_type: str + ): python_command = f""" from couchbase.cluster import Cluster from couchbase.options import ClusterOptions diff --git a/extensions/couchbase/tests/features/environment.py b/extensions/couchbase/tests/features/environment.py index d48dc49e15..7e4efa22a5 100644 --- a/extensions/couchbase/tests/features/environment.py +++ b/extensions/couchbase/tests/features/environment.py @@ -13,11 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker -from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario from containers.couchbase_server_container import CouchbaseServerContainer +from minifi_behave.containers.docker_image_builder import DockerImageBuilder +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): @@ -25,8 +25,7 @@ def before_all(context): FROM python:3.13-slim-bookworm RUN pip install couchbase==4.3.5 requests""" builder = DockerImageBuilder( - image_tag="minifi-couchbase-helper:latest", - dockerfile_content=dockerfile + image_tag="minifi-couchbase-helper:latest", dockerfile_content=dockerfile ) builder.build() docker.from_env().images.pull(CouchbaseServerContainer.IMAGE) diff --git a/extensions/couchbase/tests/features/steps/steps.py b/extensions/couchbase/tests/features/steps/steps.py index 9f0edeffc3..c986140dc4 100644 --- a/extensions/couchbase/tests/features/steps/steps.py +++ b/extensions/couchbase/tests/features/steps/steps.py @@ -13,16 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. from behave import step, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.steps.flow_building_steps import add_ssl_context_service_for_minifi -from minifi_behave.core.minifi_test_context import MinifiTestContext +from containers.couchbase_server_container import CouchbaseServerContainer from minifi_behave.core.helpers import log_due_to_failure +from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.minifi.controller_service import ControllerService -from containers.couchbase_server_container import CouchbaseServerContainer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) +from minifi_behave.steps.flow_building_steps import add_ssl_context_service_for_minifi @step("a Couchbase server is started") @@ -31,37 +32,67 @@ def start_couchbase_server(context: MinifiTestContext): assert context.containers["couchbase-server"].deploy(context) -@step("a CouchbaseClusterService controller service is set up to communicate with the Couchbase server") +@step( + "a CouchbaseClusterService controller service is set up to communicate with the Couchbase server" +) def setup_couchbase_cluster_service(context: MinifiTestContext): - controller_service = ControllerService(class_name="CouchbaseClusterService", service_name="CouchbaseClusterService") - controller_service.add_property("Connection String", f"couchbase://couchbase-server-{context.scenario_id}") + controller_service = ControllerService( + class_name="CouchbaseClusterService", service_name="CouchbaseClusterService" + ) + controller_service.add_property( + "Connection String", f"couchbase://couchbase-server-{context.scenario_id}" + ) controller_service.add_property("User Name", "Administrator") controller_service.add_property("User Password", "password123") - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + controller_service + ) @step("a CouchbaseClusterService is set up using SSL connection") def setup_couchbase_cluster_service_ssl(context): - ssl_context_service = ControllerService(class_name="SSLContextService", service_name="SSLContextService") + ssl_context_service = ControllerService( + class_name="SSLContextService", service_name="SSLContextService" + ) ssl_context_service.add_property("CA Certificate", "/tmp/resources/root_ca.crt") - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(ssl_context_service) - couchbase_cluster_service = ControllerService(class_name="CouchbaseClusterService", service_name="CouchbaseClusterService") - couchbase_cluster_service.add_property("Connection String", f"couchbases://couchbase-server-{context.scenario_id}") + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + ssl_context_service + ) + couchbase_cluster_service = ControllerService( + class_name="CouchbaseClusterService", service_name="CouchbaseClusterService" + ) + couchbase_cluster_service.add_property( + "Connection String", f"couchbases://couchbase-server-{context.scenario_id}" + ) couchbase_cluster_service.add_property("User Name", "Administrator") couchbase_cluster_service.add_property("User Password", "password123") couchbase_cluster_service.add_property("Linked Services", "SSLContextService") - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(couchbase_cluster_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + couchbase_cluster_service + ) @step("a CouchbaseClusterService is setup up using mTLS authentication") def setup_couchbase_cluster_service_mtls(context: MinifiTestContext): add_ssl_context_service_for_minifi(context, "clientuser") - controller_service = ControllerService(class_name="CouchbaseClusterService", service_name="CouchbaseClusterService") - controller_service.add_property("Connection String", f"couchbases://couchbase-server-{context.scenario_id}") + controller_service = ControllerService( + class_name="CouchbaseClusterService", service_name="CouchbaseClusterService" + ) + controller_service.add_property( + "Connection String", f"couchbases://couchbase-server-{context.scenario_id}" + ) controller_service.add_property("Linked Services", "SSLContextService") - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + controller_service + ) -@then("a document with id \"{doc_id}\" in bucket \"{bucket_name}\" is present with data '{data}' of type \"{data_type}\" in Couchbase") -def verify_couchbase_document_data(context, doc_id: str, bucket_name: str, data: str, data_type: str): - assert context.containers["couchbase-server"].is_data_present_in_couchbase(doc_id, bucket_name, data, data_type) or log_due_to_failure(context) +@then( + 'a document with id "{doc_id}" in bucket "{bucket_name}" is present with data \'{data}\' of type "{data_type}" in Couchbase' +) +def verify_couchbase_document_data( + context, doc_id: str, bucket_name: str, data: str, data_type: str +): + assert context.containers["couchbase-server"].is_data_present_in_couchbase( + doc_id, bucket_name, data, data_type + ) or log_due_to_failure(context) diff --git a/extensions/elasticsearch/tests/features/containers/elastic_base_container.py b/extensions/elasticsearch/tests/features/containers/elastic_base_container.py index 577160c135..00ba260cbf 100644 --- a/extensions/elasticsearch/tests/features/containers/elastic_base_container.py +++ b/extensions/elasticsearch/tests/features/containers/elastic_base_container.py @@ -13,13 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.core.helpers import wait_for_condition, retry_check from minifi_behave.containers.container_linux import LinuxContainer +from minifi_behave.core.helpers import retry_check, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext class ElasticBaseContainer(LinuxContainer): - def __init__(self, test_context: MinifiTestContext, image: str, container_name: str): + def __init__( + self, test_context: MinifiTestContext, image: str, container_name: str + ): super().__init__(image, container_name, test_context.network) self.user = None @@ -29,19 +31,49 @@ def deploy(self, context: MinifiTestContext | None, finished_str: str): condition=lambda: finished_str in self.get_logs(), timeout_seconds=300, bail_condition=lambda: self.exited, - context=context) + context=context, + ) def create_doc_elasticsearch(self, index_name: str, doc_id: str) -> bool: - (code, output) = self.exec_run(["/bin/bash", "-c", - "curl -s -u elastic:password -k -XPUT https://localhost:9200/" + index_name + "/_doc/" - + doc_id + " -H Content-Type:application/json -d'{\"field1\":\"value1\"}'"]) + (code, output) = self.exec_run( + [ + "/bin/bash", + "-c", + "curl -s -u elastic:password -k -XPUT https://localhost:9200/" + + index_name + + "/_doc/" + + doc_id + + ' -H Content-Type:application/json -d\'{"field1":"value1"}\'', + ] + ) return code == 0 and ('"_id":"' + doc_id + '"') in output - def check_elastic_field_value(self, index_name: str, doc_id: str, field_name: str, field_value: str) -> bool: - (code, output) = self.exec_run(["/bin/bash", "-c", "curl -s -u elastic:password -k -XGET https://localhost:9200/" + index_name + "/_doc/" + doc_id]) + def check_elastic_field_value( + self, index_name: str, doc_id: str, field_name: str, field_value: str + ) -> bool: + (code, output) = self.exec_run( + [ + "/bin/bash", + "-c", + "curl -s -u elastic:password -k -XGET https://localhost:9200/" + + index_name + + "/_doc/" + + doc_id, + ] + ) return code == 0 and (field_name + '":"' + field_value) in output @retry_check() def is_elasticsearch_empty(self) -> bool: - (code, output) = self.exec_run(["curl", "-s", "-u", "elastic:password", "-k", "-XGET", "https://localhost:9200/_search"]) + (code, output) = self.exec_run( + [ + "curl", + "-s", + "-u", + "elastic:password", + "-k", + "-XGET", + "https://localhost:9200/_search", + ] + ) return code == 0 and '"hits":[]' in output diff --git a/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py b/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py index 45403de22d..d21f3fe82c 100644 --- a/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py +++ b/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py @@ -16,41 +16,91 @@ import json import logging import os +from pathlib import Path from elastic_base_container import ElasticBaseContainer -from pathlib import Path -from minifi_behave.core.ssl_utils import make_server_cert, make_cert_without_extended_usage, dump_cert, dump_key from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import ( + dump_cert, + dump_key, + make_cert_without_extended_usage, + make_server_cert, +) + +logger = logging.getLogger(__name__) class ElasticsearchContainer(ElasticBaseContainer): IMAGE = "elasticsearch:9.1.5" def __init__(self, test_context: MinifiTestContext): - super().__init__(test_context, ElasticsearchContainer.IMAGE, f"elasticsearch-{test_context.scenario_id}") + super().__init__( + test_context, + ElasticsearchContainer.IMAGE, + f"elasticsearch-{test_context.scenario_id}", + ) - http_cert, http_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - transport_cert, transport_key = make_cert_without_extended_usage(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + http_cert, http_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) + transport_cert, transport_key = make_cert_without_extended_usage( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/usr/share/elasticsearch/config/certs/root_ca.crt", root_ca_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/elasticsearch/config/certs/root_ca.crt", + root_ca_content, + permissions=0o644, + ) + ) http_cert_content = dump_cert(http_cert) - self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.crt", http_cert_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/elasticsearch/config/certs/elastic_http.crt", + http_cert_content, + permissions=0o644, + ) + ) http_key_content = dump_key(http_key) - self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.key", http_key_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/elasticsearch/config/certs/elastic_http.key", + http_key_content, + permissions=0o644, + ) + ) transport_cert_content = dump_cert(transport_cert) - self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.crt", transport_cert_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/elasticsearch/config/certs/elastic_transport.crt", + transport_cert_content, + permissions=0o644, + ) + ) transport_key_content = dump_key(transport_key) - self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.key", transport_key_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/elasticsearch/config/certs/elastic_transport.key", + transport_key_content, + permissions=0o644, + ) + ) features_dir = Path(__file__).resolve().parent.parent - self.host_files.append(HostFile('/usr/share/elasticsearch/config/elasticsearch.yml', os.path.join(features_dir, "resources", "elasticsearch.yml"))) + self.host_files.append( + HostFile( + "/usr/share/elasticsearch/config/elasticsearch.yml", + os.path.join(features_dir, "resources", "elasticsearch.yml"), + ) + ) self.environment.append("ELASTIC_PASSWORD=password") @@ -62,37 +112,35 @@ def elastic_generate_apikey(self): api_user = "elastic:password" api_headers = "Content-Type:application/json" api_data = ( - '{' + "{" ' "name": "my-api-key",' ' "expiration": "1d",' ' "role_descriptors": {' ' "role-a": {' ' "cluster": [' ' "all"' - ' ],' + " ]," ' "index": [' - ' {' + " {" ' "names": [' ' "my_index"' - ' ],' + " ]," ' "privileges": [' ' "all"' - ' ]' - ' }' - ' ]' - ' }' - ' }' - '}' + " ]" + " }" + " ]" + " }" + " }" + "}" ) curl_cmd = ( - f"curl -s -u {api_user} -k -XPOST {api_url} " - f"-H {api_headers} " - f"-d'{api_data}'" + f"curl -s -u {api_user} -k -XPOST {api_url} -H {api_headers} -d'{api_data}'" ) (code, output) = self.exec_run(["/bin/bash", "-c", curl_cmd]) if code != 0: return None - logging.info(f"Elasticsearch generate API key output: {output}") + logger.info(f"Elasticsearch generate API key output: {output}") output_lines = output.splitlines() result = json.loads(output_lines[-1]) return result["encoded"] diff --git a/extensions/elasticsearch/tests/features/containers/opensearch_container.py b/extensions/elasticsearch/tests/features/containers/opensearch_container.py index 1f09d0cce4..2a24652ef2 100644 --- a/extensions/elasticsearch/tests/features/containers/opensearch_container.py +++ b/extensions/elasticsearch/tests/features/containers/opensearch_container.py @@ -13,39 +13,72 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import logging +import os +from pathlib import Path from elastic_base_container import ElasticBaseContainer -from pathlib import Path -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert + +logger = logging.getLogger(__name__) class OpensearchContainer(ElasticBaseContainer): IMAGE = "opensearchproject/opensearch:2.6.0" def __init__(self, test_context: MinifiTestContext): - super().__init__(test_context, OpensearchContainer.IMAGE, f"opensearch-{test_context.scenario_id}") + super().__init__( + test_context, + OpensearchContainer.IMAGE, + f"opensearch-{test_context.scenario_id}", + ) - admin_pem, admin_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + admin_pem, admin_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/usr/share/opensearch/config/root-ca.pem", root_ca_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/opensearch/config/root-ca.pem", + root_ca_content, + permissions=0o644, + ) + ) admin_pem_content = dump_cert(admin_pem) - self.files.append(File("/usr/share/opensearch/config/admin.pem", admin_pem_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/opensearch/config/admin.pem", + admin_pem_content, + permissions=0o644, + ) + ) admin_key_content = dump_key(admin_key) - self.files.append(File("/usr/share/opensearch/config/admin-key.pem", admin_key_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/opensearch/config/admin-key.pem", + admin_key_content, + permissions=0o644, + ) + ) features_dir = Path(__file__).resolve().parent.parent - self.host_files.append(HostFile('/usr/share/opensearch/config/opensearch.yml', os.path.join(features_dir, "resources", "opensearch.yml"))) + self.host_files.append( + HostFile( + "/usr/share/opensearch/config/opensearch.yml", + os.path.join(features_dir, "resources", "opensearch.yml"), + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: - return super().deploy(context, 'Hot-reloading of audit configuration is enabled') + return super().deploy( + context, "Hot-reloading of audit configuration is enabled" + ) def add_elastic_user_to_opensearch(self): curl_cmd = [ @@ -55,9 +88,9 @@ def add_elastic_user_to_opensearch(self): "-XPUT", f"https://{self.container_name}:9200/_plugins/_security/api/internalusers/elastic", "-H 'Content-Type:application/json'", - "-d '{\"password\":\"password\",\"backend_roles\":[\"admin\"]}'" + '-d \'{"password":"password","backend_roles":["admin"]}\'', ] full_cmd = " ".join(curl_cmd) (code, output) = self.exec_run(["/bin/bash", "-c", full_cmd]) - logging.info(f"Add elastic user to Opensearch output: {output}") + logger.info(f"Add elastic user to Opensearch output: {output}") return code == 0 and '"status":"CREATED"' in output diff --git a/extensions/elasticsearch/tests/features/environment.py b/extensions/elasticsearch/tests/features/environment.py index eaea25090c..2878469ce0 100644 --- a/extensions/elasticsearch/tests/features/environment.py +++ b/extensions/elasticsearch/tests/features/environment.py @@ -13,11 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker from containers.elasticsearch_container import ElasticsearchContainer from containers.opensearch_container import OpensearchContainer -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_feature(context, feature): diff --git a/extensions/elasticsearch/tests/features/steps/steps.py b/extensions/elasticsearch/tests/features/steps/steps.py index 966c7a3b11..14be9f5456 100644 --- a/extensions/elasticsearch/tests/features/steps/steps.py +++ b/extensions/elasticsearch/tests/features/steps/steps.py @@ -12,69 +12,113 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from behave import step, given, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.minifi.controller_service import ControllerService -from minifi_behave.core.helpers import log_due_to_failure +from behave import given, step, then from containers.elasticsearch_container import ElasticsearchContainer from containers.opensearch_container import OpensearchContainer - - -@step('an Elasticsearch server is set up and running') -@step('an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index"') -@step('an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"') +from minifi_behave.core.helpers import log_due_to_failure +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.minifi.controller_service import ControllerService +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) + + +@step("an Elasticsearch server is set up and running") +@step( + 'an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index"' +) +@step( + 'an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"' +) def setup_elasticsearch_server(context: MinifiTestContext): context.containers["elasticsearch"] = ElasticsearchContainer(context) assert context.containers["elasticsearch"].deploy(context) - assert context.containers["elasticsearch"].create_doc_elasticsearch("my_index", "preloaded_id") or context.containers["elasticsearch"].log_app_output() + assert ( + context.containers["elasticsearch"].create_doc_elasticsearch( + "my_index", "preloaded_id" + ) + or context.containers["elasticsearch"].log_app_output() + ) -@given('an ElasticsearchCredentialsControllerService is set up with Basic Authentication') +@given( + "an ElasticsearchCredentialsControllerService is set up with Basic Authentication" +) def setup_elasticsearch_credentials_service_basic_auth(context: MinifiTestContext): - controller_service = ControllerService(class_name="ElasticsearchCredentialsControllerService", service_name="ElasticsearchCredentialsControllerService") + controller_service = ControllerService( + class_name="ElasticsearchCredentialsControllerService", + service_name="ElasticsearchCredentialsControllerService", + ) controller_service.add_property("Username", "elastic") controller_service.add_property("Password", "password") - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + controller_service + ) -@given('an ElasticsearchCredentialsControllerService is set up with ApiKey') +@given("an ElasticsearchCredentialsControllerService is set up with ApiKey") def setup_elasticsearch_credentials_service_api_key(context: MinifiTestContext): - controller_service = ControllerService(class_name="ElasticsearchCredentialsControllerService", service_name="ElasticsearchCredentialsControllerService") + controller_service = ControllerService( + class_name="ElasticsearchCredentialsControllerService", + service_name="ElasticsearchCredentialsControllerService", + ) api_key = context.containers["elasticsearch"].elastic_generate_apikey() controller_service.add_property("API Key", api_key) - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + controller_service + ) -@then('Elasticsearch has a document with "{doc_id}" in "{index}" that has "{value}" set in "{field}"') -def verify_elasticsearch_document_field_value(context: MinifiTestContext, doc_id: str, index: str, value: str, field: str): - assert context.containers["elasticsearch"].check_elastic_field_value(index_name=index, doc_id=doc_id, field_name=field, field_value=value) or log_due_to_failure(context) +@then( + 'Elasticsearch has a document with "{doc_id}" in "{index}" that has "{value}" set in "{field}"' +) +def verify_elasticsearch_document_field_value( + context: MinifiTestContext, doc_id: str, index: str, value: str, field: str +): + assert context.containers["elasticsearch"].check_elastic_field_value( + index_name=index, doc_id=doc_id, field_name=field, field_value=value + ) or log_due_to_failure(context) @then("Elasticsearch is empty") def verify_elasticsearch_is_empty(context): - assert context.containers["elasticsearch"].is_elasticsearch_empty() or log_due_to_failure(context) - - -@given('an Opensearch server is set up and running') -@given('an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index"') -@given('an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"') + assert context.containers[ + "elasticsearch" + ].is_elasticsearch_empty() or log_due_to_failure(context) + + +@given("an Opensearch server is set up and running") +@given( + 'an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index"' +) +@given( + 'an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"' +) def setup_opensearch_server(context): context.containers["opensearch"] = OpensearchContainer(context) context.containers["opensearch"].deploy(context) context.containers["opensearch"].add_elastic_user_to_opensearch() - context.containers["opensearch"].create_doc_elasticsearch("my_index", "preloaded_id") + context.containers["opensearch"].create_doc_elasticsearch( + "my_index", "preloaded_id" + ) -@then('Opensearch has a document with "{doc_id}" in "{index}" that has "{value}" set in "{field}"') -def verify_opensearch_document_field_value(context: MinifiTestContext, doc_id: str, index: str, value: str, field: str): - assert context.containers["opensearch"].check_elastic_field_value(index_name=index, doc_id=doc_id, field_name=field, field_value=value) or log_due_to_failure(context) +@then( + 'Opensearch has a document with "{doc_id}" in "{index}" that has "{value}" set in "{field}"' +) +def verify_opensearch_document_field_value( + context: MinifiTestContext, doc_id: str, index: str, value: str, field: str +): + assert context.containers["opensearch"].check_elastic_field_value( + index_name=index, doc_id=doc_id, field_name=field, field_value=value + ) or log_due_to_failure(context) @then("Opensearch is empty") def verify_opensearch_is_empty(context): - assert context.containers["opensearch"].is_elasticsearch_empty() or log_due_to_failure(context) + assert context.containers[ + "opensearch" + ].is_elasticsearch_empty() or log_due_to_failure(context) diff --git a/extensions/gcp/tests/features/containers/fake_gcs_server_container.py b/extensions/gcp/tests/features/containers/fake_gcs_server_container.py index 506478932f..789a6b8f87 100644 --- a/extensions/gcp/tests/features/containers/fake_gcs_server_container.py +++ b/extensions/gcp/tests/features/containers/fake_gcs_server_container.py @@ -14,19 +14,28 @@ # limitations under the License. import logging -from minifi_behave.core.helpers import wait_for_condition, retry_check + from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.directory import Directory +from minifi_behave.core.helpers import retry_check, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class FakeGcsServerContainer(LinuxContainer): IMAGE = "fsouza/fake-gcs-server:1.45.1" def __init__(self, test_context: MinifiTestContext): - super().__init__(FakeGcsServerContainer.IMAGE, f"fake-gcs-server-{test_context.scenario_id}", test_context.network, - command=f'-scheme http -host fake-gcs-server-{test_context.scenario_id}') - self.dirs.append(Directory(path="/data/test-bucket", files={"test-file": "preloaded data\n"})) + super().__init__( + FakeGcsServerContainer.IMAGE, + f"fake-gcs-server-{test_context.scenario_id}", + test_context.network, + command=f"-scheme http -host fake-gcs-server-{test_context.scenario_id}", + ) + self.dirs.append( + Directory(path="/data/test-bucket", files={"test-file": "preloaded data\n"}) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -35,16 +44,17 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context) + context=context, + ) @retry_check() def check_google_cloud_storage(self, content): (code, output) = self.exec_run(["grep", "-r", content, "/storage"]) - logging.info(f"GCS storage contents matching '{content}': {output}") + logger.info(f"GCS storage contents matching '{content}': {output}") return code == 0 @retry_check() def is_gcs_bucket_empty(self): (code, output) = self.exec_run(["ls", "/storage/test-bucket"]) - logging.info(f"GCS bucket contents: {output}") + logger.info(f"GCS bucket contents: {output}") return code == 0 and output == "" diff --git a/extensions/gcp/tests/features/environment.py b/extensions/gcp/tests/features/environment.py index 057b87c71e..c4e51f658f 100644 --- a/extensions/gcp/tests/features/environment.py +++ b/extensions/gcp/tests/features/environment.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker from containers.fake_gcs_server_container import FakeGcsServerContainer -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): diff --git a/extensions/gcp/tests/features/steps/steps.py b/extensions/gcp/tests/features/steps/steps.py index b2e9308ece..f55e57b652 100644 --- a/extensions/gcp/tests/features/steps/steps.py +++ b/extensions/gcp/tests/features/steps/steps.py @@ -13,29 +13,36 @@ # See the License for the specific language governing permissions and # limitations under the License. from behave import step, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.helpers import log_due_to_failure from containers.fake_gcs_server_container import FakeGcsServerContainer +from minifi_behave.core.helpers import log_due_to_failure +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @step("a Google Cloud storage server is set up") @step("a Google Cloud storage server is set up with some test data") -@step('a Google Cloud storage server is set up and a single object with contents "preloaded data" is present') +@step( + 'a Google Cloud storage server is set up and a single object with contents "preloaded data" is present' +) def setup_gcs_server(context: MinifiTestContext): context.containers["fake-gcs-server"] = FakeGcsServerContainer(context) assert context.containers["fake-gcs-server"].deploy(context) -@then('an object with the content \"{content}\" is present in the Google Cloud storage') +@then('an object with the content "{content}" is present in the Google Cloud storage') def verify_gcs_object_content(context: MinifiTestContext, content: str): - assert context.containers["fake-gcs-server"].check_google_cloud_storage(content) or log_due_to_failure(context) + assert context.containers["fake-gcs-server"].check_google_cloud_storage( + content + ) or log_due_to_failure(context) @then("the test bucket of Google Cloud Storage is empty") def verify_gcs_bucket_is_empty(context: MinifiTestContext): - assert context.containers["fake-gcs-server"].is_gcs_bucket_empty() or log_due_to_failure(context) + assert context.containers[ + "fake-gcs-server" + ].is_gcs_bucket_empty() or log_due_to_failure(context) diff --git a/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py b/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py index d626468f77..91bc58f992 100644 --- a/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py +++ b/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py @@ -15,12 +15,14 @@ import logging -from minifi_behave.core.helpers import wait_for_condition, retry_check +from docker.errors import ContainerError from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.file import File +from minifi_behave.core.helpers import retry_check, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key -from docker.errors import ContainerError +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert + +logger = logging.getLogger(__name__) class GrafanaLokiOptions: @@ -33,19 +35,31 @@ class GrafanaLokiContainer(LinuxContainer): IMAGE = "grafana/loki:3.2.1" def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions): - super().__init__(GrafanaLokiContainer.IMAGE, f"grafana-loki-server-{test_context.scenario_id}", test_context.network) + super().__init__( + GrafanaLokiContainer.IMAGE, + f"grafana-loki-server-{test_context.scenario_id}", + test_context.network, + ) extra_ssl_settings = "" if options.enable_ssl: - grafana_loki_cert, grafana_loki_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + grafana_loki_cert, grafana_loki_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/etc/loki/root_ca.crt", root_ca_content, permissions=0o644)) + self.files.append( + File("/etc/loki/root_ca.crt", root_ca_content, permissions=0o644) + ) grafana_loki_cert_content = dump_cert(grafana_loki_cert) - self.files.append(File("/etc/loki/cert.pem", grafana_loki_cert_content, permissions=0o644)) + self.files.append( + File("/etc/loki/cert.pem", grafana_loki_cert_content, permissions=0o644) + ) grafana_loki_key_content = dump_key(grafana_loki_key) - self.files.append(File("/etc/loki/key.pem", grafana_loki_key_content, permissions=0o644)) + self.files.append( + File("/etc/loki/key.pem", grafana_loki_key_content, permissions=0o644) + ) extra_ssl_settings = """ http_tls_config: @@ -89,8 +103,8 @@ def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions) tls_insecure_skip_verify: true """ - grafana_loki_yml_content = """ -auth_enabled: {enable_multi_tenancy} + grafana_loki_yml_content = f""" +auth_enabled: {options.enable_multi_tenancy} server: http_listen_port: 3100 @@ -123,9 +137,15 @@ def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions) analytics: reporting_enabled: false -""".format(extra_ssl_settings=extra_ssl_settings, enable_multi_tenancy=options.enable_multi_tenancy) +""" - self.files.append(File("/etc/loki/local-config.yaml", grafana_loki_yml_content.encode(), permissions=0o644)) + self.files.append( + File( + "/etc/loki/local-config.yaml", + grafana_loki_yml_content.encode(), + permissions=0o644, + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -134,19 +154,48 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=120, bail_condition=lambda: self.exited, - context=context) + context=context, + ) @retry_check() - def are_lines_present(self, lines: str, timeout: int, ssl: bool, tenant_id: str = "") -> bool: + def are_lines_present( + self, lines: str, timeout: int, ssl: bool, tenant_id: str = "" + ) -> bool: try: - self.client.containers.run("minifi-grafana-loki-helper:latest", ["python", "/scripts/check_log_lines_on_grafana.py", self.container_name, lines, str(timeout), str(ssl), tenant_id], - remove=True, stdout=True, stderr=True, network=self.network.name) + self.client.containers.run( + "minifi-grafana-loki-helper:latest", + [ + "python", + "/scripts/check_log_lines_on_grafana.py", + self.container_name, + lines, + str(timeout), + str(ssl), + tenant_id, + ], + remove=True, + stdout=True, + stderr=True, + network=self.network.name, + ) return True except ContainerError as e: - stdout = e.stdout.decode("utf-8", errors="replace") if hasattr(e, "stdout") and e.stdout else "" - stderr = e.stderr.decode("utf-8", errors="replace") if hasattr(e, "stderr") and e.stderr else "" - logging.error(f"Failed to run python command in grafana loki helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'") + stdout = ( + e.stdout.decode("utf-8", errors="replace") + if hasattr(e, "stdout") and e.stdout + else "" + ) + stderr = ( + e.stderr.decode("utf-8", errors="replace") + if hasattr(e, "stderr") and e.stderr + else "" + ) + logger.error( + f"Failed to run python command in grafana loki helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'" + ) return False except Exception as e: - logging.error(f"Unexpected error while running python command in grafana loki helper docker: '{e}'") + logger.error( + f"Unexpected error while running python command in grafana loki helper docker: '{e}'" + ) return False diff --git a/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py b/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py index f622cbda3e..a24a131841 100644 --- a/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py +++ b/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py @@ -13,14 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext class ReverseProxyContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): - super().__init__("minifi-reverse-proxy:latest", f"reverse-proxy-{test_context.scenario_id}", test_context.network) + super().__init__( + "minifi-reverse-proxy:latest", + f"reverse-proxy-{test_context.scenario_id}", + test_context.network, + ) self.environment = [ "BASIC_USERNAME=admin", "BASIC_PASSWORD=password", @@ -35,4 +39,5 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context) + context=context, + ) diff --git a/extensions/grafana-loki/tests/features/environment.py b/extensions/grafana-loki/tests/features/environment.py index 1318d2f5de..a36a9d688b 100644 --- a/extensions/grafana-loki/tests/features/environment.py +++ b/extensions/grafana-loki/tests/features/environment.py @@ -13,16 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker -from containers.grafana_loki_container import GrafanaLokiContainer from pathlib import Path + +from containers.grafana_loki_container import GrafanaLokiContainer from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): - check_log_lines_path = Path(__file__).resolve().parent / "resources" / "check_log_lines_on_grafana.py" + check_log_lines_path = ( + Path(__file__).resolve().parent / "resources" / "check_log_lines_on_grafana.py" + ) check_log_lines_content = None with open(check_log_lines_path, "rb") as f: check_log_lines_content = f.read() @@ -33,13 +36,15 @@ def before_all(context): grafana_helper_builder = DockerImageBuilder( image_tag="minifi-grafana-loki-helper:latest", dockerfile_content=dockerfile, - files_on_context={"check_log_lines_on_grafana.py": check_log_lines_content} + files_on_context={"check_log_lines_on_grafana.py": check_log_lines_content}, ) grafana_helper_builder.build() reverse_proxy_builder = DockerImageBuilder( image_tag="minifi-reverse-proxy:latest", - build_context_path=str(Path(__file__).resolve().parent / "resources" / "reverse-proxy") + build_context_path=str( + Path(__file__).resolve().parent / "resources" / "reverse-proxy" + ), ) reverse_proxy_builder.build() docker.from_env().images.pull(GrafanaLokiContainer.IMAGE) diff --git a/extensions/grafana-loki/tests/features/resources/check_log_lines_on_grafana.py b/extensions/grafana-loki/tests/features/resources/check_log_lines_on_grafana.py index 6d961a256f..90d321d8e5 100644 --- a/extensions/grafana-loki/tests/features/resources/check_log_lines_on_grafana.py +++ b/extensions/grafana-loki/tests/features/resources/check_log_lines_on_grafana.py @@ -14,9 +14,10 @@ # limitations under the License. import sys -import requests import time +import requests + def wait_for(action, timeout_seconds, *args, **kwargs) -> bool: start_time = time.perf_counter() @@ -30,7 +31,9 @@ def wait_for(action, timeout_seconds, *args, **kwargs) -> bool: return False -def verify_log_lines_on_grafana_loki(host: str, lines: list[str], ssl: bool, tenant_id: str) -> bool: +def verify_log_lines_on_grafana_loki( + host: str, lines: list[str], ssl: bool, tenant_id: str +) -> bool: labels = '{job="minifi"}' prefix = "http://" if ssl: @@ -40,14 +43,18 @@ def verify_log_lines_on_grafana_loki(host: str, lines: list[str], ssl: bool, ten headers = None if tenant_id: - headers = {'X-Scope-OrgID': tenant_id} + headers = {"X-Scope-OrgID": tenant_id} response = requests.get(query_url, verify=False, timeout=30, headers=headers) if response.status_code < 200 or response.status_code >= 300: return False json_response = response.json() - if "data" not in json_response or "result" not in json_response["data"] or len(json_response["data"]["result"]) < 1: + if ( + "data" not in json_response + or "result" not in json_response["data"] + or len(json_response["data"]["result"]) < 1 + ): return False result = json_response["data"]["result"][0] @@ -60,8 +67,13 @@ def verify_log_lines_on_grafana_loki(host: str, lines: list[str], ssl: bool, ten return True -def wait_for_lines_on_grafana_loki(host: str, lines: list[str], timeout_seconds: int, ssl: bool, tenant_id: str) -> bool: - return wait_for(lambda: verify_log_lines_on_grafana_loki(host, lines, ssl, tenant_id), timeout_seconds) +def wait_for_lines_on_grafana_loki( + host: str, lines: list[str], timeout_seconds: int, ssl: bool, tenant_id: str +) -> bool: + return wait_for( + lambda: verify_log_lines_on_grafana_loki(host, lines, ssl, tenant_id), + timeout_seconds, + ) if __name__ == "__main__": @@ -75,5 +87,7 @@ def wait_for_lines_on_grafana_loki(host: str, lines: list[str], timeout_seconds: tenant_id = "" if len(sys.argv) >= 6: tenant_id = sys.argv[5] - if not wait_for_lines_on_grafana_loki(host, lines.split(";"), timeout_seconds, ssl, tenant_id): + if not wait_for_lines_on_grafana_loki( + host, lines.split(";"), timeout_seconds, ssl, tenant_id + ): sys.exit(1) diff --git a/extensions/grafana-loki/tests/features/steps/steps.py b/extensions/grafana-loki/tests/features/steps/steps.py index 0ab0769238..2a14d3a54b 100644 --- a/extensions/grafana-loki/tests/features/steps/steps.py +++ b/extensions/grafana-loki/tests/features/steps/steps.py @@ -13,51 +13,78 @@ # See the License for the specific language governing permissions and # limitations under the License. from behave import step, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.helpers import log_due_to_failure from containers.grafana_loki_container import GrafanaLokiContainer, GrafanaLokiOptions from containers.reverse_proxy_container import ReverseProxyContainer +from minifi_behave.core.helpers import log_due_to_failure +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @step("a Grafana Loki server is set up") def setup_grafana_loki_server(context: MinifiTestContext): - context.containers["grafana-loki-server"] = GrafanaLokiContainer(context, GrafanaLokiOptions()) + context.containers["grafana-loki-server"] = GrafanaLokiContainer( + context, GrafanaLokiOptions() + ) @step("a Grafana Loki server is set up with multi-tenancy enabled") def setup_grafana_loki_server_multitenant(context: MinifiTestContext): - context.containers["grafana-loki-server"] = GrafanaLokiContainer(context, GrafanaLokiOptions(enable_multi_tenancy=True)) + context.containers["grafana-loki-server"] = GrafanaLokiContainer( + context, GrafanaLokiOptions(enable_multi_tenancy=True) + ) @step("a Grafana Loki server with SSL is set up") def setup_grafana_loki_server_ssl(context: MinifiTestContext): - context.containers["grafana-loki-server"] = GrafanaLokiContainer(context, GrafanaLokiOptions(enable_ssl=True)) + context.containers["grafana-loki-server"] = GrafanaLokiContainer( + context, GrafanaLokiOptions(enable_ssl=True) + ) -@then("\"{lines}\" lines are published to the Grafana Loki server in less than {timeout_seconds:d} seconds") -@then("\"{lines}\" line is published to the Grafana Loki server in less than {timeout_seconds:d} seconds") +@then( + '"{lines}" lines are published to the Grafana Loki server in less than {timeout_seconds:d} seconds' +) +@then( + '"{lines}" line is published to the Grafana Loki server in less than {timeout_seconds:d} seconds' +) def verify_lines_published_to_loki(context, lines: str, timeout_seconds: int): - assert context.containers["grafana-loki-server"].are_lines_present(lines, timeout_seconds, ssl=False) or log_due_to_failure(context) + assert context.containers["grafana-loki-server"].are_lines_present( + lines, timeout_seconds, ssl=False + ) or log_due_to_failure(context) -@then("\"{lines}\" lines are published to the \"{tenant_id}\" tenant on the Grafana Loki server in less than {timeout_seconds:d} seconds") -@then("\"{lines}\" line is published to the \"{tenant_id}\" tenant on the Grafana Loki server in less than {timeout_seconds:d} seconds") -def verify_lines_published_to_loki_tenant(context, lines: str, timeout_seconds: int, tenant_id: str): - assert context.containers["grafana-loki-server"].are_lines_present(lines, timeout_seconds, ssl=False, tenant_id=tenant_id) or log_due_to_failure(context) +@then( + '"{lines}" lines are published to the "{tenant_id}" tenant on the Grafana Loki server in less than {timeout_seconds:d} seconds' +) +@then( + '"{lines}" line is published to the "{tenant_id}" tenant on the Grafana Loki server in less than {timeout_seconds:d} seconds' +) +def verify_lines_published_to_loki_tenant( + context, lines: str, timeout_seconds: int, tenant_id: str +): + assert context.containers["grafana-loki-server"].are_lines_present( + lines, timeout_seconds, ssl=False, tenant_id=tenant_id + ) or log_due_to_failure(context) -@then("\"{lines}\" lines are published using SSL to the Grafana Loki server in less than {timeout_seconds:d} seconds") -@then("\"{lines}\" line is published using SSL to the Grafana Loki server in less than {timeout_seconds:d} seconds") +@then( + '"{lines}" lines are published using SSL to the Grafana Loki server in less than {timeout_seconds:d} seconds' +) +@then( + '"{lines}" line is published using SSL to the Grafana Loki server in less than {timeout_seconds:d} seconds' +) def verify_lines_published_to_loki_ssl(context, lines: str, timeout_seconds: int): - assert context.containers["grafana-loki-server"].are_lines_present(lines, timeout_seconds, ssl=True) or log_due_to_failure(context) + assert context.containers["grafana-loki-server"].are_lines_present( + lines, timeout_seconds, ssl=True + ) or log_due_to_failure(context) # Nginx reverse proxy -@step('a reverse proxy is set up to forward requests to the Grafana Loki server') +@step("a reverse proxy is set up to forward requests to the Grafana Loki server") def setup_reverse_proxy_for_loki(context): context.containers["reverse-proxy"] = ReverseProxyContainer(context) diff --git a/extensions/kafka/tests/features/containers/kafka_server_container.py b/extensions/kafka/tests/features/containers/kafka_server_container.py index d4434cd042..ffca6fa3f2 100644 --- a/extensions/kafka/tests/features/containers/kafka_server_container.py +++ b/extensions/kafka/tests/features/containers/kafka_server_container.py @@ -16,21 +16,26 @@ import logging import re -from cryptography.hazmat.primitives import serialization import jks - -from minifi_behave.core.helpers import wait_for_condition -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key +from cryptography.hazmat.primitives import serialization from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.file import File +from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert + +logger = logging.getLogger(__name__) class KafkaServer(LinuxContainer): IMAGE = "apache/kafka:4.1.0" def __init__(self, test_context: MinifiTestContext): - super().__init__(KafkaServer.IMAGE, f"kafka-server-{test_context.scenario_id}", test_context.network) + super().__init__( + KafkaServer.IMAGE, + f"kafka-server-{test_context.scenario_id}", + test_context.network, + ) self.environment.append("KAFKA_NODE_ID=1") self.environment.append("KAFKA_PROCESS_ROLES=controller,broker") self.environment.append("KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT") @@ -38,13 +43,23 @@ def __init__(self, test_context: MinifiTestContext): self.environment.append("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1") self.environment.append("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1") self.environment.append("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1") - self.environment.append(f"KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-server-{test_context.scenario_id}:9096") - self.environment.append(f"KAFKA_LISTENERS=PLAINTEXT://kafka-server-{test_context.scenario_id}:9092,SASL_PLAINTEXT://kafka-server-{test_context.scenario_id}:9094,SSL://kafka-server-{test_context.scenario_id}:9093,SASL_SSL://kafka-server-{test_context.scenario_id}:9095,CONTROLLER://kafka-server-{test_context.scenario_id}:9096") - self.environment.append(f"KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-server-{test_context.scenario_id}:9092,SASL_PLAINTEXT://kafka-server-{test_context.scenario_id}:9094,SSL://kafka-server-{test_context.scenario_id}:9093,SASL_SSL://kafka-server-{test_context.scenario_id}:9095,CONTROLLER://kafka-server-{test_context.scenario_id}:9096") - self.environment.append("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SSL:SSL,SASL_SSL:SASL_SSL,CONTROLLER:PLAINTEXT") + self.environment.append( + f"KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-server-{test_context.scenario_id}:9096" + ) + self.environment.append( + f"KAFKA_LISTENERS=PLAINTEXT://kafka-server-{test_context.scenario_id}:9092,SASL_PLAINTEXT://kafka-server-{test_context.scenario_id}:9094,SSL://kafka-server-{test_context.scenario_id}:9093,SASL_SSL://kafka-server-{test_context.scenario_id}:9095,CONTROLLER://kafka-server-{test_context.scenario_id}:9096" + ) + self.environment.append( + f"KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-server-{test_context.scenario_id}:9092,SASL_PLAINTEXT://kafka-server-{test_context.scenario_id}:9094,SSL://kafka-server-{test_context.scenario_id}:9093,SASL_SSL://kafka-server-{test_context.scenario_id}:9095,CONTROLLER://kafka-server-{test_context.scenario_id}:9096" + ) + self.environment.append( + "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SSL:SSL,SASL_SSL:SASL_SSL,CONTROLLER:PLAINTEXT" + ) self.environment.append("KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN") self.environment.append("KAFKA_SASL_ENABLED_MECHANISMS=PLAIN") - self.environment.append("KAFKA_OPTS=-Djava.security.auth.login.config=/opt/kafka/config/kafka_jaas.conf -Dlog4j2.rootLogger.level=DEBUG -Dlog4j2.logger.org.apache.kafka.controller.level=DEBUG") + self.environment.append( + "KAFKA_OPTS=-Djava.security.auth.login.config=/opt/kafka/config/kafka_jaas.conf -Dlog4j2.rootLogger.level=DEBUG -Dlog4j2.logger.org.apache.kafka.controller.level=DEBUG" + ) self.environment.append("KAFKA_SSL_PROTOCOL=TLS") self.environment.append("KAFKA_SSL_ENABLED_PROTOCOLS=TLSv1.2") self.environment.append("KAFKA_SSL_KEYSTORE_TYPE=JKS") @@ -56,23 +71,46 @@ def __init__(self, test_context: MinifiTestContext): self.environment.append("KAFKA_SSL_TRUSTSTORE_FILENAME=kafka.truststore.jks") self.environment.append("KAFKA_SSL_CLIENT_AUTH=none") - kafka_cert, kafka_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + kafka_cert, kafka_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) - pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [dump_cert(kafka_cert, encoding_type=serialization.Encoding.DER)], dump_key(kafka_key, encoding_type=serialization.Encoding.DER), 'rsa_raw') - server_keystore = jks.KeyStore.new('jks', [pke]) - server_keystore_content = server_keystore.saves('abcdefgh') - self.files.append(File("/etc/kafka/secrets/kafka.keystore.jks", server_keystore_content, permissions=0o644)) - self.files.append(File("/etc/kafka/secrets/credentials.conf", b'abcdefgh', permissions=0o644)) + pke = jks.PrivateKeyEntry.new( + "kafka-broker-cert", + [dump_cert(kafka_cert, encoding_type=serialization.Encoding.DER)], + dump_key(kafka_key, encoding_type=serialization.Encoding.DER), + "rsa_raw", + ) + server_keystore = jks.KeyStore.new("jks", [pke]) + server_keystore_content = server_keystore.saves("abcdefgh") + self.files.append( + File( + "/etc/kafka/secrets/kafka.keystore.jks", + server_keystore_content, + permissions=0o644, + ) + ) + self.files.append( + File("/etc/kafka/secrets/credentials.conf", b"abcdefgh", permissions=0o644) + ) trusted_cert = jks.TrustedCertEntry.new( - 'root-ca', # Alias for the certificate - dump_cert(test_context.root_ca_cert, encoding_type=serialization.Encoding.DER) + "root-ca", # Alias for the certificate + dump_cert( + test_context.root_ca_cert, encoding_type=serialization.Encoding.DER + ), ) # Create a JKS keystore that will serve as a truststore with the trusted cert entry. - truststore = jks.KeyStore.new('jks', [trusted_cert]) - truststore_content = truststore.saves('abcdefgh') - self.files.append(File("/etc/kafka/secrets/kafka.truststore.jks", truststore_content, permissions=0o644)) + truststore = jks.KeyStore.new("jks", [trusted_cert]) + truststore_content = truststore.saves("abcdefgh") + self.files.append( + File( + "/etc/kafka/secrets/kafka.truststore.jks", + truststore_content, + permissions=0o644, + ) + ) jaas_config_file_content = """ KafkaServer { @@ -89,7 +127,13 @@ def __init__(self, test_context: MinifiTestContext): password="admin-secret"; }; """ - self.files.append(File("/opt/kafka/config/kafka_jaas.conf", jaas_config_file_content, permissions=0o644)) + self.files.append( + File( + "/opt/kafka/config/kafka_jaas.conf", + jaas_config_file_content, + permissions=0o644, + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -98,32 +142,64 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context) + context=context, + ) def create_topic(self, topic_name: str): - (code, output) = self.exec_run(["/bin/sh", "-c", f"/opt/kafka/bin/kafka-topics.sh --create --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092"]) - logging.info(f"Create topic output: '{output}' with code {code}") + (code, output) = self.exec_run( + [ + "/bin/sh", + "-c", + f"/opt/kafka/bin/kafka-topics.sh --create --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092", + ] + ) + logger.info(f"Create topic output: '{output}' with code {code}") return code == 0 def produce_message(self, topic_name: str, message: str): - (code, output) = self.exec_run(["/bin/sh", "-c", f"echo '{message}' | /opt/kafka/bin/kafka-console-producer.sh --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092"]) - logging.info(f"Produce message output: '{output}' with code {code}") + (code, output) = self.exec_run( + [ + "/bin/sh", + "-c", + f"echo '{message}' | /opt/kafka/bin/kafka-console-producer.sh --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092", + ] + ) + logger.info(f"Produce message output: '{output}' with code {code}") return code == 0 def produce_message_with_key(self, topic_name: str, message: str, message_key: str): - (code, output) = self.exec_run(["/bin/sh", "-c", f" echo '{message_key}:{message}' | /opt/kafka/bin/kafka-console-producer.sh --property 'key.separator=:' --property 'parse.key=true' --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092"]) - logging.info(f"Produce message with key output: '{output}' with code {code}") + (code, output) = self.exec_run( + [ + "/bin/sh", + "-c", + f" echo '{message_key}:{message}' | /opt/kafka/bin/kafka-console-producer.sh --property 'key.separator=:' --property 'parse.key=true' --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092", + ] + ) + logger.info(f"Produce message with key output: '{output}' with code {code}") return code == 0 def run_python_in_kafka_helper_docker(self, command: str): - output = self.client.containers.run("minifi-kafka-helper:latest", ["python", "-c", command], remove=True, stdout=True, stderr=True, network=self.network.name) - logging.info(f"Run python in kafka helper docker output: '{output.decode('utf-8')}'") + output = self.client.containers.run( + "minifi-kafka-helper:latest", + ["python", "-c", command], + remove=True, + stdout=True, + stderr=True, + network=self.network.name, + ) + logger.info( + f"Run python in kafka helper docker output: '{output.decode('utf-8')}'" + ) return True def wait_for_kafka_consumer_to_be_registered(self, expected_consumer_count: int): message_regex = "Assignment received from leader.*for group docker_test_group" return wait_for_condition( - condition=lambda: len(re.findall(message_regex, self.get_logs())) >= expected_consumer_count, + condition=lambda: ( + len(re.findall(message_regex, self.get_logs())) + >= expected_consumer_count + ), timeout_seconds=60, bail_condition=lambda: self.exited, - context=None) + context=None, + ) diff --git a/extensions/kafka/tests/features/environment.py b/extensions/kafka/tests/features/environment.py index d7ee24d67c..6723d7c636 100644 --- a/extensions/kafka/tests/features/environment.py +++ b/extensions/kafka/tests/features/environment.py @@ -13,11 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker from containers.kafka_server_container import KafkaServer from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): @@ -25,8 +25,7 @@ def before_all(context): FROM python:3.13-slim-bookworm RUN pip install confluent-kafka""" builder = DockerImageBuilder( - image_tag="minifi-kafka-helper:latest", - dockerfile_content=dockerfile + image_tag="minifi-kafka-helper:latest", dockerfile_content=dockerfile ) builder.build() docker.from_env().images.pull(KafkaServer.IMAGE) diff --git a/extensions/kafka/tests/features/steps/steps.py b/extensions/kafka/tests/features/steps/steps.py index 7417e709f5..28eed334ee 100644 --- a/extensions/kafka/tests/features/steps/steps.py +++ b/extensions/kafka/tests/features/steps/steps.py @@ -16,15 +16,16 @@ import binascii import time -from behave import step, when, given - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from behave import given, step, when +from containers.kafka_server_container import KafkaServer from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.minifi.processor import Processor -from containers.kafka_server_container import KafkaServer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @step("a Kafka server is set up") @@ -35,7 +36,9 @@ def setup_kafka_server(context): @step("ConsumeKafka processor is set up to communicate with that server") def setup_consume_kafka_processor(context): consume_kafka = Processor("ConsumeKafka", "ConsumeKafka") - consume_kafka.add_property("Kafka Brokers", f"kafka-server-{context.scenario_id}:9092") + consume_kafka.add_property( + "Kafka Brokers", f"kafka-server-{context.scenario_id}:9092" + ) consume_kafka.add_property("Topic Names", "ConsumeKafkaTest") consume_kafka.add_property("Topic Name Format", "Names") consume_kafka.add_property("Honor Transactions", "true") @@ -45,13 +48,17 @@ def setup_consume_kafka_processor(context): consume_kafka.add_property("Message Header Encoding", "UTF-8") consume_kafka.add_property("Max Poll Time", "4 sec") consume_kafka.add_property("Session Timeout", "6 sec") - context.get_or_create_default_minifi_container().flow_definition.add_processor(consume_kafka) + context.get_or_create_default_minifi_container().flow_definition.add_processor( + consume_kafka + ) @step("PublishKafka processor is set up to communicate with that server") def setup_publish_kafka_processor(context): publish_kafka = Processor("PublishKafka", "PublishKafka") - publish_kafka.add_property("Known Brokers", f"kafka-server-{context.scenario_id}:9092") + publish_kafka.add_property( + "Known Brokers", f"kafka-server-{context.scenario_id}:9092" + ) publish_kafka.add_property("Client Name", "minifi-client") publish_kafka.add_property("Topic Name", "test") publish_kafka.add_property("Batch Size", "10") @@ -59,7 +66,9 @@ def setup_publish_kafka_processor(context): publish_kafka.add_property("Delivery Guarantee", "1") publish_kafka.add_property("Request Timeout", "10 sec") publish_kafka.add_property("Message Timeout", "12 sec") - context.get_or_create_default_minifi_container().flow_definition.add_processor(publish_kafka) + context.get_or_create_default_minifi_container().flow_definition.add_processor( + publish_kafka + ) @step("the Kafka server is started") @@ -73,25 +82,44 @@ def start_kafka_server(context: MinifiTestContext): def initialize_kafka_topic(context: MinifiTestContext, topic_name: str): kafka_server_container = context.containers["kafka-server"] assert isinstance(kafka_server_container, KafkaServer) - assert kafka_server_container.create_topic(topic_name=topic_name) or kafka_server_container.log_app_output() + assert ( + kafka_server_container.create_topic(topic_name=topic_name) + or kafka_server_container.log_app_output() + ) @when('a message with content "{message}" is published to the "{topic_name}" topic') def publish_message_to_topic(context: MinifiTestContext, message: str, topic_name: str): kafka_server_container = context.containers["kafka-server"] assert isinstance(kafka_server_container, KafkaServer) - assert kafka_server_container.produce_message(topic_name=topic_name, message=message) or kafka_server_container.log_app_output() - - -@step('a message with content "{message}" is published to the "{topic_name}" topic with key "{key}"') -def publish_message_with_key_to_topic(context: MinifiTestContext, message: str, topic_name: str, key: str): + assert ( + kafka_server_container.produce_message(topic_name=topic_name, message=message) + or kafka_server_container.log_app_output() + ) + + +@step( + 'a message with content "{message}" is published to the "{topic_name}" topic with key "{key}"' +) +def publish_message_with_key_to_topic( + context: MinifiTestContext, message: str, topic_name: str, key: str +): kafka_server_container = context.containers["kafka-server"] assert isinstance(kafka_server_container, KafkaServer) - assert kafka_server_container.produce_message_with_key(topic_name=topic_name, message=message, message_key=key) or kafka_server_container.log_app_output() - - -@given("the \"{property_name}\" property of the {processor_name} processor is set to match {key_attribute_encoding} encoded kafka message key \"{message_key}\"") -def set_property_to_match_message_key(context, property_name, processor_name, key_attribute_encoding, message_key): + assert ( + kafka_server_container.produce_message_with_key( + topic_name=topic_name, message=message, message_key=key + ) + or kafka_server_container.log_app_output() + ) + + +@given( + 'the "{property_name}" property of the {processor_name} processor is set to match {key_attribute_encoding} encoded kafka message key "{message_key}"' +) +def set_property_to_match_message_key( + context, property_name, processor_name, key_attribute_encoding, message_key +): if key_attribute_encoding.lower() == "hex": encoded_key = binascii.hexlify(message_key.encode("utf-8")).upper() elif key_attribute_encoding.lower() == "(not set)": @@ -99,11 +127,17 @@ def set_property_to_match_message_key(context, property_name, processor_name, ke else: encoded_key = message_key.encode(key_attribute_encoding) filtering = "${kafka.key:equals('" + encoded_key.decode("utf-8") + "')}" - processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) + processor = ( + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name + ) + ) processor.add_property(property_name, filtering) -@when("the publisher performs a {transaction_type} transaction publishing to the \"{topic_name}\" topic these messages: {messages}") +@when( + 'the publisher performs a {transaction_type} transaction publishing to the "{topic_name}" topic these messages: {messages}' +) def publish_to_topic_transaction_style(context, transaction_type, topic_name, messages): if transaction_type == "SINGLE_COMMITTED_TRANSACTION": python_code = f""" @@ -153,12 +187,21 @@ def publish_to_topic_transaction_style(context, transaction_type, topic_name, me producer.abort_transaction() """ else: - raise Exception("Unknown transaction type.") - assert context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) or context.containers["kafka-server"].log_app_output() - - -@when("a message with content \"{content}\" is published to the \"{topic_name}\" topic with headers \"{semicolon_separated_headers}\"") -def publish_with_headers_to_topic(context, content, topic_name, semicolon_separated_headers): + raise RuntimeError("Unknown transaction type.") + assert ( + context.containers["kafka-server"].run_python_in_kafka_helper_docker( + python_code + ) + or context.containers["kafka-server"].log_app_output() + ) + + +@when( + 'a message with content "{content}" is published to the "{topic_name}" topic with headers "{semicolon_separated_headers}"' +) +def publish_with_headers_to_topic( + context, content, topic_name, semicolon_separated_headers +): python_code = f""" from confluent_kafka import Producer headers = [] @@ -169,10 +212,17 @@ def publish_with_headers_to_topic(context, content, topic_name, semicolon_separa producer.produce("{topic_name}", "{content}".encode("utf-8"), headers=headers) producer.flush(10) """ - assert context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) or context.containers["kafka-server"].log_app_output() + assert ( + context.containers["kafka-server"].run_python_in_kafka_helper_docker( + python_code + ) + or context.containers["kafka-server"].log_app_output() + ) -@when("two messages with content \"{content_one}\" and \"{content_two}\" is published to the \"{topic_name}\" topic") +@when( + 'two messages with content "{content_one}" and "{content_two}" is published to the "{topic_name}" topic' +) def publish_two_messages_to_topic(context, content_one, content_two, topic_name): python_code = f""" from confluent_kafka import Producer @@ -182,10 +232,15 @@ def publish_two_messages_to_topic(context, content_one, content_two, topic_name) producer.produce("{topic_name}", "{content_two}") producer.flush(10) """ - assert context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) or context.containers["kafka-server"].log_app_output() + assert ( + context.containers["kafka-server"].run_python_in_kafka_helper_docker( + python_code + ) + or context.containers["kafka-server"].log_app_output() + ) -@when("{number_of_messages} kafka messages are sent to the topic \"{topic_name}\"") +@when('{number_of_messages} kafka messages are sent to the topic "{topic_name}"') def publish_batch_to_topic(context, number_of_messages, topic_name): python_code = f""" from confluent_kafka import Producer @@ -195,12 +250,20 @@ def publish_batch_to_topic(context, number_of_messages, topic_name): producer.produce("{topic_name}", str(uuid.uuid4()).encode("utf-8")) producer.flush(10) """ - assert context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) or context.containers["kafka-server"].log_app_output() + assert ( + context.containers["kafka-server"].run_python_in_kafka_helper_docker( + python_code + ) + or context.containers["kafka-server"].log_app_output() + ) @when("the Kafka consumer is registered in kafka broker") def wait_for_consumer_registration(context): - assert context.containers["kafka-server"].wait_for_kafka_consumer_to_be_registered(1) or context.containers["kafka-server"].log_app_output() + assert ( + context.containers["kafka-server"].wait_for_kafka_consumer_to_be_registered(1) + or context.containers["kafka-server"].log_app_output() + ) # After the consumer is registered there is still some time needed for consumer-broker synchronization # Unfortunately there are no additional log messages that could be checked for this time.sleep(2) @@ -208,7 +271,10 @@ def wait_for_consumer_registration(context): @when("the Kafka consumer is reregistered in kafka broker") def wait_for_consumer_reregistration(context): - assert context.containers["kafka-server"].wait_for_kafka_consumer_to_be_registered(2) or context.containers["kafka-server"].log_app_output() + assert ( + context.containers["kafka-server"].wait_for_kafka_consumer_to_be_registered(2) + or context.containers["kafka-server"].log_app_output() + ) # After the consumer is registered there is still some time needed for consumer-broker synchronization # Unfortunately there are no additional log messages that could be checked for this time.sleep(2) diff --git a/extensions/kubernetes/tests/features/environment.py b/extensions/kubernetes/tests/features/environment.py index 83408016a8..2420fbb056 100644 --- a/extensions/kubernetes/tests/features/environment.py +++ b/extensions/kubernetes/tests/features/environment.py @@ -13,15 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker import os -from minifi_behave.core.hooks import common_after_scenario, common_before_scenario, get_minifi_container_image +from minifi_behave.core.hooks import ( + common_after_scenario, + common_before_scenario, + get_minifi_container_image, +) from steps.kubernetes_proxy import KubernetesProxy +import docker + def before_feature(context, feature): - if "rpm" in os.environ['MINIFI_TAG_PREFIX']: + if "rpm" in os.environ["MINIFI_TAG_PREFIX"]: feature.skip("This feature is not yet supported on RPM installed images") minifi_image = docker.from_env().images.get(get_minifi_container_image()) diff --git a/extensions/kubernetes/tests/features/steps/kubernetes_proxy.py b/extensions/kubernetes/tests/features/steps/kubernetes_proxy.py index c23a164e35..c0183b9acf 100644 --- a/extensions/kubernetes/tests/features/steps/kubernetes_proxy.py +++ b/extensions/kubernetes/tests/features/steps/kubernetes_proxy.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker import glob import os import platform @@ -25,6 +24,7 @@ from pathlib import Path from textwrap import dedent +import docker KUBERNETES_CONTAINER_NAME = "kind-control-plane" @@ -34,11 +34,15 @@ def __init__(self): self.temp_directory = tempfile.TemporaryDirectory() self.resources_directory = Path(__file__).resolve().parent.parent / "resources" - self.minifi_conf_directory = os.path.join(self.temp_directory.name, 'minifi_conf') + self.minifi_conf_directory = os.path.join( + self.temp_directory.name, "minifi_conf" + ) os.mkdir(self.minifi_conf_directory) - self.kind_binary_path = os.path.join(self.temp_directory.name, 'kind') - self.kind_config_path = os.path.join(self.temp_directory.name, 'kind-config.yml') + self.kind_binary_path = os.path.join(self.temp_directory.name, "kind") + self.kind_config_path = os.path.join( + self.temp_directory.name, "kind-config.yml" + ) self.__download_kind() self.__create_kind_config() @@ -47,16 +51,22 @@ def __init__(self): def __download_kind(self): is_x86 = platform.machine() in ("i386", "AMD64", "x86_64") - download_link = 'https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-amd64' + download_link = "https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-amd64" if not is_x86: - if 'Linux' in platform.system(): - download_link = 'https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-arm64' + if "Linux" in platform.system(): + download_link = "https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-arm64" else: - download_link = 'https://kind.sigs.k8s.io/dl/v0.31.0/kind-darwin-arm64' + download_link = "https://kind.sigs.k8s.io/dl/v0.31.0/kind-darwin-arm64" if not os.path.exists(self.kind_binary_path): - if subprocess.run(['curl', '-Lo', self.kind_binary_path, download_link]).returncode != 0: - raise Exception("Could not download kind") + if ( + subprocess.run( + ["curl", "-Lo", self.kind_binary_path, download_link], + check=False, + ).returncode + != 0 + ): + raise RuntimeError("Could not download kind") os.chmod(self.kind_binary_path, stat.S_IXUSR) def __create_kind_config(self): @@ -73,8 +83,8 @@ def __create_kind_config(self): containerPath: /tmp/minifi_conf """) - with open(self.kind_config_path, 'wb') as config_file: - config_file.write(kind_config.encode('utf-8')) + with open(self.kind_config_path, "wb") as config_file: + config_file.write(kind_config.encode("utf-8")) def write_minifi_conf_file(self, file_name: str, content: str): file_path = os.path.join(self.minifi_conf_directory, file_name) @@ -82,67 +92,108 @@ def write_minifi_conf_file(self, file_name: str, content: str): file.write(content) def create_cluster(self): - if subprocess.run([self.kind_binary_path, 'create', 'cluster', '--config=' + self.kind_config_path]).returncode != 0: - raise Exception("Could not create the kind Kubernetes cluster") + if ( + subprocess.run( + [ + self.kind_binary_path, + "create", + "cluster", + "--config=" + self.kind_config_path, + ], + check=False, + ).returncode + != 0 + ): + raise RuntimeError("Could not create the kind Kubernetes cluster") self.status = "running" def delete_cluster(self): - if subprocess.run([self.kind_binary_path, 'delete', 'cluster']).returncode != 0: - raise Exception("Could not delete the kind Kubernetes cluster") + if ( + subprocess.run( + [self.kind_binary_path, "delete", "cluster"], check=False + ).returncode + != 0 + ): + raise RuntimeError("Could not delete the kind Kubernetes cluster") self.status = "exited" def load_docker_image(self, image_name: str): - if subprocess.run([self.kind_binary_path, 'load', 'docker-image', image_name]).returncode != 0: - raise Exception("Could not load the %s docker image into the kind Kubernetes cluster" % image_name) + if ( + subprocess.run( + [self.kind_binary_path, "load", "docker-image", image_name], + check=False, + ).returncode + != 0 + ): + raise RuntimeError( + f"Could not load the {image_name} docker image into the kind Kubernetes cluster" + ) def create_helper_objects(self): - self.__wait_for_default_service_account('default') - namespaces = self.__create_objects_of_type('namespace') + self.__wait_for_default_service_account("default") + namespaces = self.__create_objects_of_type("namespace") for namespace in namespaces: self.__wait_for_default_service_account(namespace) - self.__create_objects_of_type('dependencies') - self.__create_objects_of_type('helper-pod') - self.__create_objects_of_type('clusterrole') - self.__create_objects_of_type('clusterrolebinding') + self.__create_objects_of_type("dependencies") + self.__create_objects_of_type("helper-pod") + self.__create_objects_of_type("clusterrole") + self.__create_objects_of_type("clusterrolebinding") - self.__wait_for_pod_startup('default', 'hello-world-one') - self.__wait_for_pod_startup('default', 'hello-world-two') - self.__wait_for_pod_startup('kube-system', 'metrics-server') + self.__wait_for_pod_startup("default", "hello-world-one") + self.__wait_for_pod_startup("default", "hello-world-two") + self.__wait_for_pod_startup("kube-system", "metrics-server") def create_minifi_pod(self): - self.__create_objects_of_type('test-pod') - self.__wait_for_pod_startup('daemon', 'minifi') + self.__create_objects_of_type("test-pod") + self.__wait_for_pod_startup("daemon", "minifi") def __wait_for_pod_startup(self, namespace: str, pod_name: str): for i in range(120): if i > 0: time.sleep(1) - (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run(['kubectl', '-n', namespace, 'get', 'pods']) - if code == 0 and re.search(f'{pod_name}.*Running', output.decode('utf-8')): + (code, output) = self.docker_client.containers.get( + KUBERNETES_CONTAINER_NAME + ).exec_run(["kubectl", "-n", namespace, "get", "pods"]) + if code == 0 and re.search(f"{pod_name}.*Running", output.decode("utf-8")): return - raise Exception(f"The pod {namespace}:{pod_name} in the Kubernetes cluster failed to start up") + raise RuntimeError( + f"The pod {namespace}:{pod_name} in the Kubernetes cluster failed to start up" + ) def __wait_for_default_service_account(self, namespace: str): for i in range(120): if i > 0: time.sleep(1) - (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run(['kubectl', '-n', namespace, 'get', 'serviceaccount', 'default']) + (code, _output) = self.docker_client.containers.get( + KUBERNETES_CONTAINER_NAME + ).exec_run(["kubectl", "-n", namespace, "get", "serviceaccount", "default"]) if code == 0: return - raise Exception("Default service account for namespace '%s' not found" % namespace) + raise RuntimeError( + f"Default service account for namespace '{namespace}' not found" + ) def __create_objects_of_type(self, type: str) -> list[str]: found_objects = [] - for full_file_name in glob.iglob(os.path.join(self.resources_directory, f'*.{type}.yml')): + for full_file_name in glob.iglob( + os.path.join(self.resources_directory, f"*.{type}.yml") + ): file_name = os.path.basename(full_file_name) - file_name_in_container = os.path.join('/var/tmp', file_name) + file_name_in_container = os.path.join("/var/tmp", file_name) - (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run(['kubectl', 'apply', '-f', file_name_in_container]) + (code, output) = self.docker_client.containers.get( + KUBERNETES_CONTAINER_NAME + ).exec_run(["kubectl", "apply", "-f", file_name_in_container]) if code != 0: - raise Exception("Could not create kubernetes object from file '%s': %s" % full_file_name, output.decode('utf-8')) - - object_name = file_name.replace(f'.{type}.yml', '') + raise RuntimeError( + "Could not create kubernetes object from file '{}': {}".format( + *full_file_name + ), + output.decode("utf-8"), + ) + + object_name = file_name.replace(f".{type}.yml", "") found_objects.append(object_name) return found_objects @@ -157,8 +208,14 @@ def exec_run(self, command: str) -> tuple[int | None, bytes]: return None, b"The kind Kubernetes cluster is not running." def logs(self) -> bytes: - (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run(['kubectl', '-n', 'daemon', 'logs', 'minifi']) + (code, output) = self.docker_client.containers.get( + KUBERNETES_CONTAINER_NAME + ).exec_run(["kubectl", "-n", "daemon", "logs", "minifi"]) if code == 0: return output else: - raise Exception("Could not get logs from the kind Kubernetes cluster, error %d: %s", code, output.decode('utf-8')) + raise RuntimeError( + "Could not get logs from the kind Kubernetes cluster, error %d: %s", + code, + output.decode("utf-8"), + ) diff --git a/extensions/kubernetes/tests/features/steps/minifi_as_pod_in_kubernetes_cluster.py b/extensions/kubernetes/tests/features/steps/minifi_as_pod_in_kubernetes_cluster.py index 320e896867..f277c4cd30 100644 --- a/extensions/kubernetes/tests/features/steps/minifi_as_pod_in_kubernetes_cluster.py +++ b/extensions/kubernetes/tests/features/steps/minifi_as_pod_in_kubernetes_cluster.py @@ -18,6 +18,8 @@ from minifi_behave.containers.minifi_container import MinifiContainer from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class MinifiAsPodInKubernetesCluster(MinifiContainer): def __init__(self, container_name: str, test_context: MinifiTestContext): @@ -25,10 +27,16 @@ def __init__(self, container_name: str, test_context: MinifiTestContext): self.container = test_context.kubernetes_proxy def deploy(self, context: MinifiTestContext | None) -> bool: - logging.debug('Setting up the kind Kubernetes cluster') - self.container.write_minifi_conf_file("minifi.properties", self._get_properties_file_content()) - self.container.write_minifi_conf_file("minifi-log.properties", self._get_log_properties_file_content()) - self.container.write_minifi_conf_file("config.yml", self.flow_definition.to_yaml()) + logger.debug("Setting up the kind Kubernetes cluster") + self.container.write_minifi_conf_file( + "minifi.properties", self._get_properties_file_content() + ) + self.container.write_minifi_conf_file( + "minifi-log.properties", self._get_log_properties_file_content() + ) + self.container.write_minifi_conf_file( + "config.yml", self.flow_definition.to_yaml() + ) self.container.create_helper_objects() self.container.load_docker_image("apacheminificpp:docker_test") self.container.create_minifi_pod() diff --git a/extensions/kubernetes/tests/features/steps/steps.py b/extensions/kubernetes/tests/features/steps/steps.py index b153cadb07..ddab061327 100644 --- a/extensions/kubernetes/tests/features/steps/steps.py +++ b/extensions/kubernetes/tests/features/steps/steps.py @@ -14,46 +14,88 @@ # limitations under the License. from behave import given - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 - -from minifi_behave.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext -from minifi_behave.minifi.processor import Processor -from minifi_behave.minifi.controller_service import ControllerService from minifi_as_pod_in_kubernetes_cluster import MinifiAsPodInKubernetesCluster +from minifi_behave.core.minifi_test_context import ( + DEFAULT_MINIFI_CONTAINER_NAME, + MinifiTestContext, +) +from minifi_behave.minifi.controller_service import ControllerService +from minifi_behave.minifi.processor import Processor +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) def __ensure_kubernetes_cluster(context: MinifiTestContext): - if DEFAULT_MINIFI_CONTAINER_NAME not in context.containers or not isinstance(context.containers[DEFAULT_MINIFI_CONTAINER_NAME], MinifiAsPodInKubernetesCluster): - context.containers[DEFAULT_MINIFI_CONTAINER_NAME] = MinifiAsPodInKubernetesCluster("kubernetes", context) + if DEFAULT_MINIFI_CONTAINER_NAME not in context.containers or not isinstance( + context.containers[DEFAULT_MINIFI_CONTAINER_NAME], + MinifiAsPodInKubernetesCluster, + ): + context.containers[DEFAULT_MINIFI_CONTAINER_NAME] = ( + MinifiAsPodInKubernetesCluster("kubernetes", context) + ) @given("a {processor_type} processor in a Kubernetes cluster") @given("a {processor_type} processor in the Kubernetes cluster") -def setup_processor_in_kubernetes_cluster(context: MinifiTestContext, processor_type: str): +def setup_processor_in_kubernetes_cluster( + context: MinifiTestContext, processor_type: str +): __ensure_kubernetes_cluster(context) processor = Processor(class_name=processor_type, proc_name=processor_type) - context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) + context.get_or_create_default_minifi_container().flow_definition.add_processor( + processor + ) -def __set_up_the_kubernetes_controller_service(context: MinifiTestContext, processor_name: str, service_property_name: str, properties: dict[str, str]): - kubernetes_controller_service = ControllerService(class_name="KubernetesControllerService", service_name="Kubernetes Controller Service") +def __set_up_the_kubernetes_controller_service( + context: MinifiTestContext, + processor_name: str, + service_property_name: str, + properties: dict[str, str], +): + kubernetes_controller_service = ControllerService( + class_name="KubernetesControllerService", + service_name="Kubernetes Controller Service", + ) kubernetes_controller_service.properties = properties flow = context.get_or_create_default_minifi_container().flow_definition flow.controller_services.append(kubernetes_controller_service) - flow.get_processor(processor_name).add_property(service_property_name, kubernetes_controller_service.name) + flow.get_processor(processor_name).add_property( + service_property_name, kubernetes_controller_service.name + ) -@given("the {processor_name} processor has a {service_property_name} which is a Kubernetes Controller Service") -@given("the {processor_name} processor has an {service_property_name} which is a Kubernetes Controller Service") -def setup_kubernetes_controller_service_for_processor(context: MinifiTestContext, processor_name: str, service_property_name: str): - __set_up_the_kubernetes_controller_service(context, processor_name, service_property_name, {}) +@given( + "the {processor_name} processor has a {service_property_name} which is a Kubernetes Controller Service" +) +@given( + "the {processor_name} processor has an {service_property_name} which is a Kubernetes Controller Service" +) +def setup_kubernetes_controller_service_for_processor( + context: MinifiTestContext, processor_name: str, service_property_name: str +): + __set_up_the_kubernetes_controller_service( + context, processor_name, service_property_name, {} + ) -@given("the {processor_name} processor has a {service_property_name} which is a Kubernetes Controller Service with the \"{property_name}\" property set to \"{property_value}\"") -@given("the {processor_name} processor has an {service_property_name} which is a Kubernetes Controller Service with the \"{property_name}\" property set to \"{property_value}\"") -def setup_kubernetes_controller_service_with_property_for_processor(context: MinifiTestContext, processor_name: str, service_property_name: str, property_name: str, property_value: str): - __set_up_the_kubernetes_controller_service(context, processor_name, service_property_name, {property_name: property_value}) +@given( + 'the {processor_name} processor has a {service_property_name} which is a Kubernetes Controller Service with the "{property_name}" property set to "{property_value}"' +) +@given( + 'the {processor_name} processor has an {service_property_name} which is a Kubernetes Controller Service with the "{property_name}" property set to "{property_value}"' +) +def setup_kubernetes_controller_service_with_property_for_processor( + context: MinifiTestContext, + processor_name: str, + service_property_name: str, + property_name: str, + property_value: str, +): + __set_up_the_kubernetes_controller_service( + context, processor_name, service_property_name, {property_name: property_value} + ) diff --git a/extensions/llamacpp/tests/features/environment.py b/extensions/llamacpp/tests/features/environment.py index d89a597ae6..8d3e4f9f31 100644 --- a/extensions/llamacpp/tests/features/environment.py +++ b/extensions/llamacpp/tests/features/environment.py @@ -12,13 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path from textwrap import dedent -from pathlib import Path from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario -from minifi_behave.core.hooks import get_minifi_container_image +from minifi_behave.core.hooks import ( + common_after_scenario, + common_before_scenario, + get_minifi_container_image, +) from minifi_behave.core.minifi_test_context import MinifiTestContext # These hooks are executed by behave before and after each scenario @@ -28,12 +30,15 @@ def before_all(context: MinifiTestContext): minifi_container_image = get_minifi_container_image() - wget_with_retry_path = Path(__file__).resolve().parent / "resources" / "wget_with_retry.sh" + wget_with_retry_path = ( + Path(__file__).resolve().parent / "resources" / "wget_with_retry.sh" + ) wget_with_retry_content = None with open(wget_with_retry_path, "rb") as f: wget_with_retry_content = f.read() - dockerfile = dedent("""\ + dockerfile = dedent( + """\ FROM {base_image} USER root COPY wget_with_retry.sh /scripts/wget_with_retry.sh @@ -41,12 +46,13 @@ def before_all(context: MinifiTestContext): /scripts/wget_with_retry.sh https://huggingface.co/bartowski/Qwen2-VL-2B-Instruct-GGUF/resolve/main/Qwen2-VL-2B-Instruct-Q3_K_M.gguf {models_path} && \\ /scripts/wget_with_retry.sh https://huggingface.co/bartowski/Qwen2-VL-2B-Instruct-GGUF/resolve/main/mmproj-Qwen2-VL-2B-Instruct-f16.gguf {models_path} USER minificpp - """.format(base_image=minifi_container_image, models_path='/tmp/models')) + """.format(base_image=minifi_container_image, models_path="/tmp/models") + ) builder = DockerImageBuilder( image_tag="apacheminificpp:llama", dockerfile_content=dockerfile, - files_on_context={"wget_with_retry.sh": wget_with_retry_content} + files_on_context={"wget_with_retry.sh": wget_with_retry_content}, ) builder.build() diff --git a/extensions/llamacpp/tests/features/steps/steps.py b/extensions/llamacpp/tests/features/steps/steps.py index 25759880fa..ff5817598b 100644 --- a/extensions/llamacpp/tests/features/steps/steps.py +++ b/extensions/llamacpp/tests/features/steps/steps.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) diff --git a/extensions/lua/tests/features/environment.py b/extensions/lua/tests/features/environment.py index 7e9fcda289..6d98b376da 100644 --- a/extensions/lua/tests/features/environment.py +++ b/extensions/lua/tests/features/environment.py @@ -15,13 +15,12 @@ import os -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario def before_scenario(context, scenario): common_before_scenario(context, scenario) - context.resource_dir = os.path.join(os.path.dirname(__file__), 'resources') + context.resource_dir = os.path.join(os.path.dirname(__file__), "resources") def after_scenario(context, scenario): diff --git a/extensions/lua/tests/features/steps/steps.py b/extensions/lua/tests/features/steps/steps.py index 25759880fa..ff5817598b 100644 --- a/extensions/lua/tests/features/steps/steps.py +++ b/extensions/lua/tests/features/steps/steps.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) diff --git a/extensions/mqtt/tests/features/containers/mqtt_broker_container.py b/extensions/mqtt/tests/features/containers/mqtt_broker_container.py index 8d38663a54..78c65c5662 100644 --- a/extensions/mqtt/tests/features/containers/mqtt_broker_container.py +++ b/extensions/mqtt/tests/features/containers/mqtt_broker_container.py @@ -17,30 +17,37 @@ import re from textwrap import dedent +from docker.errors import ContainerError from minifi_behave.containers.container_linux import LinuxContainer +from minifi_behave.containers.docker_image_builder import DockerImageBuilder from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from docker.errors import ContainerError + +logger = logging.getLogger(__name__) class MqttBrokerContainer(LinuxContainer): IMAGE = "eclipse-mosquitto:2.1.1-alpine" def __init__(self, test_context: MinifiTestContext): - dockerfile = dedent("""\ - FROM {base_image} + dockerfile = dedent( + f"""\ + FROM {MqttBrokerContainer.IMAGE} RUN echo 'log_dest stderr' >> /mosquitto-no-auth.conf CMD ["/usr/sbin/mosquitto", "--verbose", "--config-file", "/mosquitto-no-auth.conf"] - """.format(base_image=MqttBrokerContainer.IMAGE)) + """ + ) builder = DockerImageBuilder( - image_tag="minifi-mqtt-broker:latest", - dockerfile_content=dockerfile + image_tag="minifi-mqtt-broker:latest", dockerfile_content=dockerfile ) builder.build() - super().__init__("minifi-mqtt-broker:latest", f"mqtt-broker-{test_context.scenario_id}", test_context.network) + super().__init__( + "minifi-mqtt-broker:latest", + f"mqtt-broker-{test_context.scenario_id}", + test_context.network, + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -49,17 +56,43 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: re.search(finished_str, self.get_logs()), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context) + context=context, + ) def publish_mqtt_message(self, topic: str, message: str): try: - self.client.containers.run("minifi-mqtt-helper:latest", ["python", "/scripts/publish_mqtt_message.py", self.container_name, topic, message], remove=True, stdout=True, stderr=True, network=self.network.name) + self.client.containers.run( + "minifi-mqtt-helper:latest", + [ + "python", + "/scripts/publish_mqtt_message.py", + self.container_name, + topic, + message, + ], + remove=True, + stdout=True, + stderr=True, + network=self.network.name, + ) return True except ContainerError as e: - stdout = e.stdout.decode("utf-8", errors="replace") if hasattr(e, "stdout") and e.stdout else "" - stderr = e.stderr.decode("utf-8", errors="replace") if hasattr(e, "stderr") and e.stderr else "" - logging.error(f"Failed to publish mqtt message in mqtt helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'") + stdout = ( + e.stdout.decode("utf-8", errors="replace") + if hasattr(e, "stdout") and e.stdout + else "" + ) + stderr = ( + e.stderr.decode("utf-8", errors="replace") + if hasattr(e, "stderr") and e.stderr + else "" + ) + logger.error( + f"Failed to publish mqtt message in mqtt helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'" + ) return False except Exception as e: - logging.error(f"Unexpected error while publishing mqtt message in mqtt helper docker: '{e}'") + logger.error( + f"Unexpected error while publishing mqtt message in mqtt helper docker: '{e}'" + ) return False diff --git a/extensions/mqtt/tests/features/environment.py b/extensions/mqtt/tests/features/environment.py index 6cabeecf03..92d4388a5e 100644 --- a/extensions/mqtt/tests/features/environment.py +++ b/extensions/mqtt/tests/features/environment.py @@ -13,16 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker -from containers.mqtt_broker_container import MqttBrokerContainer from pathlib import Path + +from containers.mqtt_broker_container import MqttBrokerContainer from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): - check_log_lines_path = Path(__file__).resolve().parent / "resources" / "publish_mqtt_message.py" + check_log_lines_path = ( + Path(__file__).resolve().parent / "resources" / "publish_mqtt_message.py" + ) check_log_lines_content = None with open(check_log_lines_path, "rb") as f: check_log_lines_content = f.read() @@ -33,7 +36,7 @@ def before_all(context): mqtt_helper_builder = DockerImageBuilder( image_tag="minifi-mqtt-helper:latest", dockerfile_content=dockerfile, - files_on_context={"publish_mqtt_message.py": check_log_lines_content} + files_on_context={"publish_mqtt_message.py": check_log_lines_content}, ) mqtt_helper_builder.build() docker.from_env().images.pull(MqttBrokerContainer.IMAGE) diff --git a/extensions/mqtt/tests/features/resources/publish_mqtt_message.py b/extensions/mqtt/tests/features/resources/publish_mqtt_message.py index 6fe709acf5..88cb17dda4 100644 --- a/extensions/mqtt/tests/features/resources/publish_mqtt_message.py +++ b/extensions/mqtt/tests/features/resources/publish_mqtt_message.py @@ -12,9 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import paho.mqtt.client as mqtt import sys +import paho.mqtt.client as mqtt + def publish_test_mqtt_message(host: str, topic: str, message: str): client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, "docker_test_client_id") diff --git a/extensions/mqtt/tests/features/steps/steps.py b/extensions/mqtt/tests/features/steps/steps.py index 1bce835ae5..11d0375896 100644 --- a/extensions/mqtt/tests/features/steps/steps.py +++ b/extensions/mqtt/tests/features/steps/steps.py @@ -14,37 +14,53 @@ # limitations under the License. import re -from behave import given, step, then, when - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext -from minifi_behave.minifi.processor import Processor -from minifi_behave.core.helpers import wait_for_condition +from behave import given, step, then, when from containers.mqtt_broker_container import MqttBrokerContainer - - -@given("a {processor_name} processor set up to communicate with an MQTT broker instance in the \"{container_name}\" flow") -def setup_mqtt_processor_in_flow(context: MinifiTestContext, processor_name: str, container_name: str): +from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import ( + DEFAULT_MINIFI_CONTAINER_NAME, + MinifiTestContext, +) +from minifi_behave.minifi.processor import Processor +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) + + +@given( + 'a {processor_name} processor set up to communicate with an MQTT broker instance in the "{container_name}" flow' +) +def setup_mqtt_processor_in_flow( + context: MinifiTestContext, processor_name: str, container_name: str +): processor = Processor(processor_name, processor_name) - processor.add_property('Broker URI', f'mqtt-broker-{context.scenario_id}:1883') - processor.add_property('Topic', 'testtopic') - if processor_name == 'PublishMQTT': - processor.add_property('Client ID', 'publisher-client') - elif processor_name == 'ConsumeMQTT': - processor.add_property('Client ID', 'consumer-client') + processor.add_property("Broker URI", f"mqtt-broker-{context.scenario_id}:1883") + processor.add_property("Topic", "testtopic") + if processor_name == "PublishMQTT": + processor.add_property("Client ID", "publisher-client") + elif processor_name == "ConsumeMQTT": + processor.add_property("Client ID", "consumer-client") else: - raise ValueError(f"Unknown processor to communicate with MQTT broker: {processor_name}") + raise ValueError( + f"Unknown processor to communicate with MQTT broker: {processor_name}" + ) - context.get_or_create_minifi_container(container_name).flow_definition.add_processor(processor) + context.get_or_create_minifi_container( + container_name + ).flow_definition.add_processor(processor) -@given("a {processor_name} processor set up to communicate with an MQTT broker instance") +@given( + "a {processor_name} processor set up to communicate with an MQTT broker instance" +) def setup_mqtt_processor(context: MinifiTestContext, processor_name: str): - context.execute_steps(f'given a {processor_name} processor set up to communicate with an MQTT broker instance in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + context.execute_steps( + f'given a {processor_name} processor set up to communicate with an MQTT broker instance in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @step("an MQTT broker is started") @@ -54,23 +70,38 @@ def start_mqtt_broker(context: MinifiTestContext): @then('the MQTT broker has a log line matching "{log_line_regex}"') -def verify_mqtt_broker_log_line_matches(context: MinifiTestContext, log_line_regex: str): +def verify_mqtt_broker_log_line_matches( + context: MinifiTestContext, log_line_regex: str +): assert wait_for_condition( - condition=lambda: re.search(log_line_regex, context.containers["mqtt-broker"].get_logs()), + condition=lambda: re.search( + log_line_regex, context.containers["mqtt-broker"].get_logs() + ), timeout_seconds=60, bail_condition=lambda: context.containers["mqtt-broker"].exited, - context=None) + context=None, + ) @then('the MQTT broker has {log_count:d} log lines matching "{log_line_regex}"') -def verify_mqtt_broker_log_line_count_matches(context: MinifiTestContext, log_count: int, log_line_regex: str): +def verify_mqtt_broker_log_line_count_matches( + context: MinifiTestContext, log_count: int, log_line_regex: str +): assert wait_for_condition( - condition=lambda: len(re.findall(log_line_regex, context.containers["mqtt-broker"].get_logs())) == log_count, + condition=lambda: ( + len( + re.findall(log_line_regex, context.containers["mqtt-broker"].get_logs()) + ) + == log_count + ), timeout_seconds=60, bail_condition=lambda: context.containers["mqtt-broker"].exited, - context=None) + context=None, + ) -@when("a test message \"{message}\" is published to the MQTT broker on topic \"{topic}\"") -def publish_test_message_to_mqtt_broker(context: MinifiTestContext, message: str, topic: str): +@when('a test message "{message}" is published to the MQTT broker on topic "{topic}"') +def publish_test_message_to_mqtt_broker( + context: MinifiTestContext, message: str, topic: str +): assert context.containers["mqtt-broker"].publish_mqtt_message(topic, message) diff --git a/extensions/opc/tests/features/containers/opc_ua_server_container.py b/extensions/opc/tests/features/containers/opc_ua_server_container.py index 5bb8a233bc..ddd79c4bb1 100644 --- a/extensions/opc/tests/features/containers/opc_ua_server_container.py +++ b/extensions/opc/tests/features/containers/opc_ua_server_container.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional + from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext @@ -22,8 +22,15 @@ class OPCUAServerContainer(LinuxContainer): IMAGE = "lordgamez/open62541:1.5.2" - def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None): - super().__init__(OPCUAServerContainer.IMAGE, f"opcua-server-{test_context.scenario_id}", test_context.network, command=command) + def __init__( + self, test_context: MinifiTestContext, command: list[str] | None = None + ): + super().__init__( + OPCUAServerContainer.IMAGE, + f"opcua-server-{test_context.scenario_id}", + test_context.network, + command=command, + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -32,4 +39,5 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context) + context=context, + ) diff --git a/extensions/opc/tests/features/environment.py b/extensions/opc/tests/features/environment.py index 76f86f8b86..0f9837f491 100644 --- a/extensions/opc/tests/features/environment.py +++ b/extensions/opc/tests/features/environment.py @@ -14,10 +14,11 @@ # limitations under the License. import os -import docker + from containers.opc_ua_server_container import OPCUAServerContainer -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): @@ -26,7 +27,7 @@ def before_all(context): def before_scenario(context, scenario): common_before_scenario(context, scenario) - context.resource_dir = os.path.join(os.path.dirname(__file__), 'resources') + context.resource_dir = os.path.join(os.path.dirname(__file__), "resources") def after_scenario(context, scenario): diff --git a/extensions/opc/tests/features/steps/steps.py b/extensions/opc/tests/features/steps/steps.py index 58f2a1ea1b..05f80b81d1 100644 --- a/extensions/opc/tests/features/steps/steps.py +++ b/extensions/opc/tests/features/steps/steps.py @@ -13,22 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import logging import os -import tempfile -import docker -import io import tarfile -import humanfriendly +import tempfile +import humanfriendly from behave import given, step, then -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.helpers import wait_for_condition from containers.opc_ua_server_container import OPCUAServerContainer +from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) + +import docker + +logger = logging.getLogger(__name__) @step("an OPC UA server is set up") @@ -38,10 +43,14 @@ def setup_opcua_server(context: MinifiTestContext): @step("an OPC UA server is set up with access control") def setup_opcua_server_with_access_control(context: MinifiTestContext): - context.containers["opcua-server-access"] = OPCUAServerContainer(context, command=["/opt/open62541/examples/access_control_server"]) + context.containers["opcua-server-access"] = OPCUAServerContainer( + context, command=["/opt/open62541/examples/access_control_server"] + ) -@then("the OPC UA server logs contain the following message: \"{log_message}\" in less than {duration}") +@then( + 'the OPC UA server logs contain the following message: "{log_message}" in less than {duration}' +) def verify_opcua_server_logs_contain_message(context, log_message, duration): timeout_seconds = humanfriendly.parse_timespan(duration) opcua_container = context.containers["opcua-server"] @@ -50,7 +59,8 @@ def verify_opcua_server_logs_contain_message(context, log_message, duration): condition=lambda: log_message in opcua_container.get_logs(), timeout_seconds=timeout_seconds, bail_condition=lambda: opcua_container.exited, - context=context) + context=context, + ) def _copy_file_from_docker_image(image_name: str, file_path: str, output_path: str): @@ -69,18 +79,38 @@ def _copy_file_from_docker_image(image_name: str, file_path: str, output_path: s return True except Exception as e: - logging.error(f"Error copying file {file_path} from Docker image {image_name}: {e}") + logger.error( + f"Error copying file {file_path} from Docker image {image_name}: {e}" + ) return False finally: container.remove(force=True) -@given('the OPC UA server certificate files are placed in the "{directory}" directory in the MiNiFi container "{container_name}"') -def place_opcua_certificate_files_in_minifi_container(context: MinifiTestContext, directory: str, container_name: str): +@given( + 'the OPC UA server certificate files are placed in the "{directory}" directory in the MiNiFi container "{container_name}"' +) +def place_opcua_certificate_files_in_minifi_container( + context: MinifiTestContext, directory: str, container_name: str +): if not hasattr(context, "opcua_cert_temp_dir"): context.opcua_cert_temp_dir = tempfile.TemporaryDirectory() - _copy_file_from_docker_image(OPCUAServerContainer.IMAGE, "/opt/open62541/pki/created/server_cert.der", os.path.join(context.opcua_cert_temp_dir.name, "server_cert.der")) - _copy_file_from_docker_image(OPCUAServerContainer.IMAGE, "/opt/open62541/pki/created/server_key.der", os.path.join(context.opcua_cert_temp_dir.name, "server_key.der")) - - context.get_or_create_minifi_container(container_name).add_host_file(os.path.join(context.opcua_cert_temp_dir.name, "server_cert.der"), os.path.join(directory, "opcua_client_cert.der")) - context.get_or_create_minifi_container(container_name).add_host_file(os.path.join(context.opcua_cert_temp_dir.name, "server_key.der"), os.path.join(directory, "opcua_client_key.der")) + _copy_file_from_docker_image( + OPCUAServerContainer.IMAGE, + "/opt/open62541/pki/created/server_cert.der", + os.path.join(context.opcua_cert_temp_dir.name, "server_cert.der"), + ) + _copy_file_from_docker_image( + OPCUAServerContainer.IMAGE, + "/opt/open62541/pki/created/server_key.der", + os.path.join(context.opcua_cert_temp_dir.name, "server_key.der"), + ) + + context.get_or_create_minifi_container(container_name).add_host_file( + os.path.join(context.opcua_cert_temp_dir.name, "server_cert.der"), + os.path.join(directory, "opcua_client_cert.der"), + ) + context.get_or_create_minifi_container(container_name).add_host_file( + os.path.join(context.opcua_cert_temp_dir.name, "server_key.der"), + os.path.join(directory, "opcua_client_key.der"), + ) diff --git a/extensions/prometheus/tests/features/containers/prometheus_container.py b/extensions/prometheus/tests/features/containers/prometheus_container.py index ee962d8ebf..7c95c0605f 100644 --- a/extensions/prometheus/tests/features/containers/prometheus_container.py +++ b/extensions/prometheus/tests/features/containers/prometheus_container.py @@ -14,50 +14,86 @@ # limitations under the License. import logging + +from minifi_behave.containers.container_linux import LinuxContainer +from minifi_behave.containers.file import File from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.ssl_utils import make_cert_without_extended_usage -from minifi_behave.containers.file import File -from minifi_behave.core.ssl_utils import dump_cert, dump_key -from minifi_behave.containers.container_linux import LinuxContainer +from minifi_behave.core.ssl_utils import ( + dump_cert, + dump_key, + make_cert_without_extended_usage, +) + +logger = logging.getLogger(__name__) class PrometheusContainer(LinuxContainer): IMAGE = "prom/prometheus:v3.9.1" def __init__(self, test_context: MinifiTestContext, ssl: bool = False): - super().__init__(PrometheusContainer.IMAGE, f"prometheus-{test_context.scenario_id}", test_context.network) + super().__init__( + PrometheusContainer.IMAGE, + f"prometheus-{test_context.scenario_id}", + test_context.network, + ) self.scenario_id = test_context.scenario_id extra_ssl_settings = "" if ssl: - prometheus_cert, prometheus_key = make_cert_without_extended_usage(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + prometheus_cert, prometheus_key = make_cert_without_extended_usage( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/etc/prometheus/certs/root-ca.pem", root_ca_content, permissions=0o644)) + self.files.append( + File( + "/etc/prometheus/certs/root-ca.pem", + root_ca_content, + permissions=0o644, + ) + ) prometheus_cert_content = dump_cert(prometheus_cert) - self.files.append(File("/etc/prometheus/certs/prometheus.crt", prometheus_cert_content, permissions=0o644)) + self.files.append( + File( + "/etc/prometheus/certs/prometheus.crt", + prometheus_cert_content, + permissions=0o644, + ) + ) prometheus_key_content = dump_key(prometheus_key) - self.files.append(File("/etc/prometheus/certs/prometheus.key", prometheus_key_content, permissions=0o644)) + self.files.append( + File( + "/etc/prometheus/certs/prometheus.key", + prometheus_key_content, + permissions=0o644, + ) + ) extra_ssl_settings = """ scheme: https tls_config: ca_file: /etc/prometheus/certs/root-ca.pem """ - prometheus_yml_content = """ + prometheus_yml_content = f""" global: scrape_interval: 2s evaluation_interval: 15s scrape_configs: - job_name: "minifi" static_configs: - - targets: ["minifi-primary-{scenario_id}:9936"] + - targets: ["minifi-primary-{test_context.scenario_id}:9936"] {extra_ssl_settings} -""".format(scenario_id=test_context.scenario_id, extra_ssl_settings=extra_ssl_settings) +""" - self.files.append(File("/etc/prometheus/prometheus.yml", prometheus_yml_content, permissions=0o666)) + self.files.append( + File( + "/etc/prometheus/prometheus.yml", + prometheus_yml_content, + permissions=0o666, + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -66,29 +102,71 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context + context=context, ) def check_metric_class_on_prometheus(self, metric_class: str) -> bool: try: - self.client.containers.run("minifi-prometheus-helper:latest", ["python", "/scripts/prometheus_checker.py", "--prometheus-host", self.container_name, "--metric-class", metric_class], remove=True, network=self.network.name) + self.client.containers.run( + "minifi-prometheus-helper:latest", + [ + "python", + "/scripts/prometheus_checker.py", + "--prometheus-host", + self.container_name, + "--metric-class", + metric_class, + ], + remove=True, + network=self.network.name, + ) return True except Exception: - logging.error(f"Failed to check metric class {metric_class} on Prometheus") + logger.error(f"Failed to check metric class {metric_class} on Prometheus") return False - def check_processor_metric_on_prometheus(self, metric_class: str, processor_name: str) -> bool: + def check_processor_metric_on_prometheus( + self, metric_class: str, processor_name: str + ) -> bool: try: - self.client.containers.run("minifi-prometheus-helper:latest", ["python", "/scripts/prometheus_checker.py", "--prometheus-host", self.container_name, "--metric-class", metric_class, "--processor-name", processor_name], remove=True, network=self.network.name) + self.client.containers.run( + "minifi-prometheus-helper:latest", + [ + "python", + "/scripts/prometheus_checker.py", + "--prometheus-host", + self.container_name, + "--metric-class", + metric_class, + "--processor-name", + processor_name, + ], + remove=True, + network=self.network.name, + ) return True except Exception: - logging.error(f"Failed to check processor metric class {metric_class} for processor {processor_name} on Prometheus") + logger.error( + f"Failed to check processor metric class {metric_class} for processor {processor_name} on Prometheus" + ) return False def check_all_metric_types_defined_once(self) -> bool: try: - self.client.containers.run("minifi-prometheus-helper:latest", ["python", "/scripts/prometheus_checker.py", "--verify-metrics-defined-once", f"minifi-primary-{self.scenario_id}"], remove=True, network=self.network.name) + self.client.containers.run( + "minifi-prometheus-helper:latest", + [ + "python", + "/scripts/prometheus_checker.py", + "--verify-metrics-defined-once", + f"minifi-primary-{self.scenario_id}", + ], + remove=True, + network=self.network.name, + ) return True except Exception: - logging.error("Failed check that all metric types are defined once on Prometheus") + logger.error( + "Failed check that all metric types are defined once on Prometheus" + ) return False diff --git a/extensions/prometheus/tests/features/environment.py b/extensions/prometheus/tests/features/environment.py index c62553bdf7..4c9f91739a 100644 --- a/extensions/prometheus/tests/features/environment.py +++ b/extensions/prometheus/tests/features/environment.py @@ -12,16 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import docker from pathlib import Path -from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario + from containers.prometheus_container import PrometheusContainer +from minifi_behave.containers.docker_image_builder import DockerImageBuilder +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): - check_log_lines_path = Path(__file__).resolve().parent / "resources" / "prometheus_checker.py" + check_log_lines_path = ( + Path(__file__).resolve().parent / "resources" / "prometheus_checker.py" + ) check_log_lines_content = None with open(check_log_lines_path, "rb") as f: check_log_lines_content = f.read() @@ -32,7 +35,7 @@ def before_all(context): prometheus_helper_builder = DockerImageBuilder( image_tag="minifi-prometheus-helper:latest", dockerfile_content=dockerfile, - files_on_context={"prometheus_checker.py": check_log_lines_content} + files_on_context={"prometheus_checker.py": check_log_lines_content}, ) prometheus_helper_builder.build() docker.from_env().images.pull(PrometheusContainer.IMAGE) diff --git a/extensions/prometheus/tests/features/resources/prometheus_checker.py b/extensions/prometheus/tests/features/resources/prometheus_checker.py index e37662890c..204b0c18c5 100644 --- a/extensions/prometheus/tests/features/resources/prometheus_checker.py +++ b/extensions/prometheus/tests/features/resources/prometheus_checker.py @@ -14,13 +14,16 @@ # limitations under the License. import argparse import sys + import requests from prometheus_api_client import PrometheusConnect class PrometheusChecker: def __init__(self, prometheus_host: str): - self.prometheus_client = PrometheusConnect(url=f"http://{prometheus_host}:9090", disable_ssl=True) + self.prometheus_client = PrometheusConnect( + url=f"http://{prometheus_host}:9090", disable_ssl=True + ) def verify_processor_metric(self, metric_class: str, processor_name: str) -> bool: if metric_class == "GetFileMetrics": @@ -40,80 +43,237 @@ def verify_metric_class(self, metric_class: str) -> bool: elif metric_class == "AgentStatus": return self._verify_agent_status_metrics() else: - raise Exception("Metric class '%s' verification is not implemented" % metric_class) + raise RuntimeError( + f"Metric class '{metric_class}' verification is not implemented" + ) def _verify_repository_metrics(self) -> bool: - label_list = [{'repository_name': 'provenance'}, {'repository_name': 'flowfile'}, {'repository_name': 'content'}] + label_list = [ + {"repository_name": "provenance"}, + {"repository_name": "flowfile"}, + {"repository_name": "content"}, + ] # Only flowfile and content repositories are using rocksdb by default, so rocksdb specific metrics are only present there - return all((self._verify_metrics_exist(['minifi_is_running', 'minifi_is_full', 'minifi_repository_size_bytes', 'minifi_max_repository_size_bytes', 'minifi_repository_entry_count'], 'RepositoryMetrics', labels) for labels in label_list)) and \ - all((self._verify_metric_larger_than_zero('minifi_repository_size_bytes', 'RepositoryMetrics', labels) for labels in label_list[1:3])) and \ - all((self._verify_metrics_exist(['minifi_rocksdb_table_readers_size_bytes', 'minifi_rocksdb_all_memory_tables_size_bytes'], 'RepositoryMetrics', labels) for labels in label_list[1:3])) + return ( + all( + self._verify_metrics_exist( + [ + "minifi_is_running", + "minifi_is_full", + "minifi_repository_size_bytes", + "minifi_max_repository_size_bytes", + "minifi_repository_entry_count", + ], + "RepositoryMetrics", + labels, + ) + for labels in label_list + ) + and all( + self._verify_metric_larger_than_zero( + "minifi_repository_size_bytes", "RepositoryMetrics", labels + ) + for labels in label_list[1:3] + ) + and all( + self._verify_metrics_exist( + [ + "minifi_rocksdb_table_readers_size_bytes", + "minifi_rocksdb_all_memory_tables_size_bytes", + ], + "RepositoryMetrics", + labels, + ) + for labels in label_list[1:3] + ) + ) def _verify_queue_metrics(self) -> bool: - return self._verify_metrics_exist(['minifi_queue_data_size', 'minifi_queue_data_size_max', 'minifi_queue_size', 'minifi_queue_size_max'], 'QueueMetrics') + return self._verify_metrics_exist( + [ + "minifi_queue_data_size", + "minifi_queue_data_size_max", + "minifi_queue_size", + "minifi_queue_size_max", + ], + "QueueMetrics", + ) - def _verify_general_processor_metrics(self, metric_class: str, processor_name: str) -> bool: - labels = {'processor_name': processor_name} - return self._verify_metrics_exist(['minifi_average_onTrigger_runtime_milliseconds', 'minifi_last_onTrigger_runtime_milliseconds', - 'minifi_average_session_commit_runtime_milliseconds', 'minifi_last_session_commit_runtime_milliseconds', - 'minifi_incoming_flow_files', 'minifi_incoming_bytes', 'minifi_bytes_read', 'minifi_bytes_written'], metric_class, labels) and \ - self._verify_metrics_larger_than_zero(['minifi_onTrigger_invocations', 'minifi_transferred_flow_files', 'minifi_transferred_to_success', - 'minifi_transferred_bytes', 'minifi_processing_nanos'], - metric_class, labels) + def _verify_general_processor_metrics( + self, metric_class: str, processor_name: str + ) -> bool: + labels = {"processor_name": processor_name} + return self._verify_metrics_exist( + [ + "minifi_average_onTrigger_runtime_milliseconds", + "minifi_last_onTrigger_runtime_milliseconds", + "minifi_average_session_commit_runtime_milliseconds", + "minifi_last_session_commit_runtime_milliseconds", + "minifi_incoming_flow_files", + "minifi_incoming_bytes", + "minifi_bytes_read", + "minifi_bytes_written", + ], + metric_class, + labels, + ) and self._verify_metrics_larger_than_zero( + [ + "minifi_onTrigger_invocations", + "minifi_transferred_flow_files", + "minifi_transferred_to_success", + "minifi_transferred_bytes", + "minifi_processing_nanos", + ], + metric_class, + labels, + ) def _verify_getfile_metrics(self, metric_class: str, processor_name: str) -> bool: - labels = {'processor_name': processor_name} - return self._verify_general_processor_metrics(metric_class, processor_name) and \ - self._verify_metrics_exist(['minifi_input_bytes', 'minifi_accepted_files'], metric_class, labels) + labels = {"processor_name": processor_name} + return self._verify_general_processor_metrics( + metric_class, processor_name + ) and self._verify_metrics_exist( + ["minifi_input_bytes", "minifi_accepted_files"], metric_class, labels + ) def _verify_flow_information_metrics(self) -> bool: - return self._verify_metrics_exist(['minifi_queue_data_size', 'minifi_queue_data_size_max', 'minifi_queue_size', 'minifi_queue_size_max', - 'minifi_bytes_read', 'minifi_bytes_written', 'minifi_flow_files_in', 'minifi_flow_files_out', 'minifi_bytes_in', 'minifi_bytes_out', - 'minifi_invocations', 'minifi_processing_nanos'], 'FlowInformation') and \ - self._verify_metric_exists('minifi_is_running', 'FlowInformation', {'component_name': 'FlowController'}) + return self._verify_metrics_exist( + [ + "minifi_queue_data_size", + "minifi_queue_data_size_max", + "minifi_queue_size", + "minifi_queue_size_max", + "minifi_bytes_read", + "minifi_bytes_written", + "minifi_flow_files_in", + "minifi_flow_files_out", + "minifi_bytes_in", + "minifi_bytes_out", + "minifi_invocations", + "minifi_processing_nanos", + ], + "FlowInformation", + ) and self._verify_metric_exists( + "minifi_is_running", "FlowInformation", {"component_name": "FlowController"} + ) def _verify_device_info_node_metrics(self) -> bool: - return self._verify_metrics_exist(['minifi_physical_mem', 'minifi_memory_usage', 'minifi_cpu_utilization', 'minifi_cpu_load_average'], 'DeviceInfoNode') + return self._verify_metrics_exist( + [ + "minifi_physical_mem", + "minifi_memory_usage", + "minifi_cpu_utilization", + "minifi_cpu_load_average", + ], + "DeviceInfoNode", + ) def _verify_agent_status_metrics(self) -> bool: - label_list = [{'repository_name': 'flowfile'}, {'repository_name': 'content'}] + label_list = [{"repository_name": "flowfile"}, {"repository_name": "content"}] # Only flowfile and content repositories are using rocksdb by default, so rocksdb specific metrics are only present there for labels in label_list: - if not (self._verify_metric_exists('minifi_is_running', 'AgentStatus', labels) - and self._verify_metric_exists('minifi_is_full', 'AgentStatus', labels) - and self._verify_metric_exists('minifi_max_repository_size_bytes', 'AgentStatus', labels) - and self._verify_metric_larger_than_zero('minifi_repository_size_bytes', 'AgentStatus', labels) - and self._verify_metric_exists('minifi_repository_entry_count', 'AgentStatus', labels) - and self._verify_metric_exists('minifi_rocksdb_table_readers_size_bytes', 'AgentStatus', labels) - and self._verify_metric_exists('minifi_rocksdb_all_memory_tables_size_bytes', 'AgentStatus', labels)): + if not ( + self._verify_metric_exists("minifi_is_running", "AgentStatus", labels) + and self._verify_metric_exists("minifi_is_full", "AgentStatus", labels) + and self._verify_metric_exists( + "minifi_max_repository_size_bytes", "AgentStatus", labels + ) + and self._verify_metric_larger_than_zero( + "minifi_repository_size_bytes", "AgentStatus", labels + ) + and self._verify_metric_exists( + "minifi_repository_entry_count", "AgentStatus", labels + ) + and self._verify_metric_exists( + "minifi_rocksdb_table_readers_size_bytes", "AgentStatus", labels + ) + and self._verify_metric_exists( + "minifi_rocksdb_all_memory_tables_size_bytes", "AgentStatus", labels + ) + ): return False # provenance repository is NoOpRepository by default which has zero size - if not (self._verify_metric_exists('minifi_is_running', 'AgentStatus', {'repository_name': 'provenance'}) - and self._verify_metric_exists('minifi_is_full', 'AgentStatus', {'repository_name': 'provenance'}) - and self._verify_metric_exists('minifi_max_repository_size_bytes', 'AgentStatus', {'repository_name': 'provenance'}) - and self._verify_metric_exists('minifi_repository_size_bytes', 'AgentStatus', {'repository_name': 'provenance'}) - and self._verify_metric_exists('minifi_repository_entry_count', 'AgentStatus', {'repository_name': 'provenance'})): + if not ( + self._verify_metric_exists( + "minifi_is_running", "AgentStatus", {"repository_name": "provenance"} + ) + and self._verify_metric_exists( + "minifi_is_full", "AgentStatus", {"repository_name": "provenance"} + ) + and self._verify_metric_exists( + "minifi_max_repository_size_bytes", + "AgentStatus", + {"repository_name": "provenance"}, + ) + and self._verify_metric_exists( + "minifi_repository_size_bytes", + "AgentStatus", + {"repository_name": "provenance"}, + ) + and self._verify_metric_exists( + "minifi_repository_entry_count", + "AgentStatus", + {"repository_name": "provenance"}, + ) + ): return False - return self._verify_metric_exists('minifi_uptime_milliseconds', 'AgentStatus') and \ - self._verify_metric_exists('minifi_agent_memory_usage_bytes', 'AgentStatus') and \ - self._verify_metric_exists('minifi_agent_cpu_utilization', 'AgentStatus') + return ( + self._verify_metric_exists("minifi_uptime_milliseconds", "AgentStatus") + and self._verify_metric_exists( + "minifi_agent_memory_usage_bytes", "AgentStatus" + ) + and self._verify_metric_exists( + "minifi_agent_cpu_utilization", "AgentStatus" + ) + ) - def _verify_metric_exists(self, metric_name: str, metric_class: str, labels: dict = {}) -> bool: - labels['metric_class'] = metric_class - labels['agent_identifier'] = "Agent1" - return len(self.prometheus_client.get_current_metric_value(metric_name=metric_name, label_config=labels)) > 0 + def _verify_metric_exists( + self, metric_name: str, metric_class: str, labels: dict | None = None + ) -> bool: + if labels is None: + labels = {} + labels["metric_class"] = metric_class + labels["agent_identifier"] = "Agent1" + return ( + len( + self.prometheus_client.get_current_metric_value( + metric_name=metric_name, label_config=labels + ) + ) + > 0 + ) - def _verify_metrics_exist(self, metric_names: list, metric_class: str, labels: dict = {}) -> bool: - return all((self._verify_metric_exists(metric_name, metric_class, labels) for metric_name in metric_names)) + def _verify_metrics_exist( + self, metric_names: list, metric_class: str, labels: dict | None = None + ) -> bool: + if labels is None: + labels = {} + return all( + self._verify_metric_exists(metric_name, metric_class, labels) + for metric_name in metric_names + ) - def _verify_metric_larger_than_zero(self, metric_name: str, metric_class: str, labels: dict = {}) -> bool: - labels['metric_class'] = metric_class - result = self.prometheus_client.get_current_metric_value(metric_name=metric_name, label_config=labels) - return len(result) > 0 and int(result[0]['value'][1]) > 0 + def _verify_metric_larger_than_zero( + self, metric_name: str, metric_class: str, labels: dict | None = None + ) -> bool: + if labels is None: + labels = {} + labels["metric_class"] = metric_class + result = self.prometheus_client.get_current_metric_value( + metric_name=metric_name, label_config=labels + ) + return len(result) > 0 and int(result[0]["value"][1]) > 0 - def _verify_metrics_larger_than_zero(self, metric_names: list, metric_class: str, labels: dict = {}) -> bool: - return all((self._verify_metric_larger_than_zero(metric_name, metric_class, labels) for metric_name in metric_names)) + def _verify_metrics_larger_than_zero( + self, metric_names: list, metric_class: str, labels: dict | None = None + ) -> bool: + if labels is None: + labels = {} + return all( + self._verify_metric_larger_than_zero(metric_name, metric_class, labels) + for metric_name in metric_names + ) def verify_all_metric_types_are_defined_once(prometheus_target: str) -> bool: @@ -133,17 +293,30 @@ def verify_all_metric_types_are_defined_once(prometheus_target: str) -> bool: if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Prometheus Checker') - parser.add_argument('--verify-metrics-defined-once', type=str, required=False, help='Prometheus target to verify all defined metrics are unique') - parser.add_argument('--prometheus-host', type=str, required=False, help='Prometheus server host') - parser.add_argument('--metric-class', type=str, required=False, help='Metric class to verify') - parser.add_argument('--processor-name', type=str, required=False, help='Processor name to verify') + parser = argparse.ArgumentParser(description="Prometheus Checker") + parser.add_argument( + "--verify-metrics-defined-once", + type=str, + required=False, + help="Prometheus target to verify all defined metrics are unique", + ) + parser.add_argument( + "--prometheus-host", type=str, required=False, help="Prometheus server host" + ) + parser.add_argument( + "--metric-class", type=str, required=False, help="Metric class to verify" + ) + parser.add_argument( + "--processor-name", type=str, required=False, help="Processor name to verify" + ) args = parser.parse_args() checker = PrometheusChecker(args.prometheus_host) if args.verify_metrics_defined_once: - if not verify_all_metric_types_are_defined_once(args.verify_metrics_defined_once): + if not verify_all_metric_types_are_defined_once( + args.verify_metrics_defined_once + ): sys.exit(1) elif args.processor_name and args.metric_class: if not checker.verify_processor_metric(args.metric_class, args.processor_name): diff --git a/extensions/prometheus/tests/features/steps/steps.py b/extensions/prometheus/tests/features/steps/steps.py index eb27aecb77..30d9eeaa60 100644 --- a/extensions/prometheus/tests/features/steps/steps.py +++ b/extensions/prometheus/tests/features/steps/steps.py @@ -12,18 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from behave import step, then, given - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from behave import given, step, then +from containers.prometheus_container import PrometheusContainer from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from containers.prometheus_container import PrometheusContainer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) -@step('a Prometheus server is set up') +@step("a Prometheus server is set up") def setup_prometheus_server(context: MinifiTestContext): context.containers["prometheus"] = PrometheusContainer(context) @@ -33,19 +34,42 @@ def setup_prometheus_server_with_ssl(context: MinifiTestContext): context.containers["prometheus"] = PrometheusContainer(context, ssl=True) -@then("\"{metric_class}\" are published to the Prometheus server in less than {timeout_seconds:d} seconds") -@then("\"{metric_class}\" is published to the Prometheus server in less than {timeout_seconds:d} seconds") -def verify_metric_class_published_to_prometheus(context: MinifiTestContext, metric_class: str, timeout_seconds: int): +@then( + '"{metric_class}" are published to the Prometheus server in less than {timeout_seconds:d} seconds' +) +@then( + '"{metric_class}" is published to the Prometheus server in less than {timeout_seconds:d} seconds' +) +def verify_metric_class_published_to_prometheus( + context: MinifiTestContext, metric_class: str, timeout_seconds: int +): assert wait_for_condition( - condition=lambda: context.containers["prometheus"].check_metric_class_on_prometheus(metric_class), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers["prometheus"].exited, context=context) - - -@then("\"{metric_class}\" processor metric is published to the Prometheus server in less than {timeout_seconds:d} seconds for \"{processor_name}\" processor") -def verify_processor_metric_published_to_prometheus(context: MinifiTestContext, metric_class: str, timeout_seconds: int, processor_name: str): + condition=lambda: context.containers[ + "prometheus" + ].check_metric_class_on_prometheus(metric_class), + timeout_seconds=timeout_seconds, + bail_condition=lambda: context.containers["prometheus"].exited, + context=context, + ) + + +@then( + '"{metric_class}" processor metric is published to the Prometheus server in less than {timeout_seconds:d} seconds for "{processor_name}" processor' +) +def verify_processor_metric_published_to_prometheus( + context: MinifiTestContext, + metric_class: str, + timeout_seconds: int, + processor_name: str, +): assert wait_for_condition( - condition=lambda: context.containers["prometheus"].check_processor_metric_on_prometheus(metric_class, processor_name), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers["prometheus"].exited, context=context) + condition=lambda: context.containers[ + "prometheus" + ].check_processor_metric_on_prometheus(metric_class, processor_name), + timeout_seconds=timeout_seconds, + bail_condition=lambda: context.containers["prometheus"].exited, + context=context, + ) @then("all Prometheus metric types are only defined once") @@ -54,11 +78,19 @@ def verify_all_prometheus_metric_types_defined_once(context: MinifiTestContext): def _enable_prometheus(context: MinifiTestContext): - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.agent.identifier", "Agent1") - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.PrometheusMetricsPublisher.port", "9936") - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.PrometheusMetricsPublisher.metrics", - "RepositoryMetrics,QueueMetrics,PutFileMetrics,processorMetrics/Get.*,FlowInformation,DeviceInfoNode,AgentStatus") - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.class", "PrometheusMetricsPublisher") + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.agent.identifier", "Agent1" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.PrometheusMetricsPublisher.port", "9936" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.PrometheusMetricsPublisher.metrics", + "RepositoryMetrics,QueueMetrics,PutFileMetrics,processorMetrics/Get.*,FlowInformation,DeviceInfoNode,AgentStatus", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.class", "PrometheusMetricsPublisher" + ) @given("Prometheus is enabled in MiNiFi") @@ -69,5 +101,11 @@ def enable_prometheus_in_minifi(context: MinifiTestContext): @given("Prometheus with SSL is enabled in MiNiFi") def enable_prometheus_with_ssl_in_minifi(context: MinifiTestContext): _enable_prometheus(context) - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.PrometheusMetricsPublisher.certificate", "/tmp/resources/minifi_merged_cert.crt") - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.PrometheusMetricsPublisher.ca.certificate", "/tmp/resources/root_ca.crt") + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.PrometheusMetricsPublisher.certificate", + "/tmp/resources/minifi_merged_cert.crt", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.PrometheusMetricsPublisher.ca.certificate", + "/tmp/resources/root_ca.crt", + ) diff --git a/extensions/python/CMakeLists.txt b/extensions/python/CMakeLists.txt index 34b1df6730..317bdeb881 100644 --- a/extensions/python/CMakeLists.txt +++ b/extensions/python/CMakeLists.txt @@ -38,14 +38,14 @@ target_link_libraries(minifi-python-script-extension PRIVATE ${LIBMINIFI} Thread include(GenericPython) if(APPLE) - target_compile_definitions(minifi-python-lib-loader-extension PUBLIC Py_LIMITED_API=0x03090000) - target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x03090000) + target_compile_definitions(minifi-python-lib-loader-extension PUBLIC Py_LIMITED_API=0x030A0000) + target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x030A0000) target_link_options(minifi-python-script-extension PRIVATE "-Wl,-undefined,dynamic_lookup") elseif(WIN32) - target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x03060000) + target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x030A0000) else() - target_compile_definitions(minifi-python-lib-loader-extension PUBLIC Py_LIMITED_API=0x03060000) - target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x03060000) + target_compile_definitions(minifi-python-lib-loader-extension PUBLIC Py_LIMITED_API=0x030A0000) + target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x030A0000) endif() target_compile_definitions(minifi-python-script-extension PUBLIC PY_SSIZE_T_CLEAN) diff --git a/extensions/python/PYTHON.md b/extensions/python/PYTHON.md index 66e7c29950..7bdeb341df 100644 --- a/extensions/python/PYTHON.md +++ b/extensions/python/PYTHON.md @@ -180,7 +180,7 @@ Describe is passed the processor and is a required function. You must set the de ```python def describe(processor): - processor.setDescription("Adds an attribute to your flow files") + processor.setDescription("Adds an attribute to your flow files") ``` onInitialize is also passed the processor reference and can be where you set properties. The first argument is the property display name, @@ -200,9 +200,11 @@ The last parameter of addProperty is the controller service type. If the propert ```python def onInitialize(processor): - processor.setSupportsDynamicProperties() - # arguments: property name, description, default value, is required, expression language supported, is sensitive, property type code, controller service type name - processor.addProperty("property name", "description", "default value", True, False, False, 1, None) + processor.setSupportsDynamicProperties() + # arguments: property name, description, default value, is required, expression language supported, is sensitive, property type code, controller service type name + processor.addProperty( + "property name", "description", "default value", True, False, False, 1, None + ) ``` The onSchedule function is passed the context and session factory. This should be where your processor loads and reads properties via @@ -214,19 +216,19 @@ VaderSentiment ```python class VaderSentiment(object): - def __init__(self): - self.content = None + def __init__(self): + self.content = None - def process(self, input_stream): - self.content = codecs.getreader('utf-8')(input_stream).read() - return len(self.content) + def process(self, input_stream): + self.content = codecs.getreader("utf-8")(input_stream).read() + return len(self.content) ``` By default, the MiNiFi C++ style native python processors are executed on multiple threads in parallel if 'max concurrent tasks' is set to more than 1 in the flow configuration, but it is possible to set the processor to be single threaded by calling the `setSingleThreaded()` method in the processor while initializing. When a processor is set to be single threaded, only one thread will execute the processor, and setting 'max concurrent tasks' to more than 1 will not have any effect. ```python def onInitialize(processor): - processor.setSingleThreaded() + processor.setSingleThreaded() ``` ## Using NiFi Python Processors @@ -262,13 +264,13 @@ from nifiapi.decorators import trigger_serially @trigger_serially class SingleThreadedProcessor(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements = ["org.apache.nifi.python.processor.FlowFileSource"] def __init__(self, **kwargs): pass def create(self, context): - return FlowFileSourceResult(relationship='success', contents="result") + return FlowFileSourceResult(relationship="success", contents="result") ``` ## Use Python processors from virtualenv diff --git a/extensions/python/pythonprocessor-examples/AddPythonAttribute.py b/extensions/python/pythonprocessor-examples/AddPythonAttribute.py index 53c8a60009..c7944de14a 100644 --- a/extensions/python/pythonprocessor-examples/AddPythonAttribute.py +++ b/extensions/python/pythonprocessor-examples/AddPythonAttribute.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. diff --git a/extensions/python/pythonprocessor-examples/CountingProcessor.py b/extensions/python/pythonprocessor-examples/CountingProcessor.py index 024f62f305..1dac431cb7 100644 --- a/extensions/python/pythonprocessor-examples/CountingProcessor.py +++ b/extensions/python/pythonprocessor-examples/CountingProcessor.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -global i i = 0 @@ -22,7 +21,7 @@ def __init__(self, content): self.content = content def process(self, output_stream): - output_stream.write(self.content.encode('utf-8')) + output_stream.write(self.content.encode("utf-8")) return len(self.content) diff --git a/extensions/python/pythonprocessor-examples/GaussianDistributionWithNumpy.py b/extensions/python/pythonprocessor-examples/GaussianDistributionWithNumpy.py index 0ad00fce91..342971f6e9 100644 --- a/extensions/python/pythonprocessor-examples/GaussianDistributionWithNumpy.py +++ b/extensions/python/pythonprocessor-examples/GaussianDistributionWithNumpy.py @@ -21,12 +21,14 @@ def __init__(self, content): self.content = content def process(self, output_stream): - output_stream.write(self.content.encode('utf-8')) + output_stream.write(self.content.encode("utf-8")) return len(self.content) def describe(processor): - processor.setDescription("Draw random samples from a normal (Gaussian) distribution.") + processor.setDescription( + "Draw random samples from a normal (Gaussian) distribution." + ) def onInitialize(processor): diff --git a/extensions/python/pythonprocessor-examples/MoveContentToJson.py b/extensions/python/pythonprocessor-examples/MoveContentToJson.py index 1b3068b07c..ec6ea03d07 100644 --- a/extensions/python/pythonprocessor-examples/MoveContentToJson.py +++ b/extensions/python/pythonprocessor-examples/MoveContentToJson.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -19,7 +18,7 @@ class ReadCallback: def process(self, input_stream): - self.content = codecs.getreader('utf-8')(input_stream).read() + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) @@ -29,12 +28,14 @@ def __init__(self, content): def process(self, output_stream): json_content = json.dumps({"content": self.content}) - output_stream.write(json_content.encode('utf-8')) + output_stream.write(json_content.encode("utf-8")) return len(json_content) def describe(processor): - processor.setDescription("Moves content of flow file to JSON file under 'content' key") + processor.setDescription( + "Moves content of flow file to JSON file under 'content' key" + ) def onInitialize(processor): diff --git a/extensions/python/pythonprocessor-examples/RemoveFlowFile.py b/extensions/python/pythonprocessor-examples/RemoveFlowFile.py index 5235636b49..27ea7760fa 100644 --- a/extensions/python/pythonprocessor-examples/RemoveFlowFile.py +++ b/extensions/python/pythonprocessor-examples/RemoveFlowFile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. diff --git a/extensions/python/pythonprocessor-examples/SentimentAnalysis.py b/extensions/python/pythonprocessor-examples/SentimentAnalysis.py index 4db22608d6..5b54194a2f 100644 --- a/extensions/python/pythonprocessor-examples/SentimentAnalysis.py +++ b/extensions/python/pythonprocessor-examples/SentimentAnalysis.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,23 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. import codecs + from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer def describe(processor): - processor.setDescription("Provides a sentiment analysis of the content within the flow file") + processor.setDescription( + "Provides a sentiment analysis of the content within the flow file" + ) def onInitialize(processor): processor.setSupportsDynamicProperties() -class VaderSentiment(object): +class VaderSentiment: def __init__(self): self.content = None def process(self, input_stream): - self.content = codecs.getreader('utf-8')(input_stream).read() + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) @@ -41,7 +43,7 @@ def onTrigger(context, session): session.read(flow_file, sentiment) analyzer = SentimentIntensityAnalyzer() vs = analyzer.polarity_scores(sentiment.content) - flow_file.addAttribute("positive", str(vs['pos'])) - flow_file.addAttribute("negative", str(vs['neg'])) - flow_file.addAttribute("neutral", str(vs['neu'])) + flow_file.addAttribute("positive", str(vs["pos"])) + flow_file.addAttribute("negative", str(vs["neg"])) + flow_file.addAttribute("neutral", str(vs["neu"])) session.transfer(flow_file, REL_SUCCESS) diff --git a/extensions/python/pythonprocessor-examples/google/SentimentAnalyzer.py b/extensions/python/pythonprocessor-examples/google/SentimentAnalyzer.py index c7ab9c0a05..7a9e2bc2fe 100644 --- a/extensions/python/pythonprocessor-examples/google/SentimentAnalyzer.py +++ b/extensions/python/pythonprocessor-examples/google/SentimentAnalyzer.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,35 +13,44 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - Install the following with pip ( or pip3 ) +Install the following with pip ( or pip3 ) - pip install google-cloud-language +pip install google-cloud-language - -- the following were needed during development as we saw SSL timeout errors - pip install requests[security] - pip install -U httplib2 +-- the following were needed during development as we saw SSL timeout errors +pip install requests[security] +pip install -U httplib2 """ + import codecs + from google.cloud import language -from google.cloud.language import enums -from google.cloud.language import types +from google.cloud.language import enums, types def describe(processor): - processor.setDescription("Performs a sentiment Analysis of incoming flowfile content using Google Cloud.") + processor.setDescription( + "Performs a sentiment Analysis of incoming flowfile content using Google Cloud." + ) def onInitialize(processor): # is required, - processor.addProperty("Credentials Path", "Path to your Google Credentials JSON File. Must exist on agent hosts.", "", True, False) + processor.addProperty( + "Credentials Path", + "Path to your Google Credentials JSON File. Must exist on agent hosts.", + "", + True, + False, + ) -class ContentExtract(object): +class ContentExtract: def __init__(self): self.content = None def process(self, input_stream): - self.content = codecs.getreader('utf-8')(input_stream).read() + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) @@ -52,10 +60,16 @@ def onTrigger(context, session): credentials_filename = context.getProperty("Credentials Path") sentiment = ContentExtract() session.read(flow_file, sentiment) - client = language.LanguageServiceClient.from_service_account_json(credentials_filename) - document = types.Document(content=sentiment.content, type=enums.Document.Type.PLAIN_TEXT) + client = language.LanguageServiceClient.from_service_account_json( + credentials_filename + ) + document = types.Document( + content=sentiment.content, type=enums.Document.Type.PLAIN_TEXT + ) - annotations = client.analyze_sentiment(document=document, retry=None, timeout=1.0) + annotations = client.analyze_sentiment( + document=document, retry=None, timeout=1.0 + ) score = annotations.document_sentiment.score magnitude = annotations.document_sentiment.magnitude diff --git a/extensions/python/pythonprocessor-examples/h2o/ConvertDsToCsv.py b/extensions/python/pythonprocessor-examples/h2o/ConvertDsToCsv.py index 566f4e3126..3cf9c6bf2a 100644 --- a/extensions/python/pythonprocessor-examples/h2o/ConvertDsToCsv.py +++ b/extensions/python/pythonprocessor-examples/h2o/ConvertDsToCsv.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,61 +13,61 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - Install the following with pip +Install the following with pip - pip install datatable - pip install pandas +pip install datatable +pip install pandas """ + import codecs -import pandas as pd # noqa F401 -import datatable as dt from io import StringIO +import datatable as dt +import pandas as pd # noqa F401 + def describe(processor): - """ describe what this processor does - """ - processor.setDescription("Converts the data source content of incoming flow file to csv. It \ + """describe what this processor does""" + processor.setDescription( + "Converts the data source content of incoming flow file to csv. It \ supports a variety of data sources: pandas DataFrames, csv, numpy \ - arrays, dictionary, list, raw Python objects, etc") + arrays, dictionary, list, raw Python objects, etc" + ) def onInitialize(processor): - """ onInitialize is where you can set properties - """ + """onInitialize is where you can set properties""" processor.setSupportsDynamicProperties() -class ContentExtract(object): - """ ContentExtract callback class is defined for reading streams of data through the session - and has a process function that accepts the input stream +class ContentExtract: + """ContentExtract callback class is defined for reading streams of data through the session + and has a process function that accepts the input stream """ + def __init__(self): self.content = None def process(self, input_stream): - """ Use codecs getReader to read that data - """ - self.content = codecs.getreader('utf-8')(input_stream).read() + """Use codecs getReader to read that data""" + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) -class ContentWrite(object): - """ ContentWrite callback class is defined for writing streams of data through the session - """ +class ContentWrite: + """ContentWrite callback class is defined for writing streams of data through the session""" + def __init__(self, data): self.content = data def process(self, output_stream): - """ Use codecs getWriter to write data encoded to the stream - """ - codecs.getwriter('utf-8')(output_stream).write(self.content) + """Use codecs getWriter to write data encoded to the stream""" + codecs.getwriter("utf-8")(output_stream).write(self.content) return len(self.content) def onTrigger(context, session): - """ onTrigger is executed and passed processor context and session - """ + """onTrigger is executed and passed processor context and session""" flow_file = session.get() if flow_file is not None: read_cb = ContentExtract() diff --git a/extensions/python/pythonprocessor-examples/h2o/h2o3/mojo/ExecuteH2oMojoScoring.py b/extensions/python/pythonprocessor-examples/h2o/h2o3/mojo/ExecuteH2oMojoScoring.py index 0c6f09207a..8b4e02cbd6 100644 --- a/extensions/python/pythonprocessor-examples/h2o/h2o3/mojo/ExecuteH2oMojoScoring.py +++ b/extensions/python/pythonprocessor-examples/h2o/h2o3/mojo/ExecuteH2oMojoScoring.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,85 +13,117 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - -- after downloading the mojo model from h2o3, the following packages - are needed to execute the model to do batch or real-time scoring +-- after downloading the mojo model from h2o3, the following packages + are needed to execute the model to do batch or real-time scoring - Make all packages available on your machine: +Make all packages available on your machine: - sudo apt-get -y update +sudo apt-get -y update - Install Java to include open source H2O-3 algorithms: +Install Java to include open source H2O-3 algorithms: - sudo apt-get -y install openjdk-8-jdk +sudo apt-get -y install openjdk-8-jdk - Install Datatable and pandas: +Install Datatable and pandas: - pip install datatable - pip install pandas +pip install datatable +pip install pandas - Option 1: Install H2O-3 with conda +Option 1: Install H2O-3 with conda - conda create -n h2o3-nifi-minifi python=3.6 - conda activate h2o3-nifi-minifi - conda config --append channels conda-forge - conda install -y -c h2oai h2o +conda create -n h2o3-nifi-minifi python=3.6 +conda activate h2o3-nifi-minifi +conda config --append channels conda-forge +conda install -y -c h2oai h2o - Option 2: Install H2O-3 with pip +Option 2: Install H2O-3 with pip - pip install requests - pip install tabulate - pip install "colorama>=0.3.8" - pip install future - pip uninstall h2o - If on Mac OS X, must include --user: - pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o --user - else: - pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o +pip install requests +pip install tabulate +pip install "colorama>=0.3.8" +pip install future +pip uninstall h2o +If on Mac OS X, must include --user: + pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o --user +else: + pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o """ -import h2o + import codecs -import pandas as pd # noqa: F401 + import datatable as dt +import h2o +import pandas as pd # noqa: F401 mojo_model = None def describe(processor): - """ describe what this processor does - """ - processor.setDescription("Executes H2O-3's MOJO Model in Python to do batch scoring or \ + """describe what this processor does""" + processor.setDescription( + "Executes H2O-3's MOJO Model in Python to do batch scoring or \ real-time scoring for one or more predicted label(s) on the tabular test data in \ the incoming flow file content. If tabular data is one row, then MOJO does real-time \ - scoring. If tabular data is multiple rows, then MOJO does batch scoring.") + scoring. If tabular data is multiple rows, then MOJO does batch scoring." + ) def onInitialize(processor): - """ onInitialize is where you can set properties - processor.addProperty(name, description, defaultValue, required, el) + """onInitialize is where you can set properties + processor.addProperty(name, description, defaultValue, required, el) """ - processor.addProperty("MOJO Model Filepath", "Add the filepath to the MOJO Model file. For example, \ - 'path/to/mojo-model/GBM_grid__1_AutoML_20200511_075150_model_180.zip'.", "", True, False) - - processor.addProperty("Is First Line Header", "Add True or False for whether first line is header.", - "True", True, False) - - processor.addProperty("Input Schema", "If first line is not header, then you must add Input Schema for \ + processor.addProperty( + "MOJO Model Filepath", + "Add the filepath to the MOJO Model file. For example, \ + 'path/to/mojo-model/GBM_grid__1_AutoML_20200511_075150_model_180.zip'.", + "", + True, + False, + ) + + processor.addProperty( + "Is First Line Header", + "Add True or False for whether first line is header.", + "True", + True, + False, + ) + + processor.addProperty( + "Input Schema", + "If first line is not header, then you must add Input Schema for \ incoming data.If there is more than one column name, write a comma separated list of \ - column names. Else, you do not need to add an Input Schema.", "", False, False) - - processor.addProperty("Use Output Header", "Add True or False for whether you want to use an output \ - for your predictions.", "False", False, False) - - processor.addProperty("Output Schema", "To set Output Schema, 'Use Output Header' must be set to 'True' \ + column names. Else, you do not need to add an Input Schema.", + "", + False, + False, + ) + + processor.addProperty( + "Use Output Header", + "Add True or False for whether you want to use an output \ + for your predictions.", + "False", + False, + False, + ) + + processor.addProperty( + "Output Schema", + "To set Output Schema, 'Use Output Header' must be set to 'True' \ If you want more descriptive column names for your predictions, then add an Output Schema. If there \ is more than one column name, write a comma separated list of column names. Else, H2O-3 will include \ - them by default", "", False, False) + them by default", + "", + False, + False, + ) def onSchedule(context): - """ onSchedule is where you load and read properties - this function is called 1 time when the processor is scheduled to run + """onSchedule is where you load and read properties + this function is called 1 time when the processor is scheduled to run """ global mojo_model h2o.init() @@ -101,37 +132,34 @@ def onSchedule(context): mojo_model = h2o.import_mojo(mojo_model_filepath) -class ContentExtract(object): - """ ContentExtract callback class is defined for reading streams of data through the session - and has a process function that accepts the input stream +class ContentExtract: + """ContentExtract callback class is defined for reading streams of data through the session + and has a process function that accepts the input stream """ + def __init__(self): self.content = None def process(self, input_stream): - """ Use codecs getReader to read that data - """ - self.content = codecs.getreader('utf-8')(input_stream).read() + """Use codecs getReader to read that data""" + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) -class ContentWrite(object): - """ ContentWrite callback class is defined for writing streams of data through the session - """ +class ContentWrite: + """ContentWrite callback class is defined for writing streams of data through the session""" + def __init__(self, data): self.content = data def process(self, output_stream): - """ Use codecs getWriter to write data encoded to the stream - """ - codecs.getwriter('utf-8')(output_stream).write(self.content) + """Use codecs getWriter to write data encoded to the stream""" + codecs.getwriter("utf-8")(output_stream).write(self.content) return len(self.content) def onTrigger(context, session): - """ onTrigger is executed and passed processor context and session - """ - global mojo_model + """onTrigger is executed and passed processor context and session""" flow_file = session.get() if flow_file is not None: # read test data of flow file content into read_cb.content @@ -141,7 +169,9 @@ def onTrigger(context, session): # flow_file.addAttribute("mojo_model_id", mojo_model.model_id) # load tabular data str of 1 or more rows into datatable frame test_dt_frame = dt.Frame(read_cb.content) - test_h2o_frame = h2o.H2OFrame(python_obj=test_dt_frame.to_numpy(), column_names=list(test_dt_frame.names)) + test_h2o_frame = h2o.H2OFrame( + python_obj=test_dt_frame.to_numpy(), column_names=list(test_dt_frame.names) + ) # does test dt frame column names (header) equal m_scorer feature_names (exp_header) first_line_header = context.getProperty("Is First Line Header") if first_line_header == "False": @@ -167,5 +197,7 @@ def onTrigger(context, session): for i in range(len(pred_header)): ff_attr_name = pred_header[i] + "_pred_0" flow_file.addAttribute(ff_attr_name, str(preds_pd_df.at[0, pred_header[i]])) - log.info("getAttribute({}): {}".format(ff_attr_name, flow_file.getAttribute(ff_attr_name))) + log.info( + f"getAttribute({ff_attr_name}): {flow_file.getAttribute(ff_attr_name)}" + ) session.transfer(flow_file, REL_SUCCESS) diff --git a/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py b/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py index 0365f20052..25dbdc9131 100644 --- a/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py +++ b/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py @@ -1,7 +1,7 @@ import ast -import sys -import subprocess import os +import subprocess +import sys # Extract the list of PIP dependency packages from the visited processor class AST node @@ -16,11 +16,14 @@ def visit_ClassDef(self, node): # Iterate through the body of the class to find the ProcessorDetails nested class for child in node.body: - if isinstance(child, ast.ClassDef) and child.name == 'ProcessorDetails': + if isinstance(child, ast.ClassDef) and child.name == "ProcessorDetails": # Iterate through the nodes of the 'ProcessorDetails' class for detail in child.body: # Check if the child node is an assignment of the 'dependencies' member variable - if isinstance(detail, ast.Assign) and detail.targets[0].id == 'dependencies': + if ( + isinstance(detail, ast.Assign) + and detail.targets[0].id == "dependencies" + ): # Iterate through values of the 'dependencies' list member variable for elt in detail.value.elts: # Check if the element is a string constant and add it to the dependencies list @@ -31,8 +34,8 @@ def visit_ClassDef(self, node): def extract_dependencies(file_path): - class_name = file_path.split(os.sep)[-1].split('.')[0] - with open(file_path, 'r') as file: + class_name = file_path.split(os.sep)[-1].split(".")[0] + with open(file_path, "r") as file: code = file.read() tree = ast.parse(code) @@ -42,10 +45,10 @@ def extract_dependencies(file_path): def has_progress_bar() -> bool: - return sys.version_info.major > 3 or (sys.version_info.major == 3 and sys.version_info.minor >= 9) + return sys.version_info >= (3, 9) -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) < 2: sys.exit(1) @@ -53,20 +56,30 @@ def has_progress_bar() -> bool: # --no-cache-dir is used to be in line with NiFi's dependency install behavior if has_progress_bar(): - command = [sys.executable, "-m", "pip", "install", "--no-cache-dir", "--progress-bar", "off"] + command = [ + sys.executable, + "-m", + "pip", + "install", + "--no-cache-dir", + "--progress-bar", + "off", + ] else: command = [sys.executable, "-m", "pip", "install", "--no-cache-dir"] dependencies_found = False for i in range(1, len(sys.argv)): if "requirements.txt" in sys.argv[i]: command += ["-r", sys.argv[i]] - print("Installing dependencies from requirements file: {}".format(sys.argv[i])) + print(f"Installing dependencies from requirements file: {sys.argv[i]}") dependencies_found = True else: dependencies = extract_dependencies(sys.argv[i]) if dependencies: dependencies_found = True - print("Installing dependencies for processor {}: {}".format(sys.argv[i], str(dependencies))) + print( + f"Installing dependencies for processor {sys.argv[i]}: {dependencies!s}" + ) command += dependencies if dependencies_found: @@ -74,7 +87,9 @@ def has_progress_bar() -> bool: subprocess.check_call(command) print("Done installing dependencies for MiNiFi python processors.") except subprocess.CalledProcessError as e: - print("Error occurred while installing dependencies: {}".format(e)) + print(f"Error occurred while installing dependencies: {e}") sys.exit(1) else: - print("No dependencies were found for MiNiFi python processors. No python packages were installed.") + print( + "No dependencies were found for MiNiFi python processors. No python packages were installed." + ) diff --git a/extensions/python/pythonprocessors/nifiapi/componentstate.py b/extensions/python/pythonprocessors/nifiapi/componentstate.py index f6cf7e6b70..0b017ddcfe 100644 --- a/extensions/python/pythonprocessors/nifiapi/componentstate.py +++ b/extensions/python/pythonprocessors/nifiapi/componentstate.py @@ -14,7 +14,7 @@ # limitations under the License. from enum import Enum -from typing import Dict + from minifi_native import StateManager as CppFlowFile @@ -24,7 +24,7 @@ class Scope(Enum): class StateMap: - def __init__(self, state_map: Dict[str, str]): + def __init__(self, state_map: dict[str, str]): self.state_map = state_map if state_map is not None else {} def getStateVersion(self) -> int: @@ -36,7 +36,7 @@ def get(self, key) -> str: return self.state_map[key] - def toMap(self) -> Dict[str, str]: + def toMap(self) -> dict[str, str]: return self.state_map @@ -48,7 +48,7 @@ class StateManager: def __init__(self, cpp_state_manager: CppFlowFile): self.cpp_state_manager = cpp_state_manager - def setState(self, state: Dict[str, str], scope: Scope) -> bool: + def setState(self, state: dict[str, str], scope: Scope) -> bool: try: return self.cpp_state_manager.set(state) except Exception as exception: @@ -60,7 +60,9 @@ def getState(self, scope: Scope) -> StateMap: except Exception as exception: raise StateException("Get state failed") from exception - def replace(self, old_state: StateMap, new_values: Dict[str, str], scope: Scope) -> bool: + def replace( + self, old_state: StateMap, new_values: dict[str, str], scope: Scope + ) -> bool: try: return self.cpp_state_manager.replace(old_state.toMap(), new_values) except Exception as exception: diff --git a/extensions/python/pythonprocessors/nifiapi/decorators.py b/extensions/python/pythonprocessors/nifiapi/decorators.py index 502817f621..a0aea48002 100644 --- a/extensions/python/pythonprocessors/nifiapi/decorators.py +++ b/extensions/python/pythonprocessors/nifiapi/decorators.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + def trigger_serially(cls): cls._trigger_serially = True return cls diff --git a/extensions/python/pythonprocessors/nifiapi/documentation.py b/extensions/python/pythonprocessors/nifiapi/documentation.py index e09f0bb7e8..79f453cd9f 100644 --- a/extensions/python/pythonprocessors/nifiapi/documentation.py +++ b/extensions/python/pythonprocessors/nifiapi/documentation.py @@ -1,3 +1,5 @@ +from typing import ClassVar + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -16,7 +18,9 @@ class ProcessorConfiguration: class Java: - implements = ['org.apache.nifi.python.processor.documentation.ProcessorConfigurationDetails'] + implements: ClassVar[list] = [ + "org.apache.nifi.python.processor.documentation.ProcessorConfigurationDetails" + ] def __init__(self, processor_type: str, configuration: str): self.processor_type = processor_type @@ -29,25 +33,43 @@ def getConfiguration(self): return self.configuration -def use_case(description: str, configuration: str = None, notes: str = None, keywords: list[str] = None): +def use_case( + description: str, + configuration: str | None = None, + notes: str | None = None, + keywords: list[str] | None = None, +): """Decorator to explain how to perform a specific use case with a given processor""" + def decorator(func): return func + return decorator -def multi_processor_use_case(description: str, configurations: list[ProcessorConfiguration], notes: str = None, keywords: list[str] = None): +def multi_processor_use_case( + description: str, + configurations: list[ProcessorConfiguration], + notes: str | None = None, + keywords: list[str] | None = None, +): """Decorator to explain how to perform a specific use case that involves the decorated Processor, in addition to additional Processors""" + def decorator(func): return func + return decorator class UseCaseDetails: class Java: - implements = ['org.apache.nifi.python.processor.documentation.UseCaseDetails'] + implements: ClassVar[list] = [ + "org.apache.nifi.python.processor.documentation.UseCaseDetails" + ] - def __init__(self, description: str, notes: str, keywords: list[str], configuration: str): + def __init__( + self, description: str, notes: str, keywords: list[str], configuration: str + ): self.description = description self.notes = notes self.keywords = keywords @@ -71,9 +93,17 @@ def __str__(self): class MultiProcessorUseCaseDetails: class Java: - implements = ['org.apache.nifi.python.processor.documentation.MultiProcessorUseCaseDetails'] - - def __init__(self, description: str, notes: str, keywords: list[str], configurations: list[ProcessorConfiguration]): + implements: ClassVar[list] = [ + "org.apache.nifi.python.processor.documentation.MultiProcessorUseCaseDetails" + ] + + def __init__( + self, + description: str, + notes: str, + keywords: list[str], + configurations: list[ProcessorConfiguration], + ): self.description = description self.notes = notes self.keywords = keywords @@ -97,17 +127,21 @@ def __str__(self): class PropertyDescription: class Java: - implements = ['org.apache.nifi.python.processor.documentation.PropertyDescription'] - - def __init__(self, - name: str, - description: str, - display_name: str = None, - required: bool = False, - sensitive: bool = True, - default_value: str = None, - expression_language_scope: str = None, - controller_service_definition: str = None): + implements: ClassVar[list] = [ + "org.apache.nifi.python.processor.documentation.PropertyDescription" + ] + + def __init__( + self, + name: str, + description: str, + display_name: str | None = None, + required: bool = False, + sensitive: bool = True, + default_value: str | None = None, + expression_language_scope: str | None = None, + controller_service_definition: str | None = None, + ): self.name = name self.description = description self.display_name = display_name diff --git a/extensions/python/pythonprocessors/nifiapi/flowfilesource.py b/extensions/python/pythonprocessors/nifiapi/flowfilesource.py index 74041d7804..32da4179b7 100644 --- a/extensions/python/pythonprocessors/nifiapi/flowfilesource.py +++ b/extensions/python/pythonprocessors/nifiapi/flowfilesource.py @@ -14,10 +14,12 @@ # limitations under the License. import traceback -from .properties import ProcessContext as ProcessContextProxy from abc import abstractmethod -from minifi_native import ProcessContext, ProcessSession, FlowFile + +from minifi_native import FlowFile, ProcessContext, ProcessSession + from .processorbase import ProcessorBase, WriteCallback +from .properties import ProcessContext as ProcessContextProxy class FlowFileSourceResult: @@ -47,10 +49,14 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): if not result: return except Exception: - self.logger.error("Failed to create flow file due to error:\n{}".format(traceback.format_exc())) + self.logger.error( + f"Failed to create flow file due to error:\n{traceback.format_exc()}" + ) return - flow_file = self.createFlowFile(session, result.getAttributes(), result.getContents()) + flow_file = self.createFlowFile( + session, result.getAttributes(), result.getContents() + ) if result.getRelationship() == "success": session.transfer(flow_file, self.REL_SUCCESS) diff --git a/extensions/python/pythonprocessors/nifiapi/flowfiletransform.py b/extensions/python/pythonprocessors/nifiapi/flowfiletransform.py index c72ebfb54c..f8507fd2b9 100644 --- a/extensions/python/pythonprocessors/nifiapi/flowfiletransform.py +++ b/extensions/python/pythonprocessors/nifiapi/flowfiletransform.py @@ -15,7 +15,9 @@ import traceback from abc import abstractmethod + from minifi_native import ProcessContext, ProcessSession + from .processorbase import ProcessorBase, WriteCallback from .properties import FlowFile as FlowFileProxy from .properties import ProcessContext as ProcessContextProxy @@ -53,14 +55,18 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): try: result = self.transform(context_proxy, flow_file_proxy) except Exception: - self.logger.error("Failed to transform flow file due to error:\n{}".format(traceback.format_exc())) + self.logger.error( + f"Failed to transform flow file due to error:\n{traceback.format_exc()}" + ) session.remove(flow_file) session.transfer(original_flow_file, self.REL_FAILURE) return if result.getRelationship() == "original": session.remove(flow_file) - self.logger.error("Result relationship cannot be 'original', it is reserved for the original flow file, and transferred automatically in non-failure cases.") + self.logger.error( + "Result relationship cannot be 'original', it is reserved for the original flow file, and transferred automatically in non-failure cases." + ) session.transfer(original_flow_file, self.REL_FAILURE) return @@ -71,7 +77,9 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): for name, value in result_attributes.items(): original_flow_file.setAttribute(name, value) if result.getContents() is not None: - self.logger.error("'failure' relationship should not have content, the original flow file will be transferred automatically in this case.") + self.logger.error( + "'failure' relationship should not have content, the original flow file will be transferred automatically in this case." + ) session.transfer(original_flow_file, self.REL_FAILURE) return @@ -90,5 +98,7 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): session.transfer(original_flow_file, self.REL_ORIGINAL) @abstractmethod - def transform(self, context: ProcessContextProxy, flowFile: FlowFileProxy) -> FlowFileTransformResult: + def transform( + self, context: ProcessContextProxy, flowFile: FlowFileProxy + ) -> FlowFileTransformResult: pass diff --git a/extensions/python/pythonprocessors/nifiapi/processorbase.py b/extensions/python/pythonprocessors/nifiapi/processorbase.py index 2088b03ff9..77b24a9278 100644 --- a/extensions/python/pythonprocessors/nifiapi/processorbase.py +++ b/extensions/python/pythonprocessors/nifiapi/processorbase.py @@ -14,10 +14,16 @@ # limitations under the License. from abc import ABC, abstractmethod -from typing import List -from .properties import ExpressionLanguageScope, PropertyDescriptor, translateStandardValidatorToMiNiFiPropertype, MinifiPropertyTypes + +from minifi_native import OutputStream, ProcessContext, Processor, ProcessSession + +from .properties import ( + ExpressionLanguageScope, + MinifiPropertyTypes, + PropertyDescriptor, + translateStandardValidatorToMiNiFiPropertype, +) from .properties import ProcessContext as ProcessContextProxy -from minifi_native import OutputStream, Processor, ProcessContext, ProcessSession class WriteCallback: @@ -37,38 +43,65 @@ class ProcessorBase(ABC): REL_ORIGINAL = None def describe(self, processor: Processor): - if hasattr(self, 'ProcessorDetails') and hasattr(self.ProcessorDetails, 'description'): + if hasattr(self, "ProcessorDetails") and hasattr( + self.ProcessorDetails, "description" + ): processor.setDescription(self.ProcessorDetails.description) else: processor.setDescription(self.__class__.__name__) - if hasattr(self, 'ProcessorDetails') and hasattr(self.ProcessorDetails, 'version'): + if hasattr(self, "ProcessorDetails") and hasattr( + self.ProcessorDetails, "version" + ): processor.setVersion(self.ProcessorDetails.version) def onInitialize(self, processor: Processor): - get_dynamic_property_descriptor_attr = getattr(self, 'getDynamicPropertyDescriptor', None) - if get_dynamic_property_descriptor_attr and callable(get_dynamic_property_descriptor_attr): + get_dynamic_property_descriptor_attr = getattr( + self, "getDynamicPropertyDescriptor", None + ) + if get_dynamic_property_descriptor_attr and callable( + get_dynamic_property_descriptor_attr + ): processor.setSupportsDynamicProperties() self.supports_dynamic_properties = True else: self.supports_dynamic_properties = False - if hasattr(self, '_trigger_serially') and self._trigger_serially: + if hasattr(self, "_trigger_serially") and self._trigger_serially: processor.setSingleThreaded() for property in self.getPropertyDescriptors(): - expression_language_supported = True if property.expressionLanguageScope != ExpressionLanguageScope.NONE else False - property_type_code = translateStandardValidatorToMiNiFiPropertype(property.validators) + expression_language_supported = ( + property.expressionLanguageScope != ExpressionLanguageScope.NONE + ) + property_type_code = translateStandardValidatorToMiNiFiPropertype( + property.validators + ) # MiNiFi C++ does not support validators for expression language enabled properties - if expression_language_supported and property_type_code is not None and property_type_code != MinifiPropertyTypes.NON_BLANK_TYPE: - self.logger.warn("Property '{}' has validators defined, but since it also supports Expression Language, the validators will be ignored.".format(property.name)) + if ( + expression_language_supported + and property_type_code is not None + and property_type_code != MinifiPropertyTypes.NON_BLANK_TYPE + ): + self.logger.warn( + f"Property '{property.name}' has validators defined, but since it also supports Expression Language, the validators will be ignored." + ) property_type_code = None # MiNiFi C++ does not support dependant properties, so if a property depends on another property, it should not be required - is_required = True if property.required and not property.dependencies else False - processor.addProperty(property.name, property.description, property.defaultValue, is_required, expression_language_supported, - property.sensitive, property_type_code, property.allowableValues, property.controllerServiceDefinition) + is_required = bool(property.required and not property.dependencies) + processor.addProperty( + property.name, + property.description, + property.defaultValue, + is_required, + expression_language_supported, + property.sensitive, + property_type_code, + property.allowableValues, + property.controllerServiceDefinition, + ) def onScheduled(self, context_proxy: ProcessContextProxy): pass @@ -81,5 +114,5 @@ def onSchedule(self, context: ProcessContext): def onTrigger(self, context: ProcessContext, session: ProcessSession): pass - def getPropertyDescriptors(self) -> List[PropertyDescriptor]: + def getPropertyDescriptors(self) -> list[PropertyDescriptor]: return [] diff --git a/extensions/python/pythonprocessors/nifiapi/properties.py b/extensions/python/pythonprocessors/nifiapi/properties.py index 3cbd4c58c3..e176a1f4f3 100644 --- a/extensions/python/pythonprocessors/nifiapi/properties.py +++ b/extensions/python/pythonprocessors/nifiapi/properties.py @@ -14,10 +14,15 @@ # limitations under the License. from enum import Enum -from typing import List, Dict -from minifi_native import ProcessSession, timePeriodStringToMilliseconds, dataSizeStringToBytes + from minifi_native import FlowFile as CppFlowFile from minifi_native import ProcessContext as CppProcessContext +from minifi_native import ( + ProcessSession, + dataSizeStringToBytes, + timePeriodStringToMilliseconds, +) + from .componentstate import StateManager @@ -89,7 +94,7 @@ class MinifiPropertyTypes: PORT_TYPE = 6 -def translateStandardValidatorToMiNiFiPropertype(validators: List[int]) -> int: +def translateStandardValidatorToMiNiFiPropertype(validators: list[int]) -> int: if validators is None or len(validators) == 0 or len(validators) > 1: return None @@ -121,7 +126,14 @@ def __init__(self, property_descriptor, *dependent_values): class ResourceDefinition: - def __init__(self, allow_multiple=False, allow_file=True, allow_url=False, allow_directory=False, allow_text=False): + def __init__( + self, + allow_multiple=False, + allow_file=True, + allow_url=False, + allow_directory=False, + allow_text=False, + ): self.allow_multiple = allow_multiple self.allow_file = allow_file self.allow_url = allow_url @@ -136,10 +148,22 @@ class ExpressionLanguageScope(Enum): class PropertyDescriptor: - def __init__(self, name: str, description: str, required: bool = False, sensitive: bool = False, - display_name: str = None, default_value: str = None, allowable_values: List[str] = None, - dependencies: List[PropertyDependency] = None, expression_language_scope: ExpressionLanguageScope = ExpressionLanguageScope.NONE, - dynamic: bool = False, validators: List[int] = None, resource_definition: ResourceDefinition = None, controller_service_definition: str = None): + def __init__( + self, + name: str, + description: str, + required: bool = False, + sensitive: bool = False, + display_name: str | None = None, + default_value: str | None = None, + allowable_values: list[str] | None = None, + dependencies: list[PropertyDependency] | None = None, + expression_language_scope: ExpressionLanguageScope = ExpressionLanguageScope.NONE, + dynamic: bool = False, + validators: list[int] | None = None, + resource_definition: ResourceDefinition = None, + controller_service_definition: str | None = None, + ): if validators is None: validators = [StandardValidators.ALWAYS_VALID] @@ -159,20 +183,20 @@ def __init__(self, name: str, description: str, required: bool = False, sensitiv class TimeUnit(Enum): - NANOSECONDS = "NANOSECONDS", - MICROSECONDS = "MICROSECONDS", - MILLISECONDS = "MILLISECONDS", - SECONDS = "SECONDS", - MINUTES = "MINUTES", - HOURS = "HOURS", + NANOSECONDS = ("NANOSECONDS",) + MICROSECONDS = ("MICROSECONDS",) + MILLISECONDS = ("MILLISECONDS",) + SECONDS = ("SECONDS",) + MINUTES = ("MINUTES",) + HOURS = ("HOURS",) DAYS = "DAYS" class DataUnit(Enum): - B = "B", - KB = "KB", - MB = "MB", - GB = "GB", + B = ("B",) + KB = ("KB",) + MB = ("MB",) + GB = ("GB",) TB = "TB" @@ -195,7 +219,15 @@ def getAttributes(self): class PythonPropertyValue: - def __init__(self, cpp_context: CppProcessContext, name: str, string_value: str, el_supported: bool, controller_service_definition: str, is_dynamic: bool = False): + def __init__( + self, + cpp_context: CppProcessContext, + name: str, + string_value: str, + el_supported: bool, + controller_service_definition: str, + is_dynamic: bool = False, + ): self.cpp_context = cpp_context self.value = string_value self.name = name @@ -217,7 +249,7 @@ def asInteger(self) -> int: def asBoolean(self) -> bool: if not self.value: return None - return self.value.lower() == 'true' + return self.value.lower() == "true" def asFloat(self) -> float: if not self.value: @@ -235,13 +267,13 @@ def asTimePeriod(self, time_unit: TimeUnit) -> int: if time_unit == TimeUnit.MILLISECONDS: return milliseconds if time_unit == TimeUnit.SECONDS: - return int(round(milliseconds / 1000)) + return round(milliseconds / 1000) if time_unit == TimeUnit.MINUTES: - return int(round(milliseconds / 1000 / 60)) + return round(milliseconds / 1000 / 60) if time_unit == TimeUnit.HOURS: - return int(round(milliseconds / 1000 / 60 / 60)) + return round(milliseconds / 1000 / 60 / 60) if time_unit == TimeUnit.DAYS: - return int(round(milliseconds / 1000 / 60 / 60 / 24)) + return round(milliseconds / 1000 / 60 / 60 / 24) return 0 def asDataSize(self, data_unit: DataUnit) -> float: @@ -266,15 +298,30 @@ def evaluateAttributeExpressions(self, flow_file: FlowFile = None): if not self.el_supported or not self.value: return self - getter = self.cpp_context.getDynamicProperty if self.is_dynamic else self.cpp_context.getProperty + getter = ( + self.cpp_context.getDynamicProperty + if self.is_dynamic + else self.cpp_context.getProperty + ) args = () if flow_file is None else (flow_file.cpp_flow_file,) new_string_value = getter(self.name, *args) - return PythonPropertyValue(self.cpp_context, self.name, new_string_value, self.el_supported, self.controller_service_definition, self.is_dynamic) + return PythonPropertyValue( + self.cpp_context, + self.name, + new_string_value, + self.el_supported, + self.controller_service_definition, + self.is_dynamic, + ) def asControllerService(self): if not self.controller_service_definition: - raise Exception("Controller Service definition is not set, getProperty must be called with a property descriptor instead of string value") - return self.cpp_context.getControllerService(self.value, self.controller_service_definition) + raise RuntimeError( + "Controller Service definition is not set, getProperty must be called with a property descriptor instead of string value" + ) + return self.cpp_context.getControllerService( + self.value, self.controller_service_definition + ) class ProcessContext: @@ -291,7 +338,9 @@ def getProperty(self, descriptor) -> PythonPropertyValue: controller_service_definition = None else: property_name = descriptor.name - expression_language_support = descriptor.expressionLanguageScope != ExpressionLanguageScope.NONE + expression_language_support = ( + descriptor.expressionLanguageScope != ExpressionLanguageScope.NONE + ) controller_service_definition = descriptor.controllerServiceDefinition is_dynamic = False property_value = self.cpp_context.getRawProperty(property_name) @@ -299,7 +348,14 @@ def getProperty(self, descriptor) -> PythonPropertyValue: property_value = self.cpp_context.getRawDynamicProperty(property_name) if property_value is not None: is_dynamic = True - return PythonPropertyValue(self.cpp_context, property_name, property_value, expression_language_support, controller_service_definition, is_dynamic) + return PythonPropertyValue( + self.cpp_context, + property_name, + property_value, + expression_language_support, + controller_service_definition, + is_dynamic, + ) def getStateManager(self) -> StateManager: return StateManager(self.cpp_context.getStateManager()) @@ -307,13 +363,15 @@ def getStateManager(self) -> StateManager: def getName(self) -> str: return self.cpp_context.getName() - def getProperties(self) -> Dict[PropertyDescriptor, str]: - properties = dict() + def getProperties(self) -> dict[PropertyDescriptor, str]: + properties = {} cpp_properties = self.cpp_context.getProperties() for property_descriptor in self.processor.getPropertyDescriptors(): if property_descriptor.name in cpp_properties: - properties[property_descriptor] = cpp_properties[property_descriptor.name] + properties[property_descriptor] = cpp_properties[ + property_descriptor.name + ] return properties diff --git a/extensions/python/pythonprocessors/nifiapi/recordtransform.py b/extensions/python/pythonprocessors/nifiapi/recordtransform.py index bdeb2ffb60..82f963072b 100644 --- a/extensions/python/pythonprocessors/nifiapi/recordtransform.py +++ b/extensions/python/pythonprocessors/nifiapi/recordtransform.py @@ -13,10 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import traceback import json +import traceback from abc import abstractmethod -from minifi_native import ProcessContext, ProcessSession, Processor + +from minifi_native import ProcessContext, Processor, ProcessSession + from .processorbase import ProcessorBase from .properties import FlowFile as FlowFileProxy from .properties import ProcessContext as ProcessContextProxy @@ -42,7 +44,9 @@ def getPartition(self): class RecordTransformResult: - def __init__(self, record=None, schema=None, relationship="success", partition=None): + def __init__( + self, record=None, schema=None, relationship="success", partition=None + ): self.record = record self.schema = schema self.relationship = relationship @@ -63,24 +67,44 @@ def getPartition(self): class RecordTransform(ProcessorBase): RECORD_READER = PropertyDescriptor( - name='Record Reader', - display_name='Record Reader', - description='''Specifies the Controller Service to use for reading incoming data''', + name="Record Reader", + display_name="Record Reader", + description="""Specifies the Controller Service to use for reading incoming data""", required=True, - controller_service_definition='RecordSetReader' + controller_service_definition="RecordSetReader", ) RECORD_WRITER = PropertyDescriptor( - name='Record Writer', - display_name='Record Writer', - description='''Specifies the Controller Service to use for writing out the records''', + name="Record Writer", + display_name="Record Writer", + description="""Specifies the Controller Service to use for writing out the records""", required=True, - controller_service_definition='RecordSetWriter', + controller_service_definition="RecordSetWriter", ) def onInitialize(self, processor: Processor): - super(RecordTransform, self).onInitialize(processor) - processor.addProperty(self.RECORD_READER.name, self.RECORD_READER.description, None, self.RECORD_READER.required, False, False, None, None, self.RECORD_READER.controllerServiceDefinition) - processor.addProperty(self.RECORD_WRITER.name, self.RECORD_WRITER.description, None, self.RECORD_WRITER.required, False, False, None, None, self.RECORD_WRITER.controllerServiceDefinition) + super().onInitialize(processor) + processor.addProperty( + self.RECORD_READER.name, + self.RECORD_READER.description, + None, + self.RECORD_READER.required, + False, + False, + None, + None, + self.RECORD_READER.controllerServiceDefinition, + ) + processor.addProperty( + self.RECORD_WRITER.name, + self.RECORD_WRITER.description, + None, + self.RECORD_WRITER.required, + False, + False, + None, + None, + self.RECORD_WRITER.controllerServiceDefinition, + ) def onTrigger(self, context: ProcessContext, session: ProcessSession): flow_file = session.get() @@ -88,12 +112,16 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): return context_proxy = ProcessContextProxy(context, self) - record_reader = context_proxy.getProperty(self.RECORD_READER).asControllerService() + record_reader = context_proxy.getProperty( + self.RECORD_READER + ).asControllerService() if not record_reader: self.logger.error("Record Reader property is invalid") session.transfer(flow_file, self.REL_FAILURE) return - record_writer = context_proxy.getProperty(self.RECORD_WRITER).asControllerService() + record_writer = context_proxy.getProperty( + self.RECORD_WRITER + ).asControllerService() if not record_writer: self.logger.error("Record Writer property is invalid") session.transfer(flow_file, self.REL_FAILURE) @@ -106,7 +134,9 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): session.transfer(flow_file, self.REL_FAILURE) return except Exception: - self.logger.error("Failed to read flow file records due to the following error:\n{}".format(traceback.format_exc())) + self.logger.error( + f"Failed to read flow file records due to the following error:\n{traceback.format_exc()}" + ) session.transfer(flow_file, self.REL_FAILURE) return @@ -115,12 +145,18 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): for record in record_list: record_json = json.loads(record) try: - result = self.transform(context_proxy, record_json, None, flow_file_proxy) + result = self.transform( + context_proxy, record_json, None, flow_file_proxy + ) result_record = result.getRecord() - resultjson = None if result_record is None else json.dumps(result_record) + resultjson = ( + None if result_record is None else json.dumps(result_record) + ) results.append(__RecordTransformResult__(result, resultjson)) except Exception: - self.logger.error("Failed to transform record due to the following error:\n{}".format(traceback.format_exc())) + self.logger.error( + f"Failed to transform record due to the following error:\n{traceback.format_exc()}" + ) session.transfer(flow_file, self.REL_FAILURE) return @@ -139,14 +175,22 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): for single_partition_results in partitioned_results_list: partitioned_flow_file = session.create(flow_file) - record_writer.write([result.getRecordJson() for result in single_partition_results], partitioned_flow_file, session) + record_writer.write( + [result.getRecordJson() for result in single_partition_results], + partitioned_flow_file, + session, + ) if result.getRelationship() == "success": session.transfer(partitioned_flow_file, self.REL_SUCCESS) else: - session.transferToCustomRelationship(partitioned_flow_file, result.getRelationship()) + session.transferToCustomRelationship( + partitioned_flow_file, result.getRelationship() + ) session.transfer(flow_file, self.REL_ORIGINAL) @abstractmethod - def transform(self, context: ProcessContextProxy, record_json, schema, flowFile: FlowFileProxy) -> RecordTransformResult: + def transform( + self, context: ProcessContextProxy, record_json, schema, flowFile: FlowFileProxy + ) -> RecordTransformResult: pass diff --git a/extensions/python/pythonprocessors/nifiapi/relationship.py b/extensions/python/pythonprocessors/nifiapi/relationship.py index 2504c5d803..58fda3d907 100644 --- a/extensions/python/pythonprocessors/nifiapi/relationship.py +++ b/extensions/python/pythonprocessors/nifiapi/relationship.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + class Relationship: def __init__(self, name, description, auto_terminated=False): self.name = name diff --git a/extensions/python/tests/features/environment.py b/extensions/python/tests/features/environment.py index 648377cda3..be12317ae9 100644 --- a/extensions/python/tests/features/environment.py +++ b/extensions/python/tests/features/environment.py @@ -12,63 +12,69 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import docker import os from pathlib import Path + from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def get_minifi_container_image(): - if 'MINIFI_TAG_PREFIX' in os.environ and 'MINIFI_VERSION' in os.environ: - minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] - minifi_version = os.environ['MINIFI_VERSION'] - return 'apacheminificpp:' + minifi_tag_prefix + minifi_version + if "MINIFI_TAG_PREFIX" in os.environ and "MINIFI_VERSION" in os.environ: + minifi_tag_prefix = os.environ["MINIFI_TAG_PREFIX"] + minifi_version = os.environ["MINIFI_VERSION"] + return "apacheminificpp:" + minifi_tag_prefix + minifi_version return "apacheminificpp:behave" def before_all(context): context.minifi_container_image = get_minifi_container_image() client: docker.DockerClient = docker.from_env() - is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(client.images.get(context.minifi_container_image).history()) + is_fhs = "MINIFI_INSTALLATION_TYPE=FHS" in str( + client.images.get(context.minifi_container_image).history() + ) pip3_install_command = "" - minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] if 'MINIFI_TAG_PREFIX' in os.environ else "" + minifi_tag_prefix = os.environ.get("MINIFI_TAG_PREFIX", "") if not minifi_tag_prefix: pip3_install_command = "RUN apk --update --no-cache add py3-pip" - minifi_python_dir_path = '/var/lib/nifi-minifi-cpp/minifi-python' if is_fhs else '/opt/minifi/minifi-current/minifi-python' - minifi_python_venv_parent = '/var/lib/nifi-minifi-cpp' if is_fhs else '/opt/minifi/minifi-current' - dockerfile = """ -FROM {base_image} + minifi_python_dir_path = ( + "/var/lib/nifi-minifi-cpp/minifi-python" + if is_fhs + else "/opt/minifi/minifi-current/minifi-python" + ) + minifi_python_venv_parent = ( + "/var/lib/nifi-minifi-cpp" if is_fhs else "/opt/minifi/minifi-current" + ) + dockerfile = f""" +FROM {context.minifi_container_image} USER root {pip3_install_command} USER minificpp -COPY RotatingForwarder.py {minifi_python_dir}/nifi_python_processors/RotatingForwarder.py -COPY SpecialPropertyTypeChecker.py {minifi_python_dir}/nifi_python_processors/SpecialPropertyTypeChecker.py -COPY ProcessContextInterfaceChecker.py {minifi_python_dir}/nifi_python_processors/ProcessContextInterfaceChecker.py -COPY CreateFlowFile.py {minifi_python_dir}/nifi_python_processors/CreateFlowFile.py -COPY FailureWithAttributes.py {minifi_python_dir}/nifi_python_processors/FailureWithAttributes.py -COPY subtractutils.py {minifi_python_dir}/nifi_python_processors/compute/subtractutils.py -COPY RelativeImporterProcessor.py {minifi_python_dir}/nifi_python_processors/compute/processors/RelativeImporterProcessor.py -COPY multiplierutils.py {minifi_python_dir}/nifi_python_processors/compute/processors/multiplierutils.py -COPY CreateNothing.py {minifi_python_dir}/nifi_python_processors/CreateNothing.py -COPY FailureWithContent.py {minifi_python_dir}/nifi_python_processors/FailureWithContent.py -COPY TransferToOriginal.py {minifi_python_dir}/nifi_python_processors/TransferToOriginal.py -COPY SetRecordField.py {minifi_python_dir}/nifi_python_processors/SetRecordField.py -COPY TestStateManager.py {minifi_python_dir}/nifi_python_processors/TestStateManager.py -COPY NifiStyleLogDynamicProperties.py {minifi_python_dir}/nifi_python_processors/NifiStyleLogDynamicProperties.py -COPY LogDynamicProperties.py {minifi_python_dir}/LogDynamicProperties.py -COPY ExpressionLanguagePropertyWithValidator.py {minifi_python_dir}/nifi_python_processors/ExpressionLanguagePropertyWithValidator.py -COPY EvaluateExpressionLanguageChecker.py {minifi_python_dir}/nifi_python_processors/EvaluateExpressionLanguageChecker.py -COPY SleepForever.py {minifi_python_dir}/nifi_python_processors/SleepForever.py -COPY SingleThreadedSleepForever.py {minifi_python_dir}/nifi_python_processors/SingleThreadedSleepForever.py -COPY MinifiSleepForever.py {minifi_python_dir}/MinifiSleepForever.py -COPY SingleThreadedMinifiSleepForever.py {minifi_python_dir}/SingleThreadedMinifiSleepForever.py +COPY RotatingForwarder.py {minifi_python_dir_path}/nifi_python_processors/RotatingForwarder.py +COPY SpecialPropertyTypeChecker.py {minifi_python_dir_path}/nifi_python_processors/SpecialPropertyTypeChecker.py +COPY ProcessContextInterfaceChecker.py {minifi_python_dir_path}/nifi_python_processors/ProcessContextInterfaceChecker.py +COPY CreateFlowFile.py {minifi_python_dir_path}/nifi_python_processors/CreateFlowFile.py +COPY FailureWithAttributes.py {minifi_python_dir_path}/nifi_python_processors/FailureWithAttributes.py +COPY subtractutils.py {minifi_python_dir_path}/nifi_python_processors/compute/subtractutils.py +COPY RelativeImporterProcessor.py {minifi_python_dir_path}/nifi_python_processors/compute/processors/RelativeImporterProcessor.py +COPY multiplierutils.py {minifi_python_dir_path}/nifi_python_processors/compute/processors/multiplierutils.py +COPY CreateNothing.py {minifi_python_dir_path}/nifi_python_processors/CreateNothing.py +COPY FailureWithContent.py {minifi_python_dir_path}/nifi_python_processors/FailureWithContent.py +COPY TransferToOriginal.py {minifi_python_dir_path}/nifi_python_processors/TransferToOriginal.py +COPY SetRecordField.py {minifi_python_dir_path}/nifi_python_processors/SetRecordField.py +COPY TestStateManager.py {minifi_python_dir_path}/nifi_python_processors/TestStateManager.py +COPY NifiStyleLogDynamicProperties.py {minifi_python_dir_path}/nifi_python_processors/NifiStyleLogDynamicProperties.py +COPY LogDynamicProperties.py {minifi_python_dir_path}/LogDynamicProperties.py +COPY ExpressionLanguagePropertyWithValidator.py {minifi_python_dir_path}/nifi_python_processors/ExpressionLanguagePropertyWithValidator.py +COPY EvaluateExpressionLanguageChecker.py {minifi_python_dir_path}/nifi_python_processors/EvaluateExpressionLanguageChecker.py +COPY SleepForever.py {minifi_python_dir_path}/nifi_python_processors/SleepForever.py +COPY SingleThreadedSleepForever.py {minifi_python_dir_path}/nifi_python_processors/SingleThreadedSleepForever.py +COPY MinifiSleepForever.py {minifi_python_dir_path}/MinifiSleepForever.py +COPY SingleThreadedMinifiSleepForever.py {minifi_python_dir_path}/SingleThreadedMinifiSleepForever.py RUN python3 -m venv {minifi_python_venv_parent}/venv - """.format(base_image=context.minifi_container_image, - pip3_install_command=pip3_install_command, - minifi_python_dir=minifi_python_dir_path, - minifi_python_venv_parent=minifi_python_venv_parent) + """ build_context_path = str(Path(__file__).resolve().parent / "resources") files_on_context = {} @@ -80,7 +86,7 @@ def before_all(context): builder = DockerImageBuilder( image_tag="apacheminificpp-python:latest", dockerfile_content=dockerfile, - files_on_context=files_on_context + files_on_context=files_on_context, ) builder.build() @@ -89,7 +95,7 @@ def is_conda_available_in_minifi_image(context): client: docker.DockerClient = docker.from_env() container = client.containers.create( image=context.minifi_container_image, - command=['conda', '--version'], + command=["conda", "--version"], ) try: container.start() @@ -99,26 +105,31 @@ def is_conda_available_in_minifi_image(context): container.remove(force=True) return False - return result.decode('utf-8').startswith('conda ') + return result.decode("utf-8").startswith("conda ") def get_minifi_image_python_version(context): client: docker.DockerClient = docker.from_env() result = client.containers.run( image=context.minifi_container_image, - command=['python3', '-c', 'import platform; print(platform.python_version())'], - remove=True + command=["python3", "-c", "import platform; print(platform.python_version())"], + remove=True, ) - python_ver_str = result.decode('utf-8') - return tuple(map(int, python_ver_str.split('.'))) + python_ver_str = result.decode("utf-8") + return tuple(map(int, python_ver_str.split("."))) def before_scenario(context, scenario): - if "USE_NIFI_PYTHON_PROCESSORS_WITH_LANGCHAIN" in scenario.effective_tags: - if not is_conda_available_in_minifi_image(context) and get_minifi_image_python_version(context) < (3, 8, 1): - scenario.skip("NiFi Python processor tests use langchain library which requires Python 3.8.1 or later.") - return + if ( + "USE_NIFI_PYTHON_PROCESSORS_WITH_LANGCHAIN" in scenario.effective_tags + and not is_conda_available_in_minifi_image(context) + and get_minifi_image_python_version(context) < (3, 8, 1) + ): + scenario.skip( + "NiFi Python processor tests use langchain library which requires Python 3.8.1 or later." + ) + return common_before_scenario(context, scenario) context.minifi_container_image = "apacheminificpp-python:latest" diff --git a/extensions/python/tests/features/resources/CreateFlowFile.py b/extensions/python/tests/features/resources/CreateFlowFile.py index 948734982d..f6ba1b4f14 100644 --- a/extensions/python/tests/features/resources/CreateFlowFile.py +++ b/extensions/python/tests/features/resources/CreateFlowFile.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfilesource import FlowFileSource, FlowFileSourceResult from nifiapi.properties import PropertyDescriptor from nifiapi.relationship import Relationship @@ -20,23 +22,25 @@ class CreateFlowFile(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileSource"] class ProcessorDetails: - version = '2.0.0-snapshot' - description = '''Test Python source processor.''' - tags = ['text', 'test', 'python', 'source'] + version = "2.0.0-snapshot" + description = """Test Python source processor.""" + tags: ClassVar[list] = ["text", "test", "python", "source"] FF_CONTENTS = PropertyDescriptor( - name='FlowFile Contents', - description='''The contents of the FlowFile.''', + name="FlowFile Contents", + description="""The contents of the FlowFile.""", required=True, - default_value='Hello World!' + default_value="Hello World!", ) - property_descriptors = [FF_CONTENTS] + property_descriptors: ClassVar[list] = [FF_CONTENTS] - REL_MULTILINE = Relationship(name='space', description='FlowFiles that contain space characters.') + REL_MULTILINE = Relationship( + name="space", description="FlowFiles that contain space characters." + ) def __init__(self, **kwargs): pass @@ -52,7 +56,13 @@ def create(self, context): if contents is not None and isinstance(contents, str): contents_str = str.encode(contents) - if b' ' in contents_str: - return FlowFileSourceResult(relationship='space', attributes={"type": "space"}, contents=contents_str) + if b" " in contents_str: + return FlowFileSourceResult( + relationship="space", + attributes={"type": "space"}, + contents=contents_str, + ) - return FlowFileSourceResult(relationship='success', attributes={"type": "non-space"}, contents=contents) + return FlowFileSourceResult( + relationship="success", attributes={"type": "non-space"}, contents=contents + ) diff --git a/extensions/python/tests/features/resources/CreateNothing.py b/extensions/python/tests/features/resources/CreateNothing.py index acef36dd85..7c6a8f69e7 100644 --- a/extensions/python/tests/features/resources/CreateNothing.py +++ b/extensions/python/tests/features/resources/CreateNothing.py @@ -12,22 +12,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfilesource import FlowFileSource class CreateNothing(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileSource"] class ProcessorDetails: - version = '0.0.1-SNAPSHOT' - description = '''A Python processor for testing a use-case where the Source processor - does not create any output.''' - tags = ['test', 'python', 'source'] + version = "0.0.1-SNAPSHOT" + description = """A Python processor for testing a use-case where the Source processor + does not create any output.""" + tags: ClassVar[list] = ["test", "python", "source"] def __init__(self, **kwargs): pass def create(self, context): context.yield_resources() - return None diff --git a/extensions/python/tests/features/resources/EvaluateExpressionLanguageChecker.py b/extensions/python/tests/features/resources/EvaluateExpressionLanguageChecker.py index 269e7ca47c..d8cb75842b 100644 --- a/extensions/python/tests/features/resources/EvaluateExpressionLanguageChecker.py +++ b/extensions/python/tests/features/resources/EvaluateExpressionLanguageChecker.py @@ -13,13 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult -from nifiapi.properties import PropertyDescriptor, ExpressionLanguageScope +from nifiapi.properties import ExpressionLanguageScope, PropertyDescriptor class EvaluateExpressionLanguageChecker(FlowFileTransform): class Java: - implements = ["org.apache.nifi.python.processor.FlowFileTransform"] + implements: ClassVar[list] = [ + "org.apache.nifi.python.processor.FlowFileTransform" + ] class ProcessorDetails: version = "0.1.0" @@ -28,7 +32,7 @@ class ProcessorDetails: EL_PROPERTY = PropertyDescriptor( name="EL Property", description="EL property", - expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES + expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES, ) NON_EL_PROPERTY = PropertyDescriptor( @@ -36,10 +40,7 @@ class ProcessorDetails: description="Non EL property", ) - property_descriptors = [ - EL_PROPERTY, - NON_EL_PROPERTY - ] + property_descriptors: ClassVar[list] = [EL_PROPERTY, NON_EL_PROPERTY] def __init__(self, **kwargs): pass @@ -48,36 +49,55 @@ def getPropertyDescriptors(self): return self.property_descriptors def getDynamicPropertyDescriptor(self, propertyname): - return PropertyDescriptor(name=propertyname, - description="A user-defined property", - dynamic=True) + return PropertyDescriptor( + name=propertyname, description="A user-defined property", dynamic=True + ) def transform(self, context, flowFile): el_property = context.getProperty(self.EL_PROPERTY) el_property_value = el_property.getValue() self.logger.info("EL Property value: " + str(el_property_value)) - el_property_value_evaluated = el_property.evaluateAttributeExpressions(flowFile).getValue() - self.logger.info("Evaluated EL Property value: " + str(el_property_value_evaluated)) + el_property_value_evaluated = el_property.evaluateAttributeExpressions( + flowFile + ).getValue() + self.logger.info( + "Evaluated EL Property value: " + str(el_property_value_evaluated) + ) non_el_property = context.getProperty(self.NON_EL_PROPERTY) non_el_property_value = non_el_property.getValue() self.logger.info("Non EL Property value: " + str(non_el_property_value)) - non_el_property_value_evaluated = non_el_property.evaluateAttributeExpressions(flowFile).getValue() - self.logger.info("Evaluated Non EL Property value: " + str(non_el_property_value_evaluated)) - - non_existent_value = context.getProperty("non-existent-property").evaluateAttributeExpressions(flowFile).getValue() + non_el_property_value_evaluated = non_el_property.evaluateAttributeExpressions( + flowFile + ).getValue() + self.logger.info( + "Evaluated Non EL Property value: " + str(non_el_property_value_evaluated) + ) + + non_existent_value = ( + context.getProperty("non-existent-property") + .evaluateAttributeExpressions(flowFile) + .getValue() + ) if non_existent_value is None: self.logger.info("Non-existent property value is empty") dynamic_property = context.getProperty("My Dynamic Property") dynamic_property_value = dynamic_property.getValue() if dynamic_property_value: - self.logger.info("My Dynamic Property value is: " + str(dynamic_property_value)) + self.logger.info( + "My Dynamic Property value is: " + str(dynamic_property_value) + ) - dynamic_property_evaluated_value = dynamic_property.evaluateAttributeExpressions(flowFile).getValue() + dynamic_property_evaluated_value = ( + dynamic_property.evaluateAttributeExpressions(flowFile).getValue() + ) if dynamic_property_evaluated_value: - self.logger.info("My Dynamic Property evaluated value is: " + str(dynamic_property_evaluated_value)) + self.logger.info( + "My Dynamic Property evaluated value is: " + + str(dynamic_property_evaluated_value) + ) return FlowFileTransformResult("success", contents="Check successful!") diff --git a/extensions/python/tests/features/resources/ExpressionLanguagePropertyWithValidator.py b/extensions/python/tests/features/resources/ExpressionLanguagePropertyWithValidator.py index 346f1ede41..036d23dc99 100644 --- a/extensions/python/tests/features/resources/ExpressionLanguagePropertyWithValidator.py +++ b/extensions/python/tests/features/resources/ExpressionLanguagePropertyWithValidator.py @@ -13,26 +13,33 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult -from nifiapi.properties import ExpressionLanguageScope, PropertyDescriptor, StandardValidators +from nifiapi.properties import ( + ExpressionLanguageScope, + PropertyDescriptor, + StandardValidators, +) class ExpressionLanguagePropertyWithValidator(FlowFileTransform): - class Java: - implements = ['org.apache.nifi.python.processor.FlowFileTransform'] + implements: ClassVar[list] = [ + "org.apache.nifi.python.processor.FlowFileTransform" + ] class ProcessorDetails: - version = '1.2.3' + version = "1.2.3" description = "Test processor" - dependencies = [] + dependencies: ClassVar[list] = [] INTEGER_PROPERTY = PropertyDescriptor( name="Integer Property", description="A dummy integer property", required=True, validators=[StandardValidators.INTEGER_VALIDATOR], - expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES + expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES, ) def __init__(self, **kwargs): @@ -42,12 +49,12 @@ def getPropertyDescriptors(self): return [self.INTEGER_PROPERTY] def getDynamicPropertyDescriptor(self, propertyname): - return PropertyDescriptor(name=propertyname, - description="A user-defined property", - dynamic=True) + return PropertyDescriptor( + name=propertyname, description="A user-defined property", dynamic=True + ) def transform(self, context, flow_file): integer_value = context.getProperty(self.INTEGER_PROPERTY).asInteger() - self.logger.info("Integer Property value: {}".format(integer_value)) + self.logger.info(f"Integer Property value: {integer_value}") - return FlowFileTransformResult('success', contents="content") + return FlowFileTransformResult("success", contents="content") diff --git a/extensions/python/tests/features/resources/LogDynamicProperties.py b/extensions/python/tests/features/resources/LogDynamicProperties.py index 727e383634..edd850d926 100644 --- a/extensions/python/tests/features/resources/LogDynamicProperties.py +++ b/extensions/python/tests/features/resources/LogDynamicProperties.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + def describe(processor): processor.setDescription("Test description.") @@ -25,11 +26,11 @@ def onInitialize(processor): def onTrigger(context, session): flow_file = session.get() property_value = context.getProperty("Static Property") - log.info("Static Property value: {}".format(property_value)) + log.info(f"Static Property value: {property_value}") dyn_property_value = context.getDynamicProperty("Dynamic Property") - log.info("Dynamic Property value: {}".format(dyn_property_value)) + log.info(f"Dynamic Property value: {dyn_property_value}") keys = context.getDynamicPropertyKeys() - log.info("dynamic property key count: {}".format(len(keys))) + log.info(f"dynamic property key count: {len(keys)}") for dynamic_property_key in keys: - log.info("dynamic property key: {}".format(dynamic_property_key)) + log.info(f"dynamic property key: {dynamic_property_key}") session.transfer(flow_file, REL_SUCCESS) diff --git a/extensions/python/tests/features/resources/NifiStyleLogDynamicProperties.py b/extensions/python/tests/features/resources/NifiStyleLogDynamicProperties.py index dc989f6b0a..9c1204e458 100644 --- a/extensions/python/tests/features/resources/NifiStyleLogDynamicProperties.py +++ b/extensions/python/tests/features/resources/NifiStyleLogDynamicProperties.py @@ -13,26 +13,33 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult -from nifiapi.properties import ExpressionLanguageScope, PropertyDescriptor, StandardValidators +from nifiapi.properties import ( + ExpressionLanguageScope, + PropertyDescriptor, + StandardValidators, +) class NifiStyleLogDynamicProperties(FlowFileTransform): - class Java: - implements = ['org.apache.nifi.python.processor.FlowFileTransform'] + implements: ClassVar[list] = [ + "org.apache.nifi.python.processor.FlowFileTransform" + ] class ProcessorDetails: - version = '1.2.3' + version = "1.2.3" description = "Test processor" - dependencies = [] + dependencies: ClassVar[list] = [] STATIC_PROPERTY = PropertyDescriptor( name="Static Property", description="A dummy static property", required=True, validators=[StandardValidators.NON_EMPTY_VALIDATOR], - expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES + expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES, ) def __init__(self, **kwargs): @@ -42,14 +49,14 @@ def getPropertyDescriptors(self): return [self.STATIC_PROPERTY] def getDynamicPropertyDescriptor(self, propertyname): - return PropertyDescriptor(name=propertyname, - description="A user-defined property", - dynamic=True) + return PropertyDescriptor( + name=propertyname, description="A user-defined property", dynamic=True + ) def transform(self, context, flow_file): property_value = context.getProperty("Static Property") - self.logger.info("Static Property value: {}".format(property_value.getValue())) + self.logger.info(f"Static Property value: {property_value.getValue()}") dyn_property_value = context.getProperty("Dynamic Property") - self.logger.info("Dynamic Property value: {}".format(dyn_property_value.getValue())) + self.logger.info(f"Dynamic Property value: {dyn_property_value.getValue()}") - return FlowFileTransformResult('success', contents="content") + return FlowFileTransformResult("success", contents="content") diff --git a/extensions/python/tests/features/resources/ProcessContextInterfaceChecker.py b/extensions/python/tests/features/resources/ProcessContextInterfaceChecker.py index 5d0136adef..7e42ca777d 100644 --- a/extensions/python/tests/features/resources/ProcessContextInterfaceChecker.py +++ b/extensions/python/tests/features/resources/ProcessContextInterfaceChecker.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult from nifiapi.properties import PropertyDescriptor, StandardValidators from nifiapi.relationship import Relationship @@ -22,33 +24,34 @@ class ProcessContextInterfaceChecker(FlowFileTransform): """ Checks ProcessContext interface """ + SECRET_PASSWORD = PropertyDescriptor( name="Secret Password", description="Password to access", validators=[StandardValidators.NON_EMPTY_VALIDATOR], default_value="mysecret", required=True, - sensitive=True + sensitive=True, ) REQUEST_TIMEOUT = PropertyDescriptor( name="Request Timeout", description="Timeout for the request", validators=[StandardValidators.TIME_PERIOD_VALIDATOR], default_value="60 sec", - required=True + required=True, ) WISH_COUNT = PropertyDescriptor( name="Wish Count", description="Number of wishes", default_value="3", validators=[StandardValidators.POSITIVE_INTEGER_VALIDATOR], - required=True + required=True, ) - property_descriptors = [ + property_descriptors: ClassVar[list] = [ SECRET_PASSWORD, REQUEST_TIMEOUT, - WISH_COUNT + WISH_COUNT, ] def __init__(self, **kwargs): @@ -66,15 +69,26 @@ def transform(self, context, flowFile): return FlowFileTransformResult("failure") property_names = [property.name for property in properties] - if "Secret Password" not in property_names or "Request Timeout" not in property_names or "Wish Count" not in property_names: + if ( + "Secret Password" not in property_names + or "Request Timeout" not in property_names + or "Wish Count" not in property_names + ): return FlowFileTransformResult("failure") for property in properties: - if property.name == "Secret Password" and properties[property] != "mysecret": - return FlowFileTransformResult("failure") - elif property.name == "Request Timeout" and properties[property] != "60 sec": - return FlowFileTransformResult("failure") - elif property.name == "Wish Count" and properties[property] != "3": + if ( + ( + property.name == "Secret Password" + and properties[property] != "mysecret" + ) + or ( + property.name == "Request Timeout" + and properties[property] != "60 sec" + ) + or property.name == "Wish Count" + and properties[property] != "3" + ): return FlowFileTransformResult("failure") secret_password = context.getProperty(self.SECRET_PASSWORD).getValue() diff --git a/extensions/python/tests/features/resources/RelativeImporterProcessor.py b/extensions/python/tests/features/resources/RelativeImporterProcessor.py index 1ec2d254f5..5c8e2d2e37 100644 --- a/extensions/python/tests/features/resources/RelativeImporterProcessor.py +++ b/extensions/python/tests/features/resources/RelativeImporterProcessor.py @@ -14,8 +14,9 @@ # limitations under the License. from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult -from .multiplierutils import double + from ..subtractutils import minus_ten +from .multiplierutils import double class RelativeImporterProcessor(FlowFileTransform): @@ -26,4 +27,6 @@ def transform(self, context, flowFile): number = 1000 number = double(number) number = minus_ten(number) - return FlowFileTransformResult("success", contents="The final result is {}".format(number)) + return FlowFileTransformResult( + "success", contents=f"The final result is {number}" + ) diff --git a/extensions/python/tests/features/resources/RotatingForwarder.py b/extensions/python/tests/features/resources/RotatingForwarder.py index 09df09fbf5..c48fee4579 100644 --- a/extensions/python/tests/features/resources/RotatingForwarder.py +++ b/extensions/python/tests/features/resources/RotatingForwarder.py @@ -20,6 +20,7 @@ class RotatingForwarder(FlowFileTransform): """ Forwards flow files to a different relationship each time it is called in a round robin manner. """ + def __init__(self, **kwargs): self.relationship_index = 0 self.relationships = ["first", "second", "third", "fourth"] @@ -27,7 +28,9 @@ def __init__(self, **kwargs): def transform(self, context, flowFile): content = flowFile.getContentsAsBytes().decode() - relationship = self.relationships[self.relationship_index % len(self.relationships)] + relationship = self.relationships[ + self.relationship_index % len(self.relationships) + ] self.relationship_index += 1 self.relationship_index %= len(self.relationships) return FlowFileTransformResult(relationship, contents=content) diff --git a/extensions/python/tests/features/resources/SetRecordField.py b/extensions/python/tests/features/resources/SetRecordField.py index dbb3a2ae64..d5e6521068 100644 --- a/extensions/python/tests/features/resources/SetRecordField.py +++ b/extensions/python/tests/features/resources/SetRecordField.py @@ -13,45 +13,56 @@ # See the License for the specific language governing permissions and # limitations under the License. -from nifiapi.properties import PropertyDescriptor -from nifiapi.properties import StandardValidators -from nifiapi.properties import ExpressionLanguageScope -from nifiapi.recordtransform import RecordTransformResult -from nifiapi.recordtransform import RecordTransform +from typing import ClassVar + +from nifiapi.properties import ( + ExpressionLanguageScope, + PropertyDescriptor, + StandardValidators, +) +from nifiapi.recordtransform import RecordTransform, RecordTransformResult class SetRecordField(RecordTransform): class Java: - implements = ['org.apache.nifi.python.processor.RecordTransform'] + implements: ClassVar[list] = [ + "org.apache.nifi.python.processor.RecordTransform" + ] class ProcessorDetails: - version = '0.0.1-SNAPSHOT' + version = "0.0.1-SNAPSHOT" def __init__(self, **kwargs): super().__init__() def transform(self, context, record, schema, attributemap): # Update dictionary based on the dynamic properties provided by user - for key in context.getProperties().keys(): + for key in context.getProperties(): if not key.dynamic: continue propname = key.name - record[propname] = context.getProperty(propname).evaluateAttributeExpressions(attributemap).getValue() + record[propname] = ( + context.getProperty(propname) + .evaluateAttributeExpressions(attributemap) + .getValue() + ) # Determine the partition - if 'group' in record: - partition = {'group': record['group']} + if "group" in record: + partition = {"group": record["group"]} else: partition = None # Return the result - return RecordTransformResult(record=record, relationship='success', partition=partition) + return RecordTransformResult( + record=record, relationship="success", partition=partition + ) def getDynamicPropertyDescriptor(self, name): return PropertyDescriptor( name=name, description="Specifies the value to set for the '" + name + "' field", expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES, - validators=[StandardValidators.ALWAYS_VALID] + validators=[StandardValidators.ALWAYS_VALID], ) diff --git a/extensions/python/tests/features/resources/SingleThreadedSleepForever.py b/extensions/python/tests/features/resources/SingleThreadedSleepForever.py index 93c23e382b..d69b6f7187 100644 --- a/extensions/python/tests/features/resources/SingleThreadedSleepForever.py +++ b/extensions/python/tests/features/resources/SingleThreadedSleepForever.py @@ -14,19 +14,21 @@ # limitations under the License. import time -from nifiapi.flowfilesource import FlowFileSource +from typing import ClassVar + from nifiapi.decorators import trigger_serially +from nifiapi.flowfilesource import FlowFileSource @trigger_serially class SingleThreadedSleepForever(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileSource"] class ProcessorDetails: - version = '2.0.0-snapshot' - description = '''NiFi-style sleep forever python processor (single-threaded).''' - tags = ['test', 'python', 'sleep'] + version = "2.0.0-snapshot" + description = """NiFi-style sleep forever python processor (single-threaded).""" + tags: ClassVar[list] = ["test", "python", "sleep"] def __init__(self, **kwargs): pass diff --git a/extensions/python/tests/features/resources/SleepForever.py b/extensions/python/tests/features/resources/SleepForever.py index ee3d034ee9..a4cfb6b678 100644 --- a/extensions/python/tests/features/resources/SleepForever.py +++ b/extensions/python/tests/features/resources/SleepForever.py @@ -14,17 +14,19 @@ # limitations under the License. import time +from typing import ClassVar + from nifiapi.flowfilesource import FlowFileSource class SleepForever(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileSource"] class ProcessorDetails: - version = '2.0.0-snapshot' - description = '''NiFi-style sleep forever python processor.''' - tags = ['test', 'python', 'sleep'] + version = "2.0.0-snapshot" + description = """NiFi-style sleep forever python processor.""" + tags: ClassVar[list] = ["test", "python", "sleep"] def __init__(self, **kwargs): pass diff --git a/extensions/python/tests/features/resources/SpecialPropertyTypeChecker.py b/extensions/python/tests/features/resources/SpecialPropertyTypeChecker.py index f9b0a35aa2..d0b74189c5 100644 --- a/extensions/python/tests/features/resources/SpecialPropertyTypeChecker.py +++ b/extensions/python/tests/features/resources/SpecialPropertyTypeChecker.py @@ -13,37 +13,40 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult -from nifiapi.properties import PropertyDescriptor, TimeUnit, DataUnit +from nifiapi.properties import DataUnit, PropertyDescriptor, TimeUnit class SpecialPropertyTypeChecker(FlowFileTransform): """ Checks if custom property types are working as expected """ + TIME_PERIOD_PROPERTY = PropertyDescriptor( name="Time Period Property", description="Dummy property that should be a time period", default_value="2 hours", - required=True + required=True, ) DATA_SIZE_PROPERTY = PropertyDescriptor( name="Data Size Property", description="Dummy property that should be a data size", default_value="100 MB", - required=True + required=True, ) SSL_CONTEXT_PROPERTY = PropertyDescriptor( name="SSL Context Service", description="Dummy property that should be an SSL Context", controller_service_definition="SSLContextService", - required=True + required=True, ) - property_descriptors = [ + property_descriptors: ClassVar[list] = [ TIME_PERIOD_PROPERTY, DATA_SIZE_PROPERTY, - SSL_CONTEXT_PROPERTY + SSL_CONTEXT_PROPERTY, ] def __init__(self, **kwargs): @@ -53,57 +56,99 @@ def getPropertyDescriptors(self): return self.property_descriptors def transform(self, context, flowFile): - time_in_microseconds = context.getProperty(self.TIME_PERIOD_PROPERTY).asTimePeriod(TimeUnit.MICROSECONDS) + time_in_microseconds = context.getProperty( + self.TIME_PERIOD_PROPERTY + ).asTimePeriod(TimeUnit.MICROSECONDS) if time_in_microseconds != 7200000000: - self.logger.error("Time period property conversion to microseconds is not working as expected") + self.logger.error( + "Time period property conversion to microseconds is not working as expected" + ) return FlowFileTransformResult("failure") - time_in_milliseconds = context.getProperty(self.TIME_PERIOD_PROPERTY).asTimePeriod(TimeUnit.MILLISECONDS) + time_in_milliseconds = context.getProperty( + self.TIME_PERIOD_PROPERTY + ).asTimePeriod(TimeUnit.MILLISECONDS) if time_in_milliseconds != 7200000: - self.logger.error("Time period property conversion to milliseconds is not working as expected") + self.logger.error( + "Time period property conversion to milliseconds is not working as expected" + ) return FlowFileTransformResult("failure") - time_in_seconds = context.getProperty(self.TIME_PERIOD_PROPERTY).asTimePeriod(TimeUnit.SECONDS) + time_in_seconds = context.getProperty(self.TIME_PERIOD_PROPERTY).asTimePeriod( + TimeUnit.SECONDS + ) if time_in_seconds != 7200: - self.logger.error("Time period property conversion to seconds is not working as expected") + self.logger.error( + "Time period property conversion to seconds is not working as expected" + ) return FlowFileTransformResult("failure") - time_in_minutes = context.getProperty(self.TIME_PERIOD_PROPERTY).asTimePeriod(TimeUnit.MINUTES) + time_in_minutes = context.getProperty(self.TIME_PERIOD_PROPERTY).asTimePeriod( + TimeUnit.MINUTES + ) if time_in_minutes != 120: - self.logger.error("Time period property conversion to minutes is not working as expected") + self.logger.error( + "Time period property conversion to minutes is not working as expected" + ) return FlowFileTransformResult("failure") - time_in_hours = context.getProperty(self.TIME_PERIOD_PROPERTY).asTimePeriod(TimeUnit.HOURS) + time_in_hours = context.getProperty(self.TIME_PERIOD_PROPERTY).asTimePeriod( + TimeUnit.HOURS + ) if time_in_hours != 2: - self.logger.error("Time period property conversion to hours is not working as expected") + self.logger.error( + "Time period property conversion to hours is not working as expected" + ) return FlowFileTransformResult("failure") - time_in_days = context.getProperty(self.TIME_PERIOD_PROPERTY).asTimePeriod(TimeUnit.DAYS) + time_in_days = context.getProperty(self.TIME_PERIOD_PROPERTY).asTimePeriod( + TimeUnit.DAYS + ) if time_in_days != 0: - self.logger.error("Time period property conversion to days is not working as expected") + self.logger.error( + "Time period property conversion to days is not working as expected" + ) return FlowFileTransformResult("failure") - data_size_in_bytes = context.getProperty(self.DATA_SIZE_PROPERTY).asDataSize(DataUnit.B) + data_size_in_bytes = context.getProperty(self.DATA_SIZE_PROPERTY).asDataSize( + DataUnit.B + ) if data_size_in_bytes != 104857600.0: - self.logger.error("Data size property conversion to bytes is not working as expected") + self.logger.error( + "Data size property conversion to bytes is not working as expected" + ) return FlowFileTransformResult("failure") - data_size_in_kilobytes = context.getProperty(self.DATA_SIZE_PROPERTY).asDataSize(DataUnit.KB) + data_size_in_kilobytes = context.getProperty( + self.DATA_SIZE_PROPERTY + ).asDataSize(DataUnit.KB) if data_size_in_kilobytes != 102400.0: - self.logger.error("Data size property conversion to kilobytes is not working as expected") + self.logger.error( + "Data size property conversion to kilobytes is not working as expected" + ) return FlowFileTransformResult("failure") - data_size_in_megabytes = context.getProperty(self.DATA_SIZE_PROPERTY).asDataSize(DataUnit.MB) + data_size_in_megabytes = context.getProperty( + self.DATA_SIZE_PROPERTY + ).asDataSize(DataUnit.MB) if data_size_in_megabytes != 100.0: - self.logger.error("Data size property conversion to megabytes is not working as expected") + self.logger.error( + "Data size property conversion to megabytes is not working as expected" + ) return FlowFileTransformResult("failure") - data_size_in_gigabytes = context.getProperty(self.DATA_SIZE_PROPERTY).asDataSize(DataUnit.GB) + data_size_in_gigabytes = context.getProperty( + self.DATA_SIZE_PROPERTY + ).asDataSize(DataUnit.GB) if data_size_in_gigabytes != 0.09765625: - self.logger.error("Data size property conversion to gigabytes is not working as expected") + self.logger.error( + "Data size property conversion to gigabytes is not working as expected" + ) return FlowFileTransformResult("failure") - ssl_context = context.getProperty(self.SSL_CONTEXT_PROPERTY).asControllerService() + ssl_context = context.getProperty( + self.SSL_CONTEXT_PROPERTY + ).asControllerService() cert = ssl_context.getCertificateFile() if cert != "/tmp/resources/minifi_client.crt": self.logger.error("SSL Context Service property is not working as expected") diff --git a/extensions/python/tests/features/resources/TestStateManager.py b/extensions/python/tests/features/resources/TestStateManager.py index b5d657176d..b092064a11 100644 --- a/extensions/python/tests/features/resources/TestStateManager.py +++ b/extensions/python/tests/features/resources/TestStateManager.py @@ -13,18 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.componentstate import Scope from nifiapi.flowfilesource import FlowFileSource, FlowFileSourceResult class TestStateManager(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileSource"] class ProcessorDetails: - version = '0.0.1-SNAPSHOT' - description = '''A Python source processor that uses StateManager.''' - tags = ['text', 'test', 'python', 'source'] + version = "0.0.1-SNAPSHOT" + description = """A Python source processor that uses StateManager.""" + tags: ClassVar[list] = ["text", "test", "python", "source"] def __init__(self, **kwargs): pass @@ -37,12 +39,16 @@ def create(self, context): state = state_manager.getState(Scope.CLUSTER) old_value = state.get("state_key") if not old_value: - new_state = {'state_key': '1'} + new_state = {"state_key": "1"} state_manager.setState(new_state, Scope.CLUSTER) - elif old_value == '1': - new_state = {'state_key': '2'} + elif old_value == "1": + new_state = {"state_key": "2"} state_manager.replace(state, new_state, Scope.CLUSTER) else: state_manager.clear(Scope.CLUSTER) - return FlowFileSourceResult(relationship='success', attributes=state.toMap(), contents='Output FlowFile Contents') + return FlowFileSourceResult( + relationship="success", + attributes=state.toMap(), + contents="Output FlowFile Contents", + ) diff --git a/extensions/python/tests/features/resources/multiplierutils.py b/extensions/python/tests/features/resources/multiplierutils.py index 7b0bafa67a..005cbd252d 100644 --- a/extensions/python/tests/features/resources/multiplierutils.py +++ b/extensions/python/tests/features/resources/multiplierutils.py @@ -13,5 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. + def double(number): return number * 2 diff --git a/extensions/python/tests/features/resources/subtractutils.py b/extensions/python/tests/features/resources/subtractutils.py index 854a1d3c06..fd7bc076c2 100644 --- a/extensions/python/tests/features/resources/subtractutils.py +++ b/extensions/python/tests/features/resources/subtractutils.py @@ -13,5 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. + def minus_ten(number): return number - 10 diff --git a/extensions/python/tests/features/steps/steps.py b/extensions/python/tests/features/steps/steps.py index 6b0dc4a479..afd2a26885 100644 --- a/extensions/python/tests/features/steps/steps.py +++ b/extensions/python/tests/features/steps/steps.py @@ -13,29 +13,44 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import docker -from behave import given -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext +from behave import given from minifi_behave.containers.docker_image_builder import DockerImageBuilder +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) + +import docker class PythonWithDependenciesOptions: REQUIREMENTS_FILE = "apacheminificpp-python-requirements-file:latest" - SYSTEM_INSTALLED_PACKAGES = "apacheminificpp-python-system-installed-packages:latest" + SYSTEM_INSTALLED_PACKAGES = ( + "apacheminificpp-python-system-installed-packages:latest" + ) INLINE_DEFINED_PACKAGES = "apacheminificpp-python-inline-defined-packages:latest" -def build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, python_option): - minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] if 'MINIFI_TAG_PREFIX' in os.environ else "" +def build_minifi_cpp_image_with_nifi_python_processors_using_dependencies( + context, python_option +): + minifi_tag_prefix = os.environ.get("MINIFI_TAG_PREFIX", "") client: docker.DockerClient = docker.from_env() - is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(client.images.get(context.minifi_container_image).history()) - minifi_python_dir_path = '/var/lib/nifi-minifi-cpp/minifi-python' if is_fhs else '/opt/minifi/minifi-current/minifi-python' - minifi_python_venv_parent = '/var/lib/nifi-minifi-cpp' if is_fhs else '/opt/minifi/minifi-current' + is_fhs = "MINIFI_INSTALLATION_TYPE=FHS" in str( + client.images.get(context.minifi_container_image).history() + ) + minifi_python_dir_path = ( + "/var/lib/nifi-minifi-cpp/minifi-python" + if is_fhs + else "/opt/minifi/minifi-current/minifi-python" + ) + minifi_python_venv_parent = ( + "/var/lib/nifi-minifi-cpp" if is_fhs else "/opt/minifi/minifi-current" + ) parse_document_url = "https://raw.githubusercontent.com/apache/nifi-python-extensions/refs/heads/main/src/extensions/chunking/ParseDocument.py" chunk_document_url = "https://raw.githubusercontent.com/apache/nifi-python-extensions/refs/heads/main/src/extensions/chunking/ChunkDocument.py" pip3_install_command = "" @@ -45,29 +60,39 @@ def build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(contex # /class ProcessorDetails:/,/^$/: Do the following between 'class ProcessorDetails:' and the first empty line (so we don't modify other PropertyDescriptor blocks below) # /^\s*dependencies\s*=/,/\]\s*$/: Do the following between 'dependencies =' at the start of a line, and ']' at the end of a line # d: Delete line - parse_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir}/nifi_python_processors/ParseDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) - chunk_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir}/nifi_python_processors/ChunkDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) + parse_document_sed_cmd = f'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir_path}/nifi_python_processors/ParseDocument.py && \\' + chunk_document_sed_cmd = f'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir_path}/nifi_python_processors/ChunkDocument.py && \\' if python_option == PythonWithDependenciesOptions.SYSTEM_INSTALLED_PACKAGES: - if not minifi_tag_prefix or "bookworm" in minifi_tag_prefix or "noble" in minifi_tag_prefix or "trixie" in minifi_tag_prefix: - additional_cmd = "RUN pip3 install --break-system-packages 'langchain<=0.17.0'" + if ( + not minifi_tag_prefix + or "bookworm" in minifi_tag_prefix + or "noble" in minifi_tag_prefix + or "trixie" in minifi_tag_prefix + ): + additional_cmd = ( + "RUN pip3 install --break-system-packages 'langchain<=0.17.0'" + ) else: additional_cmd = "RUN pip3 install 'langchain<=0.17.0'" elif python_option == PythonWithDependenciesOptions.REQUIREMENTS_FILE: - requirements_install_command = "echo 'langchain<=0.17.0' > {minifi_python_dir}/nifi_python_processors/requirements.txt && \\".format(minifi_python_dir=minifi_python_dir_path) + requirements_install_command = f"echo 'langchain<=0.17.0' > {minifi_python_dir_path}/nifi_python_processors/requirements.txt && \\" elif python_option == PythonWithDependenciesOptions.INLINE_DEFINED_PACKAGES: - parse_document_sed_cmd = parse_document_sed_cmd[:-2] + ' sed -i "s/langchain==[0-9.]\\+/langchain<=0.17.0/" {minifi_python_dir}/nifi_python_processors/ParseDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) - chunk_document_sed_cmd = 'sed -i "s/\\[\\\'langchain\\\'\\]/\\[\\\'langchain<=0.17.0\\\'\\]/" {minifi_python_dir}/nifi_python_processors/ChunkDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) + parse_document_sed_cmd = ( + parse_document_sed_cmd[:-2] + + f' sed -i "s/langchain==[0-9.]\\+/langchain<=0.17.0/" {minifi_python_dir_path}/nifi_python_processors/ParseDocument.py && \\' + ) + chunk_document_sed_cmd = f"sed -i \"s/\\[\\'langchain\\'\\]/\\[\\'langchain<=0.17.0\\'\\]/\" {minifi_python_dir_path}/nifi_python_processors/ChunkDocument.py && \\" if not minifi_tag_prefix: pip3_install_command = "RUN apk --update --no-cache add py3-pip" - dockerfile = """\ -FROM {base_image} + dockerfile = f"""\ +FROM {context.minifi_container_image} USER root {pip3_install_command} {additional_cmd} USER minificpp -RUN wget {parse_document_url} --directory-prefix={minifi_python_dir}/nifi_python_processors && \\ - wget {chunk_document_url} --directory-prefix={minifi_python_dir}/nifi_python_processors && \\ - echo 'langchain<=0.17.0' > {minifi_python_dir}/nifi_python_processors/requirements.txt && \\ +RUN wget {parse_document_url} --directory-prefix={minifi_python_dir_path}/nifi_python_processors && \\ + wget {chunk_document_url} --directory-prefix={minifi_python_dir_path}/nifi_python_processors && \\ + echo 'langchain<=0.17.0' > {minifi_python_dir_path}/nifi_python_processors/requirements.txt && \\ {requirements_install_command} {parse_document_sed_cmd} {chunk_document_sed_cmd} @@ -75,16 +100,7 @@ def build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(contex python3 -m venv {minifi_python_venv_parent}/venv-with-langchain && \\ . {minifi_python_venv_parent}/venv-with-langchain/bin/activate && python3 -m pip install --no-cache-dir "langchain<=0.17.0" && \\ deactivate - """.format(base_image=context.minifi_container_image, - pip3_install_command=pip3_install_command, - parse_document_url=parse_document_url, - chunk_document_url=chunk_document_url, - additional_cmd=additional_cmd, - requirements_install_command=requirements_install_command, - parse_document_sed_cmd=parse_document_sed_cmd, - chunk_document_sed_cmd=chunk_document_sed_cmd, - minifi_python_dir=minifi_python_dir_path, - minifi_python_venv_parent=minifi_python_venv_parent) + """ builder = DockerImageBuilder( image_tag=python_option, dockerfile_content=dockerfile, @@ -101,22 +117,52 @@ def add_example_minifi_python_processors(context: MinifiTestContext): @given("python with langchain is installed on the MiNiFi agent {install_mode}") def install_python_with_langchain(context, install_mode): client: docker.DockerClient = docker.from_env() - is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(client.images.get(context.minifi_container_image).history()) - minifi_python_venv_parent = '/var/lib/nifi-minifi-cpp' if is_fhs else '/opt/minifi/minifi-current' + is_fhs = "MINIFI_INSTALLATION_TYPE=FHS" in str( + client.images.get(context.minifi_container_image).history() + ) + minifi_python_venv_parent = ( + "/var/lib/nifi-minifi-cpp" if is_fhs else "/opt/minifi/minifi-current" + ) if install_mode == "with required python packages": - build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.SYSTEM_INSTALLED_PACKAGES) - context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "false") + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies( + context, PythonWithDependenciesOptions.SYSTEM_INSTALLED_PACKAGES + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.install.packages.automatically", "false" + ) elif install_mode == "with a pre-created virtualenv": - build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.REQUIREMENTS_FILE) - context.get_or_create_default_minifi_container().set_property("nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv") - context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "true") - elif install_mode == "with a pre-created virtualenv containing the required python packages": - build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.REQUIREMENTS_FILE) - context.get_or_create_default_minifi_container().set_property("nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv-with-langchain") - context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "false") + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies( + context, PythonWithDependenciesOptions.REQUIREMENTS_FILE + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.install.packages.automatically", "true" + ) + elif ( + install_mode + == "with a pre-created virtualenv containing the required python packages" + ): + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies( + context, PythonWithDependenciesOptions.REQUIREMENTS_FILE + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.virtualenv.directory", + f"{minifi_python_venv_parent}/venv-with-langchain", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.install.packages.automatically", "false" + ) elif install_mode == "using inline defined Python dependencies to install packages": - build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.INLINE_DEFINED_PACKAGES) - context.get_or_create_default_minifi_container().set_property("nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv") - context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "true") + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies( + context, PythonWithDependenciesOptions.INLINE_DEFINED_PACKAGES + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.install.packages.automatically", "true" + ) else: - raise Exception("Unknown python install mode.") + raise RuntimeError("Unknown python install mode.") diff --git a/extensions/python/tests/test_python_scripts/foo_bar_processor.py b/extensions/python/tests/test_python_scripts/foo_bar_processor.py index 199a5ce898..1d29a20b61 100644 --- a/extensions/python/tests/test_python_scripts/foo_bar_processor.py +++ b/extensions/python/tests/test_python_scripts/foo_bar_processor.py @@ -15,12 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import foo import bar +import foo def describe(processor): - processor.setDescription("Processor used for testing in ExecutePythonProcessorTests.cpp") + processor.setDescription( + "Processor used for testing in ExecutePythonProcessorTests.cpp" + ) def onTrigger(context, session): diff --git a/extensions/python/tests/test_python_scripts/non_transferring_processor.py b/extensions/python/tests/test_python_scripts/non_transferring_processor.py index afb83d4cea..bb6c06f0e6 100644 --- a/extensions/python/tests/test_python_scripts/non_transferring_processor.py +++ b/extensions/python/tests/test_python_scripts/non_transferring_processor.py @@ -18,12 +18,14 @@ def describe(processor): - processor.setDescription("Processor used for testing in ExecutePythonProcessorTests.cpp") + processor.setDescription( + "Processor used for testing in ExecutePythonProcessorTests.cpp" + ) def onTrigger(context, session): flow_file = session.get() - log.info('Vrrm, vrrrm, processor is running, vrrrm!!') + log.info("Vrrm, vrrrm, processor is running, vrrrm!!") if flow_file is not None: - log.info('created flow file: %s' % flow_file.getAttribute('filename')) + log.info("created flow file: {}".format(flow_file.getAttribute("filename"))) diff --git a/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_failure.py b/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_failure.py index cad1d8e02a..7205a624fe 100644 --- a/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_failure.py +++ b/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_failure.py @@ -18,13 +18,15 @@ def describe(processor): - processor.setDescription("Processor used for testing in ExecutePythonProcessorTests.cpp") + processor.setDescription( + "Processor used for testing in ExecutePythonProcessorTests.cpp" + ) def onTrigger(context, session): flow_file = session.get() - log.info('Vrrm, vrrrm, processor is running, vrrrm!!') + log.info("Vrrm, vrrrm, processor is running, vrrrm!!") if flow_file is not None: - log.info('created flow file: %s' % flow_file.getAttribute('filename')) + log.info("created flow file: {}".format(flow_file.getAttribute("filename"))) session.transfer(flow_file, REL_FAILURE) diff --git a/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_success.py b/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_success.py index 5ef7f5fbbf..815134458a 100644 --- a/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_success.py +++ b/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_success.py @@ -18,13 +18,15 @@ def describe(processor): - processor.setDescription("Processor used for testing in ExecutePythonProcessorTests.cpp") + processor.setDescription( + "Processor used for testing in ExecutePythonProcessorTests.cpp" + ) def onTrigger(context, session): flow_file = session.get() - log.info('Vrrm, vrrrm, processor is running, vrrrm!!') + log.info("Vrrm, vrrrm, processor is running, vrrrm!!") if flow_file is not None: - log.info('created flow file: %s' % flow_file.getAttribute('filename')) + log.info("created flow file: {}".format(flow_file.getAttribute("filename"))) session.transfer(flow_file, REL_SUCCESS) diff --git a/extensions/python/tests/test_python_scripts/stateful_processor.py b/extensions/python/tests/test_python_scripts/stateful_processor.py index 12aaecca9d..a558f2e940 100644 --- a/extensions/python/tests/test_python_scripts/stateful_processor.py +++ b/extensions/python/tests/test_python_scripts/stateful_processor.py @@ -15,28 +15,29 @@ def describe(processor): - processor.setDescription("Processor used for testing in ExecutePythonProcessorTests.cpp") + processor.setDescription( + "Processor used for testing in ExecutePythonProcessorTests.cpp" + ) state = 0 -class WriteCallback(object): +class WriteCallback: def process(self, output_stream): global state - new_content = str(state).encode('utf-8') + new_content = str(state).encode("utf-8") output_stream.write(new_content) state = state + 1 return len(new_content) def onTrigger(context, session): - global state - log.info('Vrrm, vrrrm, processor is running, vrrrm!!') + log.info("Vrrm, vrrrm, processor is running, vrrrm!!") # flow_file = session.get() flow_file = session.create() flow_file.setAttribute("filename", str(state)) - log.info('created flow file: %s' % flow_file.getAttribute('filename')) + log.info("created flow file: {}".format(flow_file.getAttribute("filename"))) if flow_file is not None: session.write(flow_file, WriteCallback()) diff --git a/extensions/splunk/tests/features/containers/splunk_container.py b/extensions/splunk/tests/features/containers/splunk_container.py index f1cb7b2f5e..1ae16f6ffc 100644 --- a/extensions/splunk/tests/features/containers/splunk_container.py +++ b/extensions/splunk/tests/features/containers/splunk_container.py @@ -16,22 +16,28 @@ import json from minifi_behave.containers.container_linux import LinuxContainer -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.helpers import wait_for_condition, retry_check from minifi_behave.containers.file import File -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key +from minifi_behave.core.helpers import retry_check, wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert class SplunkContainer(LinuxContainer): IMAGE = "splunk/splunk:9.2.1-patch2" def __init__(self, test_context: MinifiTestContext): - super().__init__(SplunkContainer.IMAGE, f"splunk-{test_context.scenario_id}", test_context.network) + super().__init__( + SplunkContainer.IMAGE, + f"splunk-{test_context.scenario_id}", + test_context.network, + ) self.user = None - self.environment = ["SPLUNK_LICENSE_URI=Free", - "SPLUNK_START_ARGS=--accept-license", - "SPLUNK_PASSWORD=splunkadmin"] + self.environment = [ + "SPLUNK_LICENSE_URI=Free", + "SPLUNK_START_ARGS=--accept-license", + "SPLUNK_PASSWORD=splunkadmin", + ] splunk_config_content = """ splunk: @@ -41,34 +47,82 @@ def __init__(self, test_context: MinifiTestContext): port: 8088 token: 176fae97-f59d-4f08-939a-aa6a543f2485 """ - self.files.append(File("/tmp/defaults/default.yml", splunk_config_content, mode="rw", permissions=0o644)) + self.files.append( + File( + "/tmp/defaults/default.yml", + splunk_config_content, + mode="rw", + permissions=0o644, + ) + ) - splunk_cert, splunk_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + splunk_cert, splunk_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) splunk_cert_content = dump_cert(splunk_cert) splunk_key_content = dump_key(splunk_key) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/opt/splunk/etc/auth/splunk_cert.pem", splunk_cert_content.decode() + splunk_key_content.decode() + root_ca_content.decode(), permissions=0o644)) - self.files.append(File("/opt/splunk/etc/auth/root_ca.pem", root_ca_content.decode(), permissions=0o644)) + self.files.append( + File( + "/opt/splunk/etc/auth/splunk_cert.pem", + splunk_cert_content.decode() + + splunk_key_content.decode() + + root_ca_content.decode(), + permissions=0o644, + ) + ) + self.files.append( + File( + "/opt/splunk/etc/auth/root_ca.pem", + root_ca_content.decode(), + permissions=0o644, + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) - finished_str = "Ansible playbook complete, will begin streaming splunkd_stderr.log" + finished_str = ( + "Ansible playbook complete, will begin streaming splunkd_stderr.log" + ) return wait_for_condition( condition=lambda: finished_str in self.get_logs(), timeout_seconds=300, bail_condition=lambda: False, - context=context) + context=context, + ) @retry_check() def check_splunk_event(self, query: str) -> bool: - (code, output) = self.exec_run(["sudo", "/opt/splunk/bin/splunk", "search", query, "-auth", "admin:splunkadmin"]) + (code, output) = self.exec_run( + [ + "sudo", + "/opt/splunk/bin/splunk", + "search", + query, + "-auth", + "admin:splunkadmin", + ] + ) if code != 0: return False return query in output.decode("utf-8") @retry_check() - def check_splunk_event_with_attributes(self, query: str, attributes: dict[str, str]) -> bool: - (code, output) = self.exec_run(["sudo", "/opt/splunk/bin/splunk", "search", query, "-output", "json", "-auth", "admin:splunkadmin"]) + def check_splunk_event_with_attributes( + self, query: str, attributes: dict[str, str] + ) -> bool: + (code, output) = self.exec_run( + [ + "sudo", + "/opt/splunk/bin/splunk", + "search", + query, + "-output", + "json", + "-auth", + "admin:splunkadmin", + ] + ) if code != 0: return False result_lines = output.splitlines() @@ -79,39 +133,68 @@ def check_splunk_event_with_attributes(self, query: str, attributes: dict[str, s continue if "result" not in result_line_json: continue - if "host" in attributes: - if result_line_json["result"]["host"] != attributes["host"]: - continue - if "source" in attributes: - if result_line_json["result"]["source"] != attributes["source"]: - continue - if "sourcetype" in attributes: - if result_line_json["result"]["sourcetype"] != attributes["sourcetype"]: - continue - if "index" in attributes: - if result_line_json["result"]["index"] != attributes["index"]: - continue + if ( + "host" in attributes + and result_line_json["result"]["host"] != attributes["host"] + ): + continue + if ( + "source" in attributes + and result_line_json["result"]["source"] != attributes["source"] + ): + continue + if ( + "sourcetype" in attributes + and result_line_json["result"]["sourcetype"] != attributes["sourcetype"] + ): + continue + if ( + "index" in attributes + and result_line_json["result"]["index"] != attributes["index"] + ): + continue return True return False def enable_splunk_hec_indexer(self, hec_name: str): - (code, _) = self.exec_run(["sudo", - "/opt/splunk/bin/splunk", "http-event-collector", - "update", hec_name, - "-uri", "https://localhost:8089", - "-use-ack", "1", - "-disabled", "0", - "-auth", "admin:splunkadmin"]) + (code, _) = self.exec_run( + [ + "sudo", + "/opt/splunk/bin/splunk", + "http-event-collector", + "update", + hec_name, + "-uri", + "https://localhost:8089", + "-use-ack", + "1", + "-disabled", + "0", + "-auth", + "admin:splunkadmin", + ] + ) return code == 0 def enable_splunk_hec_ssl(self): - (code, _) = self.exec_run(["sudo", - "/opt/splunk/bin/splunk", "http-event-collector", - "update", - "-uri", "https://localhost:8089", - "-enable-ssl", "1", - "-server-cert", "/opt/splunk/etc/auth/splunk_cert.pem", - "-ca-cert-file", "/opt/splunk/etc/auth/root_ca.pem", - "-require-client-cert", "1", - "-auth", "admin:splunkadmin"]) + (code, _) = self.exec_run( + [ + "sudo", + "/opt/splunk/bin/splunk", + "http-event-collector", + "update", + "-uri", + "https://localhost:8089", + "-enable-ssl", + "1", + "-server-cert", + "/opt/splunk/etc/auth/splunk_cert.pem", + "-ca-cert-file", + "/opt/splunk/etc/auth/root_ca.pem", + "-require-client-cert", + "1", + "-auth", + "admin:splunkadmin", + ] + ) return code == 0 diff --git a/extensions/splunk/tests/features/environment.py b/extensions/splunk/tests/features/environment.py index 4598a1f0e6..52694dc0b7 100644 --- a/extensions/splunk/tests/features/environment.py +++ b/extensions/splunk/tests/features/environment.py @@ -14,11 +14,11 @@ # limitations under the License. import platform -import docker from containers.splunk_container import SplunkContainer -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_feature(context, feature): diff --git a/extensions/splunk/tests/features/steps/steps.py b/extensions/splunk/tests/features/steps/steps.py index c89a329556..0267b7869b 100644 --- a/extensions/splunk/tests/features/steps/steps.py +++ b/extensions/splunk/tests/features/steps/steps.py @@ -13,34 +13,49 @@ # See the License for the specific language governing permissions and # limitations under the License. from behave import step, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from containers.splunk_container import SplunkContainer from minifi_behave.core.helpers import log_due_to_failure from minifi_behave.core.minifi_test_context import MinifiTestContext -from containers.splunk_container import SplunkContainer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @step("a Splunk HEC is set up and running") def setup_splunk_hec(context: MinifiTestContext): context.containers["splunk"] = SplunkContainer(context) assert context.containers["splunk"].deploy(context) - assert context.containers["splunk"].enable_splunk_hec_indexer('splunk_hec_token') or context.containers["splunk"].log_app_output() + assert ( + context.containers["splunk"].enable_splunk_hec_indexer("splunk_hec_token") + or context.containers["splunk"].log_app_output() + ) -@then('an event is registered in Splunk HEC with the content \"{content}\"') +@then('an event is registered in Splunk HEC with the content "{content}"') def verify_splunk_event_content(context, content): - assert context.containers["splunk"].check_splunk_event(content) or log_due_to_failure(context) + assert context.containers["splunk"].check_splunk_event( + content + ) or log_due_to_failure(context) -@then('an event is registered in Splunk HEC with the content \"{content}\" with \"{source}\" set as source and \"{source_type}\" set as sourcetype and \"{host}\" set as host') +@then( + 'an event is registered in Splunk HEC with the content "{content}" with "{source}" set as source and "{source_type}" set as sourcetype and "{host}" set as host' +) def verify_splunk_event_with_attributes(context, content, source, source_type, host): attr = {"source": source, "sourcetype": source_type, "host": host} - assert context.containers["splunk"].check_splunk_event_with_attributes(content, attr) or log_due_to_failure(context) + assert context.containers["splunk"].check_splunk_event_with_attributes( + content, attr + ) or log_due_to_failure(context) -@step("SSL is enabled for the Splunk HEC and the SSL context service is set up for PutSplunkHTTP and QuerySplunkIndexingStatus") +@step( + "SSL is enabled for the Splunk HEC and the SSL context service is set up for PutSplunkHTTP and QuerySplunkIndexingStatus" +) def enable_splunk_hec_ssl(context): - assert context.containers["splunk"].enable_splunk_hec_ssl() or context.containers["splunk"].log_app_output() + assert ( + context.containers["splunk"].enable_splunk_hec_ssl() + or context.containers["splunk"].log_app_output() + ) diff --git a/extensions/sql/tests/features/containers/postgress_server_container.py b/extensions/sql/tests/features/containers/postgress_server_container.py index 9a9ec6f6ff..f0d6e3656a 100644 --- a/extensions/sql/tests/features/containers/postgress_server_container.py +++ b/extensions/sql/tests/features/containers/postgress_server_container.py @@ -17,18 +17,22 @@ import logging from textwrap import dedent + from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.docker_image_builder import DockerImageBuilder from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class PostgresContainer(LinuxContainer): IMAGE = "postgres:17.4" def __init__(self, context): - dockerfile = dedent("""\ - FROM {base_image} + dockerfile = dedent( + f"""\ + FROM {PostgresContainer.IMAGE} RUN mkdir -p /docker-entrypoint-initdb.d RUN echo "#!/bin/bash" > /docker-entrypoint-initdb.d/init-user-db.sh && \ echo "set -e" >> /docker-entrypoint-initdb.d/init-user-db.sh && \ @@ -41,26 +45,35 @@ def __init__(self, context): echo " INSERT INTO test_table2 (int_col, \\"tExT_Col\\") VALUES (5, 'ApPlE');" >> /docker-entrypoint-initdb.d/init-user-db.sh && \ echo " INSERT INTO test_table2 (int_col, \\"tExT_Col\\") VALUES (6, 'BaNaNa');" >> /docker-entrypoint-initdb.d/init-user-db.sh && \ echo "EOSQL" >> /docker-entrypoint-initdb.d/init-user-db.sh - """.format(base_image=PostgresContainer.IMAGE)) + """ + ) builder = DockerImageBuilder( - image_tag="minifi-postgres-server:latest", - dockerfile_content=dockerfile + image_tag="minifi-postgres-server:latest", dockerfile_content=dockerfile ) builder.build() - super(PostgresContainer, self).__init__("minifi-postgres-server:latest", f"postgres-server-{context.scenario_id}", context.network) + super().__init__( + "minifi-postgres-server:latest", + f"postgres-server-{context.scenario_id}", + context.network, + ) self.environment = ["POSTGRES_PASSWORD=password"] def deploy(self, context: MinifiTestContext | None) -> bool: - super(PostgresContainer, self).deploy(context) + super().deploy(context) finished_str = "database system is ready to accept connections" return wait_for_condition( condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context) + context=context, + ) def check_query_results(self, query: str, number_of_rows: int) -> bool: (code, output) = self.exec_run(["psql", "-U", "postgres", "-c", query]) - logging.debug(f"check_query_results: {query} -> {output}") - return code == 0 and (str(number_of_rows) + (" row" if number_of_rows == 1 else " rows")) in output + logger.debug(f"check_query_results: {query} -> {output}") + return ( + code == 0 + and (str(number_of_rows) + (" row" if number_of_rows == 1 else " rows")) + in output + ) diff --git a/extensions/sql/tests/features/environment.py b/extensions/sql/tests/features/environment.py index 5b5cf8151d..64ba734d55 100644 --- a/extensions/sql/tests/features/environment.py +++ b/extensions/sql/tests/features/environment.py @@ -16,35 +16,44 @@ # import os -import docker from textwrap import dedent from containers.postgress_server_container import PostgresContainer from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario -from minifi_behave.core.hooks import get_minifi_container_image +from minifi_behave.core.hooks import ( + common_after_scenario, + common_before_scenario, + get_minifi_container_image, +) + +import docker # These hooks are executed by behave before and after each scenario # The common_before_scenario and common_after_scenario must be called for proper setup and tear down def before_all(context): - minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] if 'MINIFI_TAG_PREFIX' in os.environ else '' + minifi_tag_prefix = os.environ.get("MINIFI_TAG_PREFIX", "") if "rocky" in minifi_tag_prefix: install_sql_cmd = "dnf -y install postgresql-odbc" so_location = "psqlodbca.so" - elif "bullseye" in minifi_tag_prefix or "bookworm" in minifi_tag_prefix or "trixie" in minifi_tag_prefix: - install_sql_cmd = "apt -y install odbc-postgresql" - so_location = "/usr/lib/$(gcc -dumpmachine)/odbc/psqlodbca.so" - elif "jammy" in minifi_tag_prefix or "noble" in minifi_tag_prefix: + elif ( + ( + "bullseye" in minifi_tag_prefix + or "bookworm" in minifi_tag_prefix + or "trixie" in minifi_tag_prefix + ) + or "jammy" in minifi_tag_prefix + or "noble" in minifi_tag_prefix + ): install_sql_cmd = "apt -y install odbc-postgresql" so_location = "/usr/lib/$(gcc -dumpmachine)/odbc/psqlodbca.so" else: install_sql_cmd = "apk --update --no-cache add psqlodbc" so_location = "psqlodbca.so" - dockerfile = dedent("""\ - FROM {base_image} + dockerfile = dedent( + f"""\ + FROM {get_minifi_container_image()} USER root RUN {install_sql_cmd} RUN echo "[PostgreSQL ANSI]" > /odbcinst.ini.template && \ @@ -73,11 +82,10 @@ def before_all(context): echo "Password = password" >> /etc/odbc.ini && \ echo "Database = postgres" >> /etc/odbc.ini USER minificpp - """.format(base_image=get_minifi_container_image(), - install_sql_cmd=install_sql_cmd, so_location=so_location)) + """ + ) builder = DockerImageBuilder( - image_tag="apacheminificpp-sql:behave", - dockerfile_content=dockerfile + image_tag="apacheminificpp-sql:behave", dockerfile_content=dockerfile ) builder.build() docker.from_env().images.pull(PostgresContainer.IMAGE) diff --git a/extensions/sql/tests/features/steps/steps.py b/extensions/sql/tests/features/steps/steps.py index 92b95d22b0..11f6cec7fa 100644 --- a/extensions/sql/tests/features/steps/steps.py +++ b/extensions/sql/tests/features/steps/steps.py @@ -16,25 +16,37 @@ # import humanfriendly -from behave import step, given, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from behave import given, step, then +from containers.postgress_server_container import PostgresContainer from minifi_behave.core.helpers import wait_for_condition -from minifi_behave.minifi.controller_service import ControllerService from minifi_behave.core.minifi_test_context import MinifiTestContext -from containers.postgress_server_container import PostgresContainer +from minifi_behave.minifi.controller_service import ControllerService +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) -@given("an ODBCService is setup up for {processor_name} with the name \"{service_name}\"") -def setup_odbc_service_for_processor(context: MinifiTestContext, processor_name: str, service_name: str): +@given('an ODBCService is setup up for {processor_name} with the name "{service_name}"') +def setup_odbc_service_for_processor( + context: MinifiTestContext, processor_name: str, service_name: str +): odb_service = ControllerService(class_name="ODBCService", service_name=service_name) postgres_server_hostname = f"postgres-server-{context.scenario_id}" - odb_service.add_property("Connection String", f"Driver={{PostgreSQL ANSI}};Server={postgres_server_hostname};Port=5432;Database=postgres;Uid=postgres;Pwd=password;") - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(odb_service) - processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) + odb_service.add_property( + "Connection String", + f"Driver={{PostgreSQL ANSI}};Server={postgres_server_hostname};Port=5432;Database=postgres;Uid=postgres;Pwd=password;", + ) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + odb_service + ) + processor = ( + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name + ) + ) processor.add_property("DB Controller Service", "ODBCService") @@ -43,7 +55,9 @@ def setup_postgresql_server(context): context.containers["postgres-server"] = PostgresContainer(context) -@then('the query "{query}" returns {rows:d} rows in less than {timeout_str} on the PostgreSQL server') +@then( + 'the query "{query}" returns {rows:d} rows in less than {timeout_str} on the PostgreSQL server' +) def verify_postgresql_query_results(context, query: str, rows: int, timeout_str: str): timeout_seconds = humanfriendly.parse_timespan(timeout_str) postgres_container = context.containers["postgres-server"] @@ -52,4 +66,5 @@ def verify_postgresql_query_results(context, query: str, rows: int, timeout_str: condition=lambda: postgres_container.check_query_results(query, int(rows)), timeout_seconds=timeout_seconds, bail_condition=lambda: postgres_container.exited, - context=context) + context=context, + ) diff --git a/extensions/standard-processors/tests/features/containers/diag_slave_container.py b/extensions/standard-processors/tests/features/containers/diag_slave_container.py index 68f78ae64f..5d461a5465 100644 --- a/extensions/standard-processors/tests/features/containers/diag_slave_container.py +++ b/extensions/standard-processors/tests/features/containers/diag_slave_container.py @@ -23,6 +23,8 @@ from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class DiagSlave(LinuxContainer): def __init__(self, test_context: MinifiTestContext): @@ -33,12 +35,15 @@ def __init__(self, test_context: MinifiTestContext): """) builder = DockerImageBuilder( - image_tag="minifi-diag-slave-tcp:latest", - dockerfile_content=dockerfile + image_tag="minifi-diag-slave-tcp:latest", dockerfile_content=dockerfile ) builder.build() - super().__init__("minifi-diag-slave-tcp:latest", f"diag-slave-tcp-{test_context.scenario_id}", test_context.network) + super().__init__( + "minifi-diag-slave-tcp:latest", + f"diag-slave-tcp-{test_context.scenario_id}", + test_context.network, + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -47,10 +52,10 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context + context=context, ) def set_value_on_plc_with_modbus(self, modbus_cmd): (code, output) = self.exec_run(["modbus", "localhost", modbus_cmd]) - logging.info("Modbus command '%s' output: %s", modbus_cmd, output) + logger.info("Modbus command '%s' output: %s", modbus_cmd, output) return code == 0 diff --git a/extensions/standard-processors/tests/features/containers/syslog_container.py b/extensions/standard-processors/tests/features/containers/syslog_container.py index 8cbb117d14..d5d7086480 100644 --- a/extensions/standard-processors/tests/features/containers/syslog_container.py +++ b/extensions/standard-processors/tests/features/containers/syslog_container.py @@ -20,5 +20,7 @@ class SyslogContainer(LinuxContainer): def __init__(self, protocol, context): - super(SyslogContainer, self).__init__("ubuntu:24.04", f"syslog-{protocol}-{context.scenario_id}", context.network) + super().__init__( + "ubuntu:24.04", f"syslog-{protocol}-{context.scenario_id}", context.network + ) self.command = f'/bin/bash -c "echo Syslog {protocol} client started; while true; do logger --{protocol} -n minifi-primary-{context.scenario_id} -P 514 sample_log; sleep 1; done"' diff --git a/extensions/standard-processors/tests/features/containers/tcp_client_container.py b/extensions/standard-processors/tests/features/containers/tcp_client_container.py index 37b8cbb318..5734c76c3c 100644 --- a/extensions/standard-processors/tests/features/containers/tcp_client_container.py +++ b/extensions/standard-processors/tests/features/containers/tcp_client_container.py @@ -27,7 +27,12 @@ def __init__(self, test_context: MinifiTestContext): f"nc minifi-primary-{test_context.scenario_id} 10254; " "sleep 1; done'" ) - super().__init__("alpine:3.17.3", f"tcp-client-{test_context.scenario_id}", test_context.network, cmd) + super().__init__( + "alpine:3.17.3", + f"tcp-client-{test_context.scenario_id}", + test_context.network, + cmd, + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -36,5 +41,5 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context + context=context, ) diff --git a/extensions/standard-processors/tests/features/environment.py b/extensions/standard-processors/tests/features/environment.py index fb2b3b72c5..df9bd06171 100644 --- a/extensions/standard-processors/tests/features/environment.py +++ b/extensions/standard-processors/tests/features/environment.py @@ -15,9 +15,9 @@ import os import platform +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + import docker -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario # These hooks are executed by behave before and after each scenario # The common_before_scenario and common_after_scenario must be called for proper setup and tear down @@ -31,12 +31,12 @@ def before_feature(context, feature): def is_minifi_image_alpine_based(context): - if os.name == 'nt': + if os.name == "nt": return False client: docker.DockerClient = docker.from_env() container = client.containers.create( image=context.minifi_container_image, - command=['cat', '/etc/os-release'], + command=["cat", "/etc/os-release"], ) try: container.start() @@ -46,14 +46,15 @@ def is_minifi_image_alpine_based(context): container.remove(force=True) return False - return "alpine" in result.decode('utf-8').lower() + return "alpine" in result.decode("utf-8").lower() def before_scenario(context, scenario): common_before_scenario(context, scenario) - if "ALPINE_ONLY" in scenario.tags: - if not is_minifi_image_alpine_based(context): - scenario.skip("This scenario is only compatible with Alpine-based Minifi images") + if "ALPINE_ONLY" in scenario.tags and not is_minifi_image_alpine_based(context): + scenario.skip( + "This scenario is only compatible with Alpine-based Minifi images" + ) def after_scenario(context, scenario): diff --git a/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py b/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py index a645499f67..b2c5166f0a 100644 --- a/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py +++ b/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py @@ -15,32 +15,53 @@ # limitations under the License. # -import jks import os - -from cryptography.hazmat.primitives import serialization -from cryptography import x509 from pathlib import Path -from cryptography.hazmat.primitives.serialization import load_pem_private_key, pkcs12, BestAvailableEncryption +import jks +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import ( + BestAvailableEncryption, + load_pem_private_key, + pkcs12, +) from minifi_behave.containers.container_linux import LinuxContainer -from minifi_behave.core.helpers import wait_for_condition -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.ssl_utils import make_server_cert from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile -from minifi_behave.core.ssl_utils import dump_key, dump_cert +from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert class MinifiC2Server(LinuxContainer): def __init__(self, test_context: MinifiTestContext, ssl: bool = False): - super().__init__("apache/nifi-minifi-c2:1.27.0", f"minifi-c2-server-{test_context.scenario_id}", test_context.network) + super().__init__( + "apache/nifi-minifi-c2:1.27.0", + f"minifi-c2-server-{test_context.scenario_id}", + test_context.network, + ) if ssl: - c2_cert, c2_key = make_server_cert(f"minifi-c2-server-{test_context.scenario_id}", test_context.root_ca_cert, test_context.root_ca_key) - pke = jks.PrivateKeyEntry.new('c2-server-cert', [dump_cert(c2_cert, encoding_type=serialization.Encoding.DER)], dump_key(c2_key, encoding_type=serialization.Encoding.DER), 'rsa_raw') - server_keystore = jks.KeyStore.new('jks', [pke]) - server_keystore_content = server_keystore.saves('abcdefgh') - self.files.append(File("/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-keystore.jks", server_keystore_content, permissions=0o644)) + c2_cert, c2_key = make_server_cert( + f"minifi-c2-server-{test_context.scenario_id}", + test_context.root_ca_cert, + test_context.root_ca_key, + ) + pke = jks.PrivateKeyEntry.new( + "c2-server-cert", + [dump_cert(c2_cert, encoding_type=serialization.Encoding.DER)], + dump_key(c2_key, encoding_type=serialization.Encoding.DER), + "rsa_raw", + ) + server_keystore = jks.KeyStore.new("jks", [pke]) + server_keystore_content = server_keystore.saves("abcdefgh") + self.files.append( + File( + "/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-keystore.jks", + server_keystore_content, + permissions=0o644, + ) + ) private_key_pem = dump_key(test_context.root_ca_key) private_key = load_pem_private_key(private_key_pem, password=None) @@ -51,21 +72,50 @@ def __init__(self, test_context: MinifiTestContext, ssl: bool = False): key=private_key, cert=certificate, cas=None, - encryption_algorithm=BestAvailableEncryption(b'abcdefgh') + encryption_algorithm=BestAvailableEncryption(b"abcdefgh"), + ) + self.files.append( + File( + "/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-truststore.p12", + pkcs12_data, + permissions=0o644, + ) ) - self.files.append(File("/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-truststore.p12", pkcs12_data, permissions=0o644)) - authorities_file_content = """ - CN=minifi-primary-{scenario_id}: + authorities_file_content = f""" + CN=minifi-primary-{test_context.scenario_id}: - CLASS_MINIFI_CPP - """.format(scenario_id=test_context.scenario_id) - self.files.append(File("/opt/minifi-c2/minifi-c2-current/conf/authorities.yaml", authorities_file_content, permissions=0o644)) + """ + self.files.append( + File( + "/opt/minifi-c2/minifi-c2-current/conf/authorities.yaml", + authorities_file_content, + permissions=0o644, + ) + ) - resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-c2-server" - self.host_files.append(HostFile("/opt/minifi-c2/minifi-c2-current/files/minifi-test-class/config.text.yml.v1", os.path.join(resource_dir, "config.yml"))) + resource_dir = ( + Path(__file__).resolve().parent / "resources" / "minifi-c2-server" + ) + self.host_files.append( + HostFile( + "/opt/minifi-c2/minifi-c2-current/files/minifi-test-class/config.text.yml.v1", + os.path.join(resource_dir, "config.yml"), + ) + ) if ssl: - self.host_files.append(HostFile("/opt/minifi-c2/minifi-c2-current/conf/authorizations.yaml", os.path.join(resource_dir, "authorizations.yaml"))) - self.host_files.append(HostFile("/opt/minifi-c2/minifi-c2-current/conf/c2.properties", os.path.join(resource_dir, "c2.properties"))) + self.host_files.append( + HostFile( + "/opt/minifi-c2/minifi-c2-current/conf/authorizations.yaml", + os.path.join(resource_dir, "authorizations.yaml"), + ) + ) + self.host_files.append( + HostFile( + "/opt/minifi-c2/minifi-c2-current/conf/c2.properties", + os.path.join(resource_dir, "c2.properties"), + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -74,5 +124,5 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context + context=context, ) diff --git a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py index f4dca56f89..a45ca2c104 100644 --- a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py +++ b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py @@ -15,77 +15,109 @@ # limitations under the License. # -from behave import given, when, then +from behave import given, then, when from minifi_behave.core.helpers import retry_check from minifi_behave.core.minifi_test_context import MinifiTestContext -@given('controller socket properties are set up') +@given("controller socket properties are set up") def setup_controller_socket_properties(context: MinifiTestContext): context.get_or_create_default_minifi_container().controller.set_controller_socket_properties() -@when('MiNiFi config is updated through MiNiFi controller') +@when("MiNiFi config is updated through MiNiFi controller") def update_minifi_config_via_controller(context: MinifiTestContext): context.get_or_create_default_minifi_container().controller.update_flow_config_through_controller() -@then('the updated config is persisted') +@then("the updated config is persisted") def verify_updated_config_is_persisted(context: MinifiTestContext): assert context.get_or_create_default_minifi_container().controller.updated_config_is_persisted() -@when('the {component} component is stopped through MiNiFi controller') +@when("the {component} component is stopped through MiNiFi controller") def stop_component_via_controller(context: MinifiTestContext, component: str): - context.get_or_create_default_minifi_container().controller.stop_component_through_controller(component) + context.get_or_create_default_minifi_container().controller.stop_component_through_controller( + component + ) -@when('the {component} component is started through MiNiFi controller') +@when("the {component} component is started through MiNiFi controller") def start_component_via_controller(context: MinifiTestContext, component: str): - context.get_or_create_default_minifi_container().controller.start_component_through_controller(component) + context.get_or_create_default_minifi_container().controller.start_component_through_controller( + component + ) -@then('the {component} component is not running') +@then("the {component} component is not running") def verify_component_not_running(context: MinifiTestContext, component: str): - assert not context.get_or_create_default_minifi_container().controller.is_component_running(component) + assert not context.get_or_create_default_minifi_container().controller.is_component_running( + component + ) -@then('the {component} component is running') +@then("the {component} component is running") def verify_component_is_running(context: MinifiTestContext, component: str): - assert context.get_or_create_default_minifi_container().controller.is_component_running(component) + assert context.get_or_create_default_minifi_container().controller.is_component_running( + component + ) -@then('connection \"{connection}\" can be seen through MiNiFi controller') +@then('connection "{connection}" can be seen through MiNiFi controller') def verify_connection_seen_via_controller(context: MinifiTestContext, connection: str): - assert context.get_or_create_default_minifi_container().controller.connection_found_through_controller(connection) + assert context.get_or_create_default_minifi_container().controller.connection_found_through_controller( + connection + ) -@then('{connection_count:d} connections can be seen full through MiNiFi controller') +@then("{connection_count:d} connections can be seen full through MiNiFi controller") def verify_full_connection_count(context: MinifiTestContext, connection_count: int): - assert context.get_or_create_default_minifi_container().controller.get_full_connection_count() == connection_count + assert ( + context.get_or_create_default_minifi_container().controller.get_full_connection_count() + == connection_count + ) @retry_check(max_tries=5, retry_interval_seconds=1) -def check_connection_size_through_controller(context: MinifiTestContext, connection: str, size: int, max_size: int) -> bool: - return context.get_or_create_default_minifi_container().controller.get_connection_size(connection) == (size, max_size) - - -@then('connection \"{connection}\" has {size:d} size and {max_size:d} max size through MiNiFi controller') -def verify_connection_size_via_controller(context: MinifiTestContext, connection: str, size: int, max_size: int): +def check_connection_size_through_controller( + context: MinifiTestContext, connection: str, size: int, max_size: int +) -> bool: + return ( + context.get_or_create_default_minifi_container().controller.get_connection_size( + connection + ) + == (size, max_size) + ) + + +@then( + 'connection "{connection}" has {size:d} size and {max_size:d} max size through MiNiFi controller' +) +def verify_connection_size_via_controller( + context: MinifiTestContext, connection: str, size: int, max_size: int +): assert check_connection_size_through_controller(context, connection, size, max_size) @retry_check(max_tries=10, retry_interval_seconds=1) -def manifest_can_be_retrieved_through_minifi_controller(context: MinifiTestContext) -> bool: - manifest = context.get_or_create_default_minifi_container().controller.get_manifest() - return '"agentManifest": {' in manifest and '"componentManifest": {' in manifest and '"agentType": "cpp"' in manifest - - -@then('manifest can be retrieved through MiNiFi controller') +def manifest_can_be_retrieved_through_minifi_controller( + context: MinifiTestContext, +) -> bool: + manifest = ( + context.get_or_create_default_minifi_container().controller.get_manifest() + ) + return ( + '"agentManifest": {' in manifest + and '"componentManifest": {' in manifest + and '"agentType": "cpp"' in manifest + ) + + +@then("manifest can be retrieved through MiNiFi controller") def verify_manifest_retrieval_via_controller(context: MinifiTestContext): assert manifest_can_be_retrieved_through_minifi_controller(context) -@then('debug bundle can be retrieved through MiNiFi controller') +@then("debug bundle can be retrieved through MiNiFi controller") def verify_debug_bundle_retrieval_via_controller(context: MinifiTestContext): assert context.get_or_create_default_minifi_container().controller.create_debug_bundle() diff --git a/extensions/standard-processors/tests/features/steps/steps.py b/extensions/standard-processors/tests/features/steps/steps.py index 689a6aef99..321301f93f 100644 --- a/extensions/standard-processors/tests/features/steps/steps.py +++ b/extensions/standard-processors/tests/features/steps/steps.py @@ -16,19 +16,19 @@ # import humanfriendly -from behave import step, then, given - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.helpers import wait_for_condition -from containers.syslog_container import SyslogContainer +from behave import given, step, then from containers.diag_slave_container import DiagSlave +from containers.syslog_container import SyslogContainer from containers.tcp_client_container import TcpClientContainer - from minifi_behave.containers.minifi_protocol import set_up_ssl_properties +from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) from minifi_c2_server_container import MinifiC2Server @@ -42,44 +42,87 @@ def setup_syslog_udp_client(context: MinifiTestContext): context.containers["syslog-udp"] = SyslogContainer("udp", context) -@step('there is an accessible PLC with modbus enabled') +@step("there is an accessible PLC with modbus enabled") def setup_plc_with_modbus(context: MinifiTestContext): modbus_container = context.containers["diag-slave-tcp"] = DiagSlave(context) assert modbus_container.deploy(context) -@step('PLC register has been set with {modbus_cmd} command') +@step("PLC register has been set with {modbus_cmd} command") def set_plc_register_with_command(context: MinifiTestContext, modbus_cmd: str): - assert context.containers["diag-slave-tcp"].set_value_on_plc_with_modbus(modbus_cmd) or context.containers["diag-slave-tcp"].log_app_output() + assert ( + context.containers["diag-slave-tcp"].set_value_on_plc_with_modbus(modbus_cmd) + or context.containers["diag-slave-tcp"].log_app_output() + ) -@step('a TCP client is set up to send a test TCP message to minifi') +@step("a TCP client is set up to send a test TCP message to minifi") def setup_tcp_client(context: MinifiTestContext): context.containers["tcp-client"] = TcpClientContainer(context) @given("C2 is enabled in MiNiFi") def enable_c2_in_minifi(context: MinifiTestContext): - context.get_or_create_default_minifi_container().set_property("nifi.c2.enable", "true") - context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url", f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/heartbeat") - context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url.ack", f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/acknowledge") - context.get_or_create_default_minifi_container().set_property("nifi.c2.flow.base.url", f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/") - context.get_or_create_default_minifi_container().set_property("nifi.c2.root.classes", "DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation") - context.get_or_create_default_minifi_container().set_property("nifi.c2.full.heartbeat", "false") - context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.class", "minifi-test-class") - context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.identifier", "minifi-test-id") + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.enable", "true" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.rest.url", + f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/heartbeat", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.rest.url.ack", + f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/acknowledge", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.flow.base.url", + f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.root.classes", + "DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.full.heartbeat", "false" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.agent.class", "minifi-test-class" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.agent.identifier", "minifi-test-id" + ) @given("ssl properties are set up for MiNiFi C2 server") def setup_ssl_properties_for_c2(context: MinifiTestContext): - context.get_or_create_default_minifi_container().set_property("nifi.c2.enable", "true") - context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url", f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/heartbeat") - context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url.ack", f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/acknowledge") - context.get_or_create_default_minifi_container().set_property("nifi.c2.flow.base.url", f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/") - context.get_or_create_default_minifi_container().set_property("nifi.c2.root.classes", "DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation") - context.get_or_create_default_minifi_container().set_property("nifi.c2.full.heartbeat", "false") - context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.class", "minifi-test-class") - context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.identifier", "minifi-test-id") + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.enable", "true" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.rest.url", + f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/heartbeat", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.rest.url.ack", + f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/acknowledge", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.flow.base.url", + f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.root.classes", + "DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.full.heartbeat", "false" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.agent.class", "minifi-test-class" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.agent.identifier", "minifi-test-id" + ) set_up_ssl_properties(context.get_or_create_default_minifi_container()) @@ -99,9 +142,18 @@ def start_minifi_c2_server(context: MinifiTestContext): assert context.containers["minifi-c2-server"].deploy(context) -@then("the MiNiFi C2 server logs contain the following message: \"{log_message}\" in less than {duration}") -def verify_c2_server_logs_contain_message(context: MinifiTestContext, log_message: str, duration: str): +@then( + 'the MiNiFi C2 server logs contain the following message: "{log_message}" in less than {duration}' +) +def verify_c2_server_logs_contain_message( + context: MinifiTestContext, log_message: str, duration: str +): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: log_message in context.containers["minifi-c2-server"].get_logs(), - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers["minifi-c2-server"].exited, - context=context) + assert wait_for_condition( + condition=lambda: ( + log_message in context.containers["minifi-c2-server"].get_logs() + ), + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers["minifi-c2-server"].exited, + context=context, + ) diff --git a/minifi_rust/extensions/minifi_rs_playground/features/environment.py b/minifi_rust/extensions/minifi_rs_playground/features/environment.py index 3ba4288e03..5ca5087ff8 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/environment.py +++ b/minifi_rust/extensions/minifi_rs_playground/features/environment.py @@ -16,16 +16,18 @@ # under the License. import os -from typing import List from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_after_scenario -from minifi_behave.core.hooks import common_before_scenario, get_minifi_container_image +from minifi_behave.core.hooks import ( + common_after_scenario, + common_before_scenario, + get_minifi_container_image, +) from minifi_behave.core.minifi_test_context import MinifiTestContext def add_extension_to_minifi_container( - extension_name: str, possible_paths: List[str], context: MinifiTestContext + extension_name: str, possible_paths: list[str], context: MinifiTestContext ): new_container_name = f"apacheminificpp:{extension_name}" is_windows = os.name == "nt" @@ -78,8 +80,12 @@ def add_extension_to_minifi_container( def before_all(context): dir_path = os.path.dirname(os.path.realpath(__file__)) build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/")) - deps_build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/deps/")) - add_extension_to_minifi_container("minifi_rs_playground", [build_path, deps_build_path], context) + deps_build_path = os.path.normpath( + os.path.join(dir_path, "../../../target/release/deps/") + ) + add_extension_to_minifi_container( + "minifi_rs_playground", [build_path, deps_build_path], context + ) def before_scenario(context, scenario): diff --git a/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py b/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py index f9a0ed64a5..9339b8bace 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py +++ b/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py @@ -15,15 +15,16 @@ # specific language governing permissions and limitations # under the License. -from behave import then, when, given import humanfriendly - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from behave import given, then, when from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @when("the MiNiFi instance is started without assertions") diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000000..664b7f6a22 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,15 @@ +builtins = ["log", "REL_SUCCESS", "REL_FAILURE", "REL_ORIGINAL", "raw_input"] + +extend-exclude = [ + "thirdparty", + "cmake-build-*", +] + +[lint] +# BLE001 (blind-except) is intentional at framework boundaries +ignore = ["BLE001"] + +[lint.per-file-ignores] +# G010 (logging-warn) is intentional in our Python code, because NiFi logging syntax differs +"extensions/python/*" = ["G010"] +"examples/scripts/python/*" = ["G010"] diff --git a/run_flake8.sh b/run_flake8.sh deleted file mode 100755 index 2288893bc5..0000000000 --- a/run_flake8.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -directory=${1:-.} -flake8 --exclude .venv,venv,thirdparty,build,cmake-build-*,github_env --builtins log,REL_SUCCESS,REL_FAILURE,REL_ORIGINAL,raw_input --ignore E501,W503,F811 "${directory}" diff --git a/thirdparty/google-styleguide/cpplint.py b/thirdparty/google-styleguide/cpplint.py old mode 100644 new mode 100755