From 17f44ac919431da7b503a84f941364250655e006 Mon Sep 17 00:00:00 2001 From: mibe Date: Tue, 26 Mar 2024 09:20:50 +0000 Subject: [PATCH 1/4] Add documentation build folder to .gitignore --- .gitignore | 1 + {changes => doc/changes}/changelog.md | 0 {changes => doc/changes}/changes_0.1.0.md | 0 exasol/python_extension_common/__init__.py | 0 .../deployment/language_container_deployer.py | 291 ++++++++++++++++++ .../language_container_deployer_cli.py | 168 ++++++++++ pyproject.toml | 15 + .../test_language_container_deployer.py | 133 ++++++++ .../test_language_container_deployer_cli.py | 29 ++ 9 files changed, 637 insertions(+) create mode 100644 .gitignore rename {changes => doc/changes}/changelog.md (100%) rename {changes => doc/changes}/changes_0.1.0.md (100%) create mode 100644 exasol/python_extension_common/__init__.py create mode 100644 exasol/python_extension_common/deployment/language_container_deployer.py create mode 100644 exasol/python_extension_common/deployment/language_container_deployer_cli.py create mode 100644 pyproject.toml create mode 100644 test/unit/deployment/test_language_container_deployer.py create mode 100644 test/unit/deployment/test_language_container_deployer_cli.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8568120 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.html-documentation diff --git a/changes/changelog.md b/doc/changes/changelog.md similarity index 100% rename from changes/changelog.md rename to doc/changes/changelog.md diff --git a/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md similarity index 100% rename from changes/changes_0.1.0.md rename to doc/changes/changes_0.1.0.md diff --git a/exasol/python_extension_common/__init__.py b/exasol/python_extension_common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exasol/python_extension_common/deployment/language_container_deployer.py b/exasol/python_extension_common/deployment/language_container_deployer.py new file mode 100644 index 0000000..c260965 --- /dev/null +++ b/exasol/python_extension_common/deployment/language_container_deployer.py @@ -0,0 +1,291 @@ +from enum import Enum +from textwrap import dedent +from typing import List, Optional +from pathlib import Path, PurePosixPath +import logging +import tempfile +import requests +import ssl +import pyexasol +from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation +from exasol_bucketfs_utils_python.bucket_config import BucketConfig, BucketFSConfig +from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig + +logger = logging.getLogger(__name__) + + +def create_bucketfs_location( + bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, + bucketfs_use_https: bool, bucketfs_user: str, bucketfs_password: str, + bucket: str, path_in_bucket: str) -> BucketFSLocation: + _bucketfs_connection = BucketFSConnectionConfig( + host=bucketfs_host, port=bucketfs_port, user=bucketfs_user, + pwd=bucketfs_password, is_https=bucketfs_use_https) + _bucketfs_config = BucketFSConfig( + bucketfs_name=bucketfs_name, connection_config=_bucketfs_connection) + _bucket_config = BucketConfig( + bucket_name=bucket, bucketfs_config=_bucketfs_config) + return BucketFSLocation( + bucket_config=_bucket_config, + base_path=PurePosixPath(path_in_bucket)) + + +def get_websocket_sslopt(use_ssl_cert_validation: bool = True, + ssl_trusted_ca: Optional[str] = None, + ssl_client_certificate: Optional[str] = None, + ssl_private_key: Optional[str] = None) -> dict: + """ + Returns a dictionary in the winsocket-client format + (see https://websocket-client.readthedocs.io/en/latest/faq.html#what-else-can-i-do-with-sslopts) + """ + + # Is server certificate validation required? + sslopt: dict[str, object] = {"cert_reqs": ssl.CERT_REQUIRED if use_ssl_cert_validation else ssl.CERT_NONE} + + # Is a bundle with trusted CAs provided? + if ssl_trusted_ca: + trusted_ca_path = Path(ssl_trusted_ca) + if trusted_ca_path.is_dir(): + sslopt["ca_cert_path"] = ssl_trusted_ca + elif trusted_ca_path.is_file(): + sslopt["ca_certs"] = ssl_trusted_ca + else: + raise ValueError(f"Trusted CA location {ssl_trusted_ca} doesn't exist.") + + # Is client's own certificate provided? + if ssl_client_certificate: + if not Path(ssl_client_certificate).is_file(): + raise ValueError(f"Certificate file {ssl_client_certificate} doesn't exist.") + sslopt["certfile"] = ssl_client_certificate + if ssl_private_key: + if not Path(ssl_private_key).is_file(): + raise ValueError(f"Private key file {ssl_private_key} doesn't exist.") + sslopt["keyfile"] = ssl_private_key + + return sslopt + + +class LanguageActivationLevel(Enum): + f""" + Language activation level, i.e. + ALTER SET SCRIPT_LANGUAGES=... + """ + Session = 'SESSION' + System = 'SYSTEM' + + +def get_language_settings(pyexasol_conn: pyexasol.ExaConnection, alter_type: LanguageActivationLevel) -> str: + """ + Reads the current language settings at the specified level. + + pyexasol_conn - Opened database connection. + alter_type - Activation level - SYSTEM or SESSION. + """ + result = pyexasol_conn.execute( + f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE + PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall() + return result[0][0] + + +class LanguageContainerDeployer: + + def __init__(self, + pyexasol_connection: pyexasol.ExaConnection, + language_alias: str, + bucketfs_location: BucketFSLocation) -> None: + + self._bucketfs_location = bucketfs_location + self._language_alias = language_alias + self._pyexasol_conn = pyexasol_connection + logger.debug(f"Init {LanguageContainerDeployer.__name__}") + + def download_and_run(self, url: str, + bucket_file_path: str, + alter_system: bool = True, + allow_override: bool = False) -> None: + """ + Downloads the language container from the provided url to a temporary file and then deploys it. + See docstring on the `run` method for details on what is involved in the deployment. + + url - Address where the container will be downloaded from. + bucket_file_path - Path within the designated bucket where the container should be uploaded. + alter_system - If True will try to activate the container at the System level. + allow_override - If True the activation of a language container with the same alias will be + overriden, otherwise a RuntimeException will be thrown. + """ + + with tempfile.NamedTemporaryFile() as tmp_file: + response = requests.get(url, stream=True) + response.raise_for_status() + tmp_file.write(response.content) + + self.run(Path(tmp_file.name), bucket_file_path, alter_system, allow_override) + + def run(self, container_file: Optional[Path] = None, + bucket_file_path: Optional[str] = None, + alter_system: bool = True, + allow_override: bool = False) -> None: + """ + Deploys the language container. This includes two steps, both of which are optional: + - Uploading the container into the database. This step can be skipped if the container + has already been uploaded. + - Activating the container. This step may have to be skipped if the user does not have + System Privileges in the database. In that case two alternative activation SQL commands + will be printed on the console. + + container_file - Path of the container tar.gz file in a local file system. + If not provided the container is assumed to be uploaded already. + bucket_file_path - Path within the designated bucket where the container should be uploaded. + If not specified the name of the container file will be used instead. + alter_system - If True will try to activate the container at the System level. + allow_override - If True the activation of a language container with the same alias will be + overriden, otherwise a RuntimeException will be thrown. + """ + + if not bucket_file_path: + if not container_file: + raise ValueError('Either a container file or a bucket file path must be specified.') + bucket_file_path = container_file.name + + if container_file: + self.upload_container(container_file, bucket_file_path) + + if alter_system: + self.activate_container(bucket_file_path, LanguageActivationLevel.System, allow_override) + else: + message = dedent(f""" + In SQL, you can activate the SLC of the Transformers Extension + by using the following statements: + + To activate the SLC only for the current session: + {self.generate_activation_command(bucket_file_path, LanguageActivationLevel.Session, True)} + + To activate the SLC on the system: + {self.generate_activation_command(bucket_file_path, LanguageActivationLevel.System, True)} + """) + print(message) + + def upload_container(self, container_file: Path, + bucket_file_path: Optional[str] = None) -> None: + """ + Upload the language container to the BucketFS. + + container_file - Path of the container tar.gz file in a local file system. + bucket_file_path - Path within the designated bucket where the container should be uploaded. + """ + if not container_file.is_file(): + raise RuntimeError(f"Container file {container_file} " + f"is not a file.") + with open(container_file, "br") as f: + self._bucketfs_location.upload_fileobj_to_bucketfs( + fileobj=f, bucket_file_path=bucket_file_path) + logging.debug("Container is uploaded to bucketfs") + + def activate_container(self, bucket_file_path: str, + alter_type: LanguageActivationLevel = LanguageActivationLevel.Session, + allow_override: bool = False) -> None: + """ + Activates the language container at the required level. + + bucket_file_path - Path within the designated bucket where the container is uploaded. + alter_type - Language activation level, defaults to the SESSION. + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. + """ + alter_command = self.generate_activation_command(bucket_file_path, alter_type, allow_override) + self._pyexasol_conn.execute(alter_command) + logging.debug(alter_command) + + def generate_activation_command(self, bucket_file_path: str, + alter_type: LanguageActivationLevel, + allow_override: bool = False) -> str: + """ + Generates an SQL command to activate the SLC container at the required level. The command will + preserve existing activations of other containers identified by different language aliases. + Activation of a container with the same alias, if exists, will be overwritten. + + bucket_file_path - Path within the designated bucket where the container is uploaded. + alter_type - Activation level - SYSTEM or SESSION. + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. + """ + path_in_udf = self._bucketfs_location.generate_bucket_udf_path(bucket_file_path) + new_settings = \ + self._update_previous_language_settings(alter_type, allow_override, path_in_udf) + alter_command = \ + f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';" + return alter_command + + def _update_previous_language_settings(self, alter_type: LanguageActivationLevel, + allow_override: bool, + path_in_udf: PurePosixPath) -> str: + prev_lang_settings = get_language_settings(self._pyexasol_conn, alter_type) + prev_lang_aliases = prev_lang_settings.split(" ") + self._check_if_requested_language_alias_already_exists( + allow_override, prev_lang_aliases) + new_definitions_str = self._generate_new_language_settings( + path_in_udf, prev_lang_aliases) + return new_definitions_str + + def get_language_definition(self, bucket_file_path: str): + """ + Generate a language definition (ALIAS=URL) for the specified bucket file path. + + bucket_file_path - Path within the designated bucket where the container is uploaded. + """ + path_in_udf = self._bucketfs_location.generate_bucket_udf_path(bucket_file_path) + result = self._generate_new_language_settings(path_in_udf=path_in_udf, prev_lang_aliases=[]) + return result + + def _generate_new_language_settings(self, path_in_udf: PurePosixPath, + prev_lang_aliases: List[str]) -> str: + other_definitions = [ + alias_definition for alias_definition in prev_lang_aliases + if not alias_definition.startswith(self._language_alias + "=")] + path_in_udf_without_buckets = PurePosixPath(*path_in_udf.parts[2:]) + new_language_alias_definition = \ + f"{self._language_alias}=localzmq+protobuf:///" \ + f"{path_in_udf_without_buckets}?lang=python#" \ + f"{path_in_udf}/exaudf/exaudfclient_py3" + new_definitions = other_definitions + [new_language_alias_definition] + new_definitions_str = " ".join(new_definitions) + return new_definitions_str + + def _check_if_requested_language_alias_already_exists( + self, allow_override: bool, + prev_lang_aliases: List[str]) -> None: + definition_for_requested_alias = [ + alias_definition for alias_definition in prev_lang_aliases + if alias_definition.startswith(self._language_alias + "=")] + if not len(definition_for_requested_alias) == 0: + warning_message = f"The requested language alias {self._language_alias} is already in use." + if allow_override: + logging.warning(warning_message) + else: + raise RuntimeError(warning_message) + + @classmethod + def create(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, + bucketfs_use_https: bool, bucketfs_user: str, + bucketfs_password: str, bucket: str, path_in_bucket: str, + dsn: str, db_user: str, db_password: str, language_alias: str, + use_ssl_cert_validation: bool = True, ssl_trusted_ca: Optional[str] = None, + ssl_client_certificate: Optional[str] = None, + ssl_private_key: Optional[str] = None) -> "LanguageContainerDeployer": + + websocket_sslopt = get_websocket_sslopt(use_ssl_cert_validation, ssl_trusted_ca, + ssl_client_certificate, ssl_private_key) + + pyexasol_conn = pyexasol.connect( + dsn=dsn, + user=db_user, + password=db_password, + encryption=True, + websocket_sslopt=websocket_sslopt + ) + + bucketfs_location = create_bucketfs_location( + bucketfs_name, bucketfs_host, bucketfs_port, bucketfs_use_https, + bucketfs_user, bucketfs_password, bucket, path_in_bucket) + + return cls(pyexasol_conn, language_alias, bucketfs_location) diff --git a/exasol/python_extension_common/deployment/language_container_deployer_cli.py b/exasol/python_extension_common/deployment/language_container_deployer_cli.py new file mode 100644 index 0000000..fb0b222 --- /dev/null +++ b/exasol/python_extension_common/deployment/language_container_deployer_cli.py @@ -0,0 +1,168 @@ +from typing import Optional, Any +import os +import re +import click +from enum import Enum +from pathlib import Path +from exasol.python_transformers_extension.deployment import deployment_utils as utils +from exasol_transformers_extension.deployment.language_container_deployer import LanguageContainerDeployer + + +class CustomizableParameters(Enum): + """ + Parameters of the cli that can be programmatically customised by a developer + of a specialised version of the cli. + The names in the enum list should match the parameter names in language_container_deployer_main. + """ + container_url = 1 + container_name = 2 + + +class _ParameterFormatters: + """ + Class facilitating customization of the cli. + + The idea is that some of the cli parameters can be programmatically customized based + on values of other parameters and externally supplied formatters. For example a specialized + version of the cli may want to provide its own url. Furthermore, this url will depend on + the user supplied parameter called "version". The solution is to set a formatter for the + url, for instance "http://my_stuff/{version}/my_data". If the user specifies non-empty version + parameter the url will be fully formed. + + A formatter may include more than one parameter. In the previous example the url could, + for instance, also include a username: "http://my_stuff/{version}/{user}/my_data". + + Note that customized parameters can only be updated in a callback function. There is no + way to inject them directly into the cli. Also, the current implementation doesn't perform + the update if the value of the parameter dressed with the callback is None. + + IMPORTANT! Please make sure that the formatters are set up before the call to the cli function, + e.g. language_container_deployer_main, is executed. + """ + def __init__(self): + self._formatters = {} + + def __call__(self, ctx: click.Context, param: click.Parameter, value: Optional[Any]) -> Optional[Any]: + + def update_parameter(parameter_name: str, formatter: str) -> None: + param_formatter = ctx.params.get(parameter_name, formatter) + if param_formatter: + # Enclose in double curly brackets all other parameters in the formatting string, + # to avoid the missing parameters' error. Below is an example of a formatter string + # before and after applying the regex, assuming the current parameter is 'version'. + # 'something-with-{version}/tailored-for-{user}' => 'something-with-{version}/tailored-for-{{user}}' + # We were looking for all occurrences of a pattern '{some_name}', where some_name is not version. + pattern = r'\{(?!' + param.name + r'\})\w+\}' + param_formatter = re.sub(pattern, lambda m: f'{{{m.group(0)}}}', param_formatter) + kwargs = {param.name: value} + ctx.params[parameter_name] = param_formatter.format(**kwargs) + + if value is not None: + for prm_name, prm_formatter in self._formatters.items(): + update_parameter(prm_name, prm_formatter) + + return value + + def set_formatter(self, custom_parameter: CustomizableParameters, formatter: str) -> None: + """ Sets a formatter for a customizable parameter. """ + self._formatters[custom_parameter.name] = formatter + + def clear_formatters(self): + """ Deletes all formatters, mainly for testing purposes. """ + self._formatters.clear() + + +# Global cli customization object. +# Specialized versions of this cli should use this object to set custom parameter formatters. +slc_parameter_formatters = _ParameterFormatters() + + +@click.command(name="language-container") +@click.option('--bucketfs-name', type=str, required=True) +@click.option('--bucketfs-host', type=str, required=True) +@click.option('--bucketfs-port', type=int, required=True) +@click.option('--bucketfs-use-https', type=bool, default=False) +@click.option('--bucketfs-user', type=str, required=True, default="w") +@click.option('--bucketfs-password', prompt='bucketFS password', hide_input=True, + default=lambda: os.environ.get(utils.BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE, "")) +@click.option('--bucket', type=str, required=True) +@click.option('--path-in-bucket', type=str, required=True, default=None) +@click.option('--container-file', + type=click.Path(exists=True, file_okay=True), default=None) +@click.option('--version', type=str, default=None, expose_value=False, + callback=slc_parameter_formatters) +@click.option('--dsn', type=str, required=True) +@click.option('--db-user', type=str, required=True) +@click.option('--db-pass', prompt='db password', hide_input=True, + default=lambda: os.environ.get(utils.DB_PASSWORD_ENVIRONMENT_VARIABLE, "")) +@click.option('--language-alias', type=str, default="PYTHON3_TE") +@click.option('--ssl-cert-path', type=str, default="") +@click.option('--ssl-client-cert-path', type=str, default="") +@click.option('--ssl-client-private-key', type=str, default="") +@click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True) +@click.option('--upload-container/--no-upload_container', type=bool, default=True) +@click.option('--alter-system/--no-alter-system', type=bool, default=True) +@click.option('--allow-override/--disallow-override', type=bool, default=False) +def language_container_deployer_main( + bucketfs_name: str, + bucketfs_host: str, + bucketfs_port: int, + bucketfs_use_https: bool, + bucketfs_user: str, + bucketfs_password: str, + bucket: str, + path_in_bucket: str, + container_file: str, + dsn: str, + db_user: str, + db_pass: str, + language_alias: str, + ssl_cert_path: str, + ssl_client_cert_path: str, + ssl_client_private_key: str, + use_ssl_cert_validation: bool, + upload_container: bool, + alter_system: bool, + allow_override: bool, + container_url: str = None, + container_name: str = None): + + deployer = LanguageContainerDeployer.create( + bucketfs_name=bucketfs_name, + bucketfs_host=bucketfs_host, + bucketfs_port=bucketfs_port, + bucketfs_use_https=bucketfs_use_https, + bucketfs_user=bucketfs_user, + bucketfs_password=bucketfs_password, + bucket=bucket, + path_in_bucket=path_in_bucket, + dsn=dsn, + db_user=db_user, + db_password=db_pass, + language_alias=language_alias, + ssl_trusted_ca=ssl_cert_path, + ssl_client_certificate=ssl_client_cert_path, + ssl_private_key=ssl_client_private_key, + use_ssl_cert_validation=use_ssl_cert_validation) + + if not upload_container: + deployer.run(alter_system=alter_system, allow_override=allow_override) + elif container_file: + deployer.run(container_file=Path(container_file), alter_system=alter_system, allow_override=allow_override) + elif container_url and container_name: + deployer.download_and_run(container_url, container_name, alter_system=alter_system, + allow_override=allow_override) + else: + # The error message should mention the parameters which the callback is specified for being missed. + raise ValueError("To upload a language container you should specify either its " + "release version or a path of the already downloaded container file.") + + +if __name__ == '__main__': + import logging + + logging.basicConfig( + format='%(asctime)s - %(module)s - %(message)s', + level=logging.DEBUG) + + language_container_deployer_main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..aa8c864 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "python-extension-common" +version = "0.1.0" +description = "A collection of common utilities for Exasol extensions." +authors = ["Your Name "] +license = "MIT" +readme = "README.md" + +[tool.poetry.dependencies] +python = "[^3.8]" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/test/unit/deployment/test_language_container_deployer.py b/test/unit/deployment/test_language_container_deployer.py new file mode 100644 index 0000000..746c442 --- /dev/null +++ b/test/unit/deployment/test_language_container_deployer.py @@ -0,0 +1,133 @@ +######################################################### +# To be migrated to the script-languages-container-tool # +######################################################### +from pathlib import Path, PurePosixPath +from unittest.mock import create_autospec, MagicMock, patch + +import pytest +from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation +from pyexasol import ExaConnection + +from exasol_transformers_extension.deployment.language_container_deployer import ( + LanguageContainerDeployer, LanguageActivationLevel) + + +@pytest.fixture(scope='module') +def container_file_name() -> str: + return 'container_xyz.tag.gz' + + +@pytest.fixture(scope='module') +def container_file_path(container_file_name) -> Path: + return Path(container_file_name) + + +@pytest.fixture(scope='module') +def language_alias() -> str: + return 'PYTHON3_TEST' + + +@pytest.fixture(scope='module') +def container_bfs_path(container_file_name) -> str: + return f'bfsdefault/default/container/{container_file_name[:-7]}' + + +@pytest.fixture(scope='module') +def mock_pyexasol_conn() -> ExaConnection: + return create_autospec(ExaConnection) + + +@pytest.fixture(scope='module') +def mock_bfs_location(container_bfs_path) -> BucketFSLocation: + mock_loc = create_autospec(BucketFSLocation) + mock_loc.generate_bucket_udf_path.return_value = PurePosixPath(f'/buckets/{container_bfs_path}') + return mock_loc + + +@pytest.fixture +def container_deployer(mock_pyexasol_conn, mock_bfs_location, language_alias) -> LanguageContainerDeployer: + deployer = LanguageContainerDeployer(pyexasol_connection=mock_pyexasol_conn, + language_alias=language_alias, + bucketfs_location=mock_bfs_location) + + deployer.upload_container = MagicMock() + deployer.activate_container = MagicMock() + return deployer + + +def test_slc_deployer_deploy(container_deployer, container_file_name, container_file_path): + container_deployer.run(container_file=container_file_path, bucket_file_path=container_file_name, alter_system=True, + allow_override=True) + container_deployer.upload_container.assert_called_once_with(container_file_path, container_file_name) + container_deployer.activate_container.assert_called_once_with(container_file_name, LanguageActivationLevel.System, + True) + + +def test_slc_deployer_upload(container_deployer, container_file_name, container_file_path): + container_deployer.run(container_file=container_file_path, alter_system=False) + container_deployer.upload_container.assert_called_once_with(container_file_path, container_file_name) + container_deployer.activate_container.assert_not_called() + + +def test_slc_deployer_activate(container_deployer, container_file_name, container_file_path): + container_deployer.run(bucket_file_path=container_file_name, alter_system=True, allow_override=True) + container_deployer.upload_container.assert_not_called() + container_deployer.activate_container.assert_called_once_with(container_file_name, LanguageActivationLevel.System, + True) + + +@patch('exasol_transformers_extension.deployment.language_container_deployer.get_language_settings') +def test_slc_deployer_generate_activation_command(mock_lang_settings, container_deployer, language_alias, + container_file_name, container_bfs_path): + mock_lang_settings.return_value = 'R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3' + + alter_type = LanguageActivationLevel.Session + expected_command = f"ALTER {alter_type.value.upper()} SET SCRIPT_LANGUAGES='" \ + "R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 " \ + f"{language_alias}=localzmq+protobuf:///{container_bfs_path}?" \ + f"lang=python#/buckets/{container_bfs_path}/exaudf/exaudfclient_py3';" + + command = container_deployer.generate_activation_command(container_file_name, alter_type) + assert command == expected_command + + +@patch('exasol_transformers_extension.deployment.language_container_deployer.get_language_settings') +def test_slc_deployer_generate_activation_command_override(mock_lang_settings, container_deployer, language_alias, + container_file_name, container_bfs_path): + current_bfs_path = 'bfsdefault/default/container_abc' + mock_lang_settings.return_value = \ + 'R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 ' \ + f'{language_alias}=localzmq+protobuf:///{current_bfs_path}?' \ + f'lang=python#/buckets/{current_bfs_path}/exaudf/exaudfclient_py3' + + alter_type = LanguageActivationLevel.Session + expected_command = f"ALTER {alter_type.value.upper()} SET SCRIPT_LANGUAGES='" \ + "R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 " \ + f"{language_alias}=localzmq+protobuf:///{container_bfs_path}?" \ + f"lang=python#/buckets/{container_bfs_path}/exaudf/exaudfclient_py3';" + + command = container_deployer.generate_activation_command(container_file_name, alter_type, allow_override=True) + assert command == expected_command + + +@patch('exasol_transformers_extension.deployment.language_container_deployer.get_language_settings') +def test_slc_deployer_generate_activation_command_failure(mock_lang_settings, container_deployer, language_alias, + container_file_name): + current_bfs_path = 'bfsdefault/default/container_abc' + mock_lang_settings.return_value = \ + 'R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 ' \ + f'{language_alias}=localzmq+protobuf:///{current_bfs_path}?' \ + f'lang=python#/buckets/{current_bfs_path}/exaudf/exaudfclient_py3' + + with pytest.raises(RuntimeError): + container_deployer.generate_activation_command(container_file_name, LanguageActivationLevel.Session, + allow_override=False) + + +def test_slc_deployer_get_language_definition(container_deployer, language_alias, + container_file_name, container_bfs_path): + expected_command = f"{language_alias}=localzmq+protobuf:///{container_bfs_path}?" \ + f"lang=python#/buckets/{container_bfs_path}/exaudf/exaudfclient_py3" + + command = container_deployer.get_language_definition(container_file_name) + assert command == expected_command diff --git a/test/unit/deployment/test_language_container_deployer_cli.py b/test/unit/deployment/test_language_container_deployer_cli.py new file mode 100644 index 0000000..dcaf837 --- /dev/null +++ b/test/unit/deployment/test_language_container_deployer_cli.py @@ -0,0 +1,29 @@ +import click +from exasol_transformers_extension.deployment.language_container_deployer_cli import ( + _ParameterFormatters, CustomizableParameters) + + +def test_parameter_formatters_1param(): + cmd = click.Command('a_command') + ctx = click.Context(cmd) + opt = click.Option(['--version']) + formatters = _ParameterFormatters() + formatters.set_formatter(CustomizableParameters.container_url, 'http://my_server/{version}/my_stuff') + formatters.set_formatter(CustomizableParameters.container_name, 'downloaded') + formatters(ctx, opt, '1.3.2') + assert ctx.params[CustomizableParameters.container_url.name] == 'http://my_server/1.3.2/my_stuff' + assert ctx.params[CustomizableParameters.container_name.name] == 'downloaded' + + +def test_parameter_formatters_2params(): + cmd = click.Command('a_command') + ctx = click.Context(cmd) + opt1 = click.Option(['--version']) + opt2 = click.Option(['--user']) + formatters = _ParameterFormatters() + formatters.set_formatter(CustomizableParameters.container_url, 'http://my_server/{version}/{user}/my_stuff') + formatters.set_formatter(CustomizableParameters.container_name, 'downloaded-{version}') + formatters(ctx, opt1, '1.3.2') + formatters(ctx, opt2, 'cezar') + assert ctx.params[CustomizableParameters.container_url.name] == 'http://my_server/1.3.2/cezar/my_stuff' + assert ctx.params[CustomizableParameters.container_name.name] == 'downloaded-1.3.2' From 8d6ef94bb1f9065d80f52d34fec068ff6fb02939 Mon Sep 17 00:00:00 2001 From: mibe Date: Wed, 17 Jun 2026 08:19:07 +0100 Subject: [PATCH 2/4] #154: Changed slow workflows --- .github/workflows/slow-checks-itde.yml | 68 +++++++++++++++++++++ .github/workflows/slow-checks-saas.yml | 45 ++++++++++++++ .github/workflows/slow-checks.yml | 81 ++++---------------------- 3 files changed, 123 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/slow-checks-itde.yml create mode 100644 .github/workflows/slow-checks-saas.yml diff --git a/.github/workflows/slow-checks-itde.yml b/.github/workflows/slow-checks-itde.yml new file mode 100644 index 0000000..4b003a7 --- /dev/null +++ b/.github/workflows/slow-checks-itde.yml @@ -0,0 +1,68 @@ +name: Slow-Checks-ITDE + +on: + workflow_call: + +jobs: + build-matrix: + name: Build Matrix + uses: ./.github/workflows/matrix-all.yml + permissions: + contents: read + + run-integration-tests: + name: Run Integration Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + needs: + - build-matrix + runs-on: "ubuntu-24.04" + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + + steps: + - name: Allow unprivileged user namespaces + id: allow-unprivileged-user-namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + large-packages: false + + - name: Free disk space by removing large directories + run: | + sudo rm -rf /usr/local/graalvm/ + sudo rm -rf /usr/local/.ghcup/ + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/lib/node_modules + sudo rm -rf /opt/ghc + + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v8 + with: + python-version: ${{ matrix.python-version }} + poetry-version: "2.3.0" + + - name: Run Integration Tests + id: run-integration-tests + run: > + poetry run -- nox -s test:integration -- -s --coverage + --backend=onprem + --db-version ${{ matrix.exasol-version }} + + - name: Upload Artifacts + id: upload-artifacts + uses: actions/upload-artifact@v7 + with: + name: coverage-python${{ matrix.python-version }}-exasol${{ matrix.exasol-version }}-slow-itde + path: .coverage + include-hidden-files: true diff --git a/.github/workflows/slow-checks-saas.yml b/.github/workflows/slow-checks-saas.yml new file mode 100644 index 0000000..9481ea5 --- /dev/null +++ b/.github/workflows/slow-checks-saas.yml @@ -0,0 +1,45 @@ +name: Slow-Checks-SaaS + +on: + workflow_call: + +jobs: + run-integration-tests: + name: Run Integration Tests with SaaS (Python-3.12) + runs-on: "ubuntu-24.04" + permissions: + contents: read + + steps: + - name: Allow unprivileged user namespaces + id: allow-unprivileged-user-namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v8 + with: + python-version: "3.12" + poetry-version: "2.3.0" + + - name: Run Integration Tests + id: run-integration-tests + run: > + poetry run -- nox -s test:integration -- -s --coverage + --backend=saas + env: + SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} + SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} + SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} + + - name: Upload Artifacts + id: upload-artifacts + uses: actions/upload-artifact@v7 + with: + name: coverage-python3.12-slow-saas + path: .coverage + include-hidden-files: true diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index 7c3c6eb..de34d1a 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -1,80 +1,19 @@ -# Please note that this workflow was manually adapted from the version generated -# by the Exasol Python Toolbox. -# -# 1. The matrix consists of backend type and Exasol version -# 2. Disabling AppArmor namespace restrictions is required to run dockerdb -# 3. To make sure space on the standard runner is freed up, -# unneeded large directories are removed -# -# Once the workflow patcher can do this modification without having to replace -# each workflow step individually, this workflow should be generated. - name: Slow-Checks on: workflow_call: jobs: - - run-integration-tests: - name: Integration Tests (${{ matrix.backend }}, ${{ matrix.exasol-version}}) - runs-on: "ubuntu-24.04" + run-slow-checks-itde: + name: Run Slow Checks with ITDE + uses: ./.github/workflows/slow-checks-itde.yml + secrets: inherit permissions: contents: read - strategy: - fail-fast: false - matrix: - include: - - backend: saas - exasol-version: provided - - backend: onprem - exasol-version: "2025.1.8" - - steps: - - name: Allow unprivileged user namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - - name: Free disk space - uses: jlumbroso/free-disk-space@main - with: - tool-cache: true - large-packages: false - - - name: Free disk space by removing large directories - run: | - sudo rm -rf /usr/local/graalvm/ - sudo rm -rf /usr/local/.ghcup/ - sudo rm -rf /usr/local/share/powershell - sudo rm -rf /usr/local/share/chromium - sudo rm -rf /usr/local/lib/node_modules - sudo rm -rf /opt/ghc - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v7 - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Run Integration Tests - id: run-integration-tests - run: > - poetry run -- nox -s test:integration -- --coverage - --backend ${{ matrix.backend }} - --db-version ${{ matrix.exasol-version }} - env: - SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} - SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} - SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} - - - name: Upload Artifacts - id: upload-artifacts - uses: actions/upload-artifact@v7 - with: - name: coverage-python3.10-${{ matrix.backend }}-${{ matrix.exasol-version }} - path: .coverage - include-hidden-files: true + run-slow-checks-saas: + name: Run Slow Checks with SaaS + uses: ./.github/workflows/slow-checks-saas.yml + secrets: inherit + permissions: + contents: read From 25159f4b35c62afc388e217a33fedf6772d25a9c Mon Sep 17 00:00:00 2001 From: mibe Date: Wed, 17 Jun 2026 09:36:11 +0100 Subject: [PATCH 3/4] #154: Restored matrix-all.yml --- .github/workflows/matrix-all.yml | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/matrix-all.yml diff --git a/.github/workflows/matrix-all.yml b/.github/workflows/matrix-all.yml new file mode 100644 index 0000000..ce5f01d --- /dev/null +++ b/.github/workflows/matrix-all.yml @@ -0,0 +1,37 @@ +name: Build Matrix (All) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the full build matrix for Python & Exasol versions" + value: ${{ jobs.set-matrix-all.outputs.matrix }} + +jobs: + set-matrix-all: + runs-on: "ubuntu-24.04" + permissions: + contents: read + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v8 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Generate Matrix + id: generate-matrix + run: poetry run -- nox -s matrix:all + + - name: Set Matrix + id: set-matrix + run: | + echo "matrix=$(poetry run -- nox -s matrix:all)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} From b20d1f1345e28823bf4c6ddaf60f6bc282fb2e50 Mon Sep 17 00:00:00 2001 From: mibe Date: Wed, 17 Jun 2026 14:18:01 +0100 Subject: [PATCH 4/4] #154: Changed test file names --- .../connections/test_bucketfs_location.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/integration/connections/test_bucketfs_location.py b/test/integration/connections/test_bucketfs_location.py index 3739822..98f8656 100644 --- a/test/integration/connections/test_bucketfs_location.py +++ b/test/integration/connections/test_bucketfs_location.py @@ -54,7 +54,7 @@ def test_create_bucketfs_location_onprem(use_onprem, onprem_bfs_params): if not use_onprem: pytest.skip("The test is not configured to use ITDE.") - extra_params = {StdParams.path_in_bucket.name: "test_create_location"} + extra_params = {StdParams.path_in_bucket.name: "test_create_location.csv"} bfs_path = create_bucketfs_location(**onprem_bfs_params, **extra_params) with write_test_file(bfs_path): validate_test_file(bfs_path) @@ -64,7 +64,7 @@ def test_create_bucketfs_location_saas_db_id(use_saas, saas_params_id): if not use_saas: pytest.skip("The test is not configured to use SaaS.") - extra_params = {StdParams.path_in_bucket.name: "test_create_location_with_id"} + extra_params = {StdParams.path_in_bucket.name: "test_create_location_with_id.csv"} bfs_path = create_bucketfs_location(**saas_params_id, **extra_params) with write_test_file(bfs_path): validate_test_file(bfs_path) @@ -74,7 +74,7 @@ def test_create_bucketfs_location_saas_db_name(use_saas, saas_params_name): if not use_saas: pytest.skip("The test is not configured to use SaaS.") - extra_params = {StdParams.path_in_bucket.name: "test_create_location_with_name"} + extra_params = {StdParams.path_in_bucket.name: "test_create_location_with_name.csv"} bfs_path = create_bucketfs_location(**saas_params_name, **extra_params) with write_test_file(bfs_path): validate_test_file(bfs_path) @@ -93,7 +93,7 @@ def test_create_bucketfs_conn_object_onprem( pytest.skip("The test is not configured to use ITDE.") write_conn_object_mock.side_effect = validate_conn_object - extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object"} + extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object.csv"} bfs_path = create_bucketfs_location(**onprem_bfs_params, **extra_params) with write_test_file(bfs_path): # onprem_db_params and onprem_bfs_params have one item in common - @@ -109,7 +109,7 @@ def test_create_bucketfs_conn_object_saas_db_id(write_conn_object_mock, use_saas pytest.skip("The test is not configured to use SaaS.") write_conn_object_mock.side_effect = validate_conn_object - extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object_with_id"} + extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object_with_id.csv"} bfs_path = create_bucketfs_location(**saas_params_id, **extra_params) with write_test_file(bfs_path): create_bucketfs_conn_object(conn_name="SAAS_TEST_BFS_ID", **saas_params_id, **extra_params) @@ -123,7 +123,7 @@ def test_create_bucketfs_conn_object_saas_db_name( pytest.skip("The test is not configured to use SaaS.") write_conn_object_mock.side_effect = validate_conn_object - extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object_with_name"} + extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object_with_name.csv"} bfs_path = create_bucketfs_location(**saas_params_name, **extra_params) with write_test_file(bfs_path): create_bucketfs_conn_object(