From 0f6ea8a1ec0f6b1852793039fa55108e1c7715e4 Mon Sep 17 00:00:00 2001 From: TrisNol Date: Sat, 9 Sep 2023 17:59:30 +0200 Subject: [PATCH 1/3] feat(config): Read secrets from .env file and environemnt variables --- README.md | 16 ++++++ poetry.lock | 16 +++++- pyproject.toml | 3 +- .../config/config_providers.py | 47 ++++++++++++++++ tests/config/config_providers_test.py | 54 ++++++++++++++++++- 5 files changed, 133 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bc669d1..5347f6a 100644 --- a/README.md +++ b/README.md @@ -37,3 +37,19 @@ Create a `secrets.json` in the root of this repo with the following structure (v } } ``` + +Alternatively, the secrets can be provided as environment variables. One option to do so is to add a `.env` file with the following layout: +```js +PYTHON_POSTGRES_USERNAME=postgres +PYTHON_POSTGRES_PASSWORD=postgres +PYTHON_POSTGRES_HOST=localhost +PYTHON_POSTGRES_DATABASE=postgres +PYTHON_POSTGRES_PORT=5432 +PYTHON_MONGO_USERNAME=username +PYTHON_MONGO_HOST=localhost +PYTHON_MONGO_PASSWORD=password +PYTHON_MONGO_PORT=27017 +PYTHON_MONGO_DATABASE=transparenzregister +``` + +The prefix `PYTHON_` can be customized by setting a different `prefix` when constructing the ConfigProvider. diff --git a/poetry.lock b/poetry.lock index a2368dd..a38f961 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3960,6 +3960,20 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "1.0.0" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.0.tar.gz", hash = "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba"}, + {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "python-json-logger" version = "2.0.7" @@ -5551,4 +5565,4 @@ ingest = ["selenium"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "2022313907bbdb9e2b74f857e3248ee83892e1ae852a381cbe182b1c7537d285" +content-hash = "d4c6d872709bbd42dc42b4f45149a0e35bc72cfa5a559be745f9ab945d3e1849" diff --git a/pyproject.toml b/pyproject.toml index 2aa450f..39b624c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,11 +45,12 @@ plotly = "^5.14.1" psycopg2-binary = "^2.9.7" pymongo = "^4.4.1" python = "^3.11" +# TODO Add dependent libraries (i.e., deutshcland, plotly, etc) +python-dotenv = "^1.0.0" seaborn = "^0.12.2" selenium = "^4.10.0" tqdm = "^4.65.0" -# TODO Add dependent libraries (i.e., deutshcland, plotly, etc) [tool.poetry.extras] ingest = ["selenium"] diff --git a/src/aki_prj23_transparenzregister/config/config_providers.py b/src/aki_prj23_transparenzregister/config/config_providers.py index c6c4483..82c5c27 100644 --- a/src/aki_prj23_transparenzregister/config/config_providers.py +++ b/src/aki_prj23_transparenzregister/config/config_providers.py @@ -4,6 +4,8 @@ import abc import json import os +from dotenv import load_dotenv + from aki_prj23_transparenzregister.config.config_template import PostgreConnectionString from aki_prj23_transparenzregister.utils.mongo.connector import MongoConnection @@ -89,3 +91,48 @@ class JsonFileConfigProvider(ConfigProvider): details["username"], details["password"], ) + + +class EnvironmentConfigProvider(ConfigProvider): + """Config provider based on .json file.""" + + __data__: dict = {} + + def __init__(self, prefix: str = "PYTHON_"): + """Reads secrets from local environment while also ingesting .env files if available. + + Args: + prefix (str, optional): Variable prefix. Defaults to "PYTHON_". + """ + load_dotenv() + relevant_keys = [key for key in os.environ if key.startswith(prefix)] + for key in relevant_keys: + self.__data__[key.replace(prefix, "")] = os.environ.get(key) + + def get_postgre_connection_string(self) -> PostgreConnectionString: + """Read PostgreSQL connection string from environment variables. + + Returns: + PostgreConnectionString: Connection details + """ + return PostgreConnectionString( + self.__data__["POSTGRES_USERNAME"], + self.__data__["POSTGRES_PASSWORD"], + self.__data__["POSTGRES_HOST"], + self.__data__["POSTGRES_DATABASE"], + self.__data__["POSTGRES_PORT"], + ) + + def get_mongo_connection_string(self) -> MongoConnection: + """Read MongodB connection string from environment variables. + + Returns: + MongoConnection: Connection details + """ + return MongoConnection( + self.__data__["MONGO_HOST"], + self.__data__["MONGO_DATABASE"], + self.__data__["MONGO_PORT"], + self.__data__["MONGO_USERNAME"], + self.__data__["MONGO_PASSWORD"], + ) diff --git a/tests/config/config_providers_test.py b/tests/config/config_providers_test.py index ca3b95b..8be0367 100644 --- a/tests/config/config_providers_test.py +++ b/tests/config/config_providers_test.py @@ -3,7 +3,10 @@ from unittest.mock import mock_open, patch import pytest -from aki_prj23_transparenzregister.config.config_providers import JsonFileConfigProvider +from aki_prj23_transparenzregister.config.config_providers import ( + EnvironmentConfigProvider, + JsonFileConfigProvider, +) def test_json_provider_init_fail() -> None: @@ -72,3 +75,52 @@ def test_json_provider_get_mongo() -> None: assert config.hostname == data["mongo"]["host"] assert config.database == data["mongo"]["database"] assert config.port == data["mongo"]["port"] + + +def test_env_provider_constructor() -> None: + with patch("os.environ.keys") as mock_keys: + keys = ["PYTHON_TEST", "NOT_PYTHON_TEST"] + mock_keys.return_value = keys + with patch("os.environ.get") as mock_get: + value = "test" + mock_get.return_value = value + provider = EnvironmentConfigProvider() + assert provider.__data__ == {"TEST": value} + + +def test_env_provider_postgres() -> None: + provider = EnvironmentConfigProvider() + env_data = { + "POSTGRES_USERNAME": "postgres", + "POSTGRES_PASSWORD": "postgres", + "POSTGRES_HOST": "localhost", + "POSTGRES_DATABASE": "postgres", + "POSTGRES_PORT": "5432", + } + provider.__data__ = env_data + conn_string = provider.get_postgre_connection_string() + + assert conn_string.database == env_data["POSTGRES_DATABASE"] + assert conn_string.host == env_data["POSTGRES_HOST"] + assert conn_string.password == env_data["POSTGRES_PASSWORD"] + assert conn_string.port == env_data["POSTGRES_PORT"] + assert conn_string.username == env_data["POSTGRES_USERNAME"] + + +def test_env_provider_mongodb() -> None: + provider = EnvironmentConfigProvider() + env_data = { + "MONGO_USERNAME": "username", + "MONGO_HOST": "localhost", + "MONGO_PASSWORD": "password", + "MONGO_PORT": 27017, + "MONGO_DATABASE": "transparenzregister", + } + provider.__data__ = env_data + conn_string = provider.get_mongo_connection_string() + + assert conn_string.database == env_data["MONGO_DATABASE"] + assert conn_string.hostname == env_data["MONGO_HOST"] + assert conn_string.password == env_data["MONGO_PASSWORD"] + assert conn_string.port == env_data["MONGO_PORT"] + assert conn_string.username == env_data["MONGO_USERNAME"] From 330eb466e341dc81e0bc0aa668c4fd28a5b2362a Mon Sep 17 00:00:00 2001 From: TrisNol Date: Sat, 9 Sep 2023 18:13:59 +0200 Subject: [PATCH 2/3] test(config): Fix test with os.environ mocking --- tests/config/config_providers_test.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/config/config_providers_test.py b/tests/config/config_providers_test.py index 8be0367..0d00cfd 100644 --- a/tests/config/config_providers_test.py +++ b/tests/config/config_providers_test.py @@ -78,14 +78,11 @@ def test_json_provider_get_mongo() -> None: def test_env_provider_constructor() -> None: - with patch("os.environ.keys") as mock_keys: - keys = ["PYTHON_TEST", "NOT_PYTHON_TEST"] - mock_keys.return_value = keys - with patch("os.environ.get") as mock_get: - value = "test" - mock_get.return_value = value - provider = EnvironmentConfigProvider() - assert provider.__data__ == {"TEST": value} + with patch("aki_prj23_transparenzregister.config.config_providers.os") as mock_os: + keys = {"PYTHON_TEST": "test", "NOT_PYTHON_TEST": ""} + mock_os.environ = keys + provider = EnvironmentConfigProvider() + assert provider.__data__ == {"TEST": "test"} def test_env_provider_postgres() -> None: From 2c8805e12f68921c42d1f946f9a2493b2703ebdf Mon Sep 17 00:00:00 2001 From: TrisNol Date: Sat, 9 Sep 2023 18:22:20 +0200 Subject: [PATCH 3/3] checkpoint: Implement PR feedback --- README.md | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 5347f6a..1e5e308 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Create a `secrets.json` in the root of this repo with the following structure (v ``` Alternatively, the secrets can be provided as environment variables. One option to do so is to add a `.env` file with the following layout: -```js +```ini PYTHON_POSTGRES_USERNAME=postgres PYTHON_POSTGRES_PASSWORD=postgres PYTHON_POSTGRES_HOST=localhost diff --git a/pyproject.toml b/pyproject.toml index 39b624c..9c0368d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ plotly = "^5.14.1" psycopg2-binary = "^2.9.7" pymongo = "^4.4.1" python = "^3.11" -# TODO Add dependent libraries (i.e., deutshcland, plotly, etc) python-dotenv = "^1.0.0" seaborn = "^0.12.2" selenium = "^4.10.0"