feat(config): Read secrets from .env file and environemnt variables (#109)

Config variables used to connect to our two databased can now also be
ingested from the local environment variables. Such variables can also
be passed in using a `.env` file as described in the README.md

README.md

@@ -37,3 +37,19 @@ Create a `secrets.json` in the root of this repo with the following structure (v
   }
 }
 ```
+
+Alternatively, the secrets can be provided as environment variables. One option to do so is to add a `.env` file with the following layout:
+```ini
+PYTHON_POSTGRES_USERNAME=postgres
+PYTHON_POSTGRES_PASSWORD=postgres
+PYTHON_POSTGRES_HOST=localhost
+PYTHON_POSTGRES_DATABASE=postgres
+PYTHON_POSTGRES_PORT=5432
+PYTHON_MONGO_USERNAME=username
+PYTHON_MONGO_HOST=localhost
+PYTHON_MONGO_PASSWORD=password
+PYTHON_MONGO_PORT=27017
+PYTHON_MONGO_DATABASE=transparenzregister
+```
+
+The prefix `PYTHON_` can be customized by setting a different `prefix` when constructing the ConfigProvider.

poetry.lock

@@ -3960,6 +3960,20 @@ files = [
 [package.dependencies]
 six = ">=1.5"
 
+[[package]]
+name = "python-dotenv"
+version = "1.0.0"
+description = "Read key-value pairs from a .env file and set them as environment variables"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "python-dotenv-1.0.0.tar.gz", hash = "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba"},
+    {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"},
+]
+
+[package.extras]
+cli = ["click (>=5.0)"]
+
 [[package]]
 name = "python-json-logger"
 version = "2.0.7"
@@ -5551,4 +5565,4 @@ ingest = ["selenium"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.11"
-content-hash = "2022313907bbdb9e2b74f857e3248ee83892e1ae852a381cbe182b1c7537d285"
+content-hash = "d4c6d872709bbd42dc42b4f45149a0e35bc72cfa5a559be745f9ab945d3e1849"

pyproject.toml

@@ -45,11 +45,11 @@ plotly = "^5.14.1"
 psycopg2-binary = "^2.9.7"
 pymongo = "^4.4.1"
 python = "^3.11"
+python-dotenv = "^1.0.0"
 seaborn = "^0.12.2"
 selenium = "^4.10.0"
 tqdm = "^4.65.0"
 
-# TODO Add dependent libraries (i.e., deutshcland, plotly, etc)
 [tool.poetry.extras]
 ingest = ["selenium"]
 

src/aki_prj23_transparenzregister/config/config_providers.py

@@ -4,6 +4,8 @@ import abc
 import json
 import os
 
+from dotenv import load_dotenv
+
 from aki_prj23_transparenzregister.config.config_template import PostgreConnectionString
 from aki_prj23_transparenzregister.utils.mongo.connector import MongoConnection
 
@@ -89,3 +91,48 @@ class JsonFileConfigProvider(ConfigProvider):
             details["username"],
             details["password"],
         )
+
+
+class EnvironmentConfigProvider(ConfigProvider):
+    """Config provider based on .json file."""
+
+    __data__: dict = {}
+
+    def __init__(self, prefix: str = "PYTHON_"):
+        """Reads secrets from local environment while also ingesting .env files if available.
+
+        Args:
+            prefix (str, optional): Variable prefix. Defaults to "PYTHON_".
+        """
+        load_dotenv()
+        relevant_keys = [key for key in os.environ if key.startswith(prefix)]
+        for key in relevant_keys:
+            self.__data__[key.replace(prefix, "")] = os.environ.get(key)
+
+    def get_postgre_connection_string(self) -> PostgreConnectionString:
+        """Read PostgreSQL connection string from environment variables.
+
+        Returns:
+            PostgreConnectionString: Connection details
+        """
+        return PostgreConnectionString(
+            self.__data__["POSTGRES_USERNAME"],
+            self.__data__["POSTGRES_PASSWORD"],
+            self.__data__["POSTGRES_HOST"],
+            self.__data__["POSTGRES_DATABASE"],
+            self.__data__["POSTGRES_PORT"],
+        )
+
+    def get_mongo_connection_string(self) -> MongoConnection:
+        """Read MongodB connection string from environment variables.
+
+        Returns:
+            MongoConnection: Connection details
+        """
+        return MongoConnection(
+            self.__data__["MONGO_HOST"],
+            self.__data__["MONGO_DATABASE"],
+            self.__data__["MONGO_PORT"],
+            self.__data__["MONGO_USERNAME"],
+            self.__data__["MONGO_PASSWORD"],
+        )

tests/config/config_providers_test.py

@@ -3,7 +3,10 @@ from unittest.mock import mock_open, patch
 
 import pytest
 
-from aki_prj23_transparenzregister.config.config_providers import JsonFileConfigProvider
+from aki_prj23_transparenzregister.config.config_providers import (
+    EnvironmentConfigProvider,
+    JsonFileConfigProvider,
+)
 
 
 def test_json_provider_init_fail() -> None:
@@ -72,3 +75,49 @@ def test_json_provider_get_mongo() -> None:
             assert config.hostname == data["mongo"]["host"]
             assert config.database == data["mongo"]["database"]
             assert config.port == data["mongo"]["port"]
+
+
+def test_env_provider_constructor() -> None:
+    with patch("aki_prj23_transparenzregister.config.config_providers.os") as mock_os:
+        keys = {"PYTHON_TEST": "test", "NOT_PYTHON_TEST": ""}
+        mock_os.environ = keys
+        provider = EnvironmentConfigProvider()
+        assert provider.__data__ == {"TEST": "test"}
+
+
+def test_env_provider_postgres() -> None:
+    provider = EnvironmentConfigProvider()
+    env_data = {
+        "POSTGRES_USERNAME": "postgres",
+        "POSTGRES_PASSWORD": "postgres",
+        "POSTGRES_HOST": "localhost",
+        "POSTGRES_DATABASE": "postgres",
+        "POSTGRES_PORT": "5432",
+    }
+    provider.__data__ = env_data
+    conn_string = provider.get_postgre_connection_string()
+
+    assert conn_string.database == env_data["POSTGRES_DATABASE"]
+    assert conn_string.host == env_data["POSTGRES_HOST"]
+    assert conn_string.password == env_data["POSTGRES_PASSWORD"]
+    assert conn_string.port == env_data["POSTGRES_PORT"]
+    assert conn_string.username == env_data["POSTGRES_USERNAME"]
+
+
+def test_env_provider_mongodb() -> None:
+    provider = EnvironmentConfigProvider()
+    env_data = {
+        "MONGO_USERNAME": "username",
+        "MONGO_HOST": "localhost",
+        "MONGO_PASSWORD": "password",
+        "MONGO_PORT": 27017,
+        "MONGO_DATABASE": "transparenzregister",
+    }
+    provider.__data__ = env_data
+    conn_string = provider.get_mongo_connection_string()
+
+    assert conn_string.database == env_data["MONGO_DATABASE"]
+    assert conn_string.hostname == env_data["MONGO_HOST"]
+    assert conn_string.password == env_data["MONGO_PASSWORD"]
+    assert conn_string.port == env_data["MONGO_PORT"]
+    assert conn_string.username == env_data["MONGO_USERNAME"]