diff --git a/.gitignore b/.gitignore index 5371c9d..a5206be 100644 --- a/.gitignore +++ b/.gitignore @@ -217,3 +217,4 @@ replay_pid* /unit-test-results.xml /lbr-audit.md /.ruff_cache/ +/Jupyter/test.ipynb diff --git a/README.md b/README.md index b9585eb..1f11c25 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,13 @@ See the [CONTRIBUTING.md](CONTRIBUTING.md) about how code should be formatted and what kind of rules we set ourselves. +## Available entrypoints + +The project has currently the following entrypoint available: + +- data-transfer > Transfers all the data from the mongodb into the sql db to make it available as production data. +- reset-sql > Resets all sql tables in the connected db. + ## DB Connection settings To connect to the SQL db see [sql/connector.py](./src/aki_prj23_transparenzregister/utils/sql/connector.py) @@ -20,7 +27,7 @@ To connect to the Mongo db see [connect] Create a `secrets.json` in the root of this repo with the following structure (values to be replaces by desired config): ```json - { +{ "postgres": { "username": "postgres", "password": "postgres", diff --git a/src/aki_prj23_transparenzregister/utils/data_transfer.py b/src/aki_prj23_transparenzregister/utils/data_transfer.py index 2d4f129..1ca6ed0 100644 --- a/src/aki_prj23_transparenzregister/utils/data_transfer.py +++ b/src/aki_prj23_transparenzregister/utils/data_transfer.py @@ -209,6 +209,9 @@ def add_company(company: dict[str, Any], db: Session) -> None: raise DataInvalidError( "The company name needs to be valid (not empty and not only whitespace)." ) + last_update: date | None = ( + date.fromisoformat(company["last_update"]) if company["last_update"] else None + ) company_entry = entities.Company( court_id=court_id, hr=company["id"]["hr_number"].strip().replace(" ", " ").replace(" ", " "), @@ -216,7 +219,7 @@ def add_company(company: dict[str, Any], db: Session) -> None: city=simplify_string(location.get("city")), zip_code=simplify_string(location.get("zip_code")), street=simplify_string(location.get("street")), - last_update=company["last_update"], + last_update=last_update, ) db.add(company_entry) db.commit() @@ -335,7 +338,12 @@ def transfer_data(db: Session | None) -> None: if db is None: db = get_session(JsonFileConfigProvider("./secrets.json")) logger.remove() - logger.add(sys.stdout, level="INFO") + logger.add( + sys.stdout, + level="INFO", + catch=True, + format="{time:YYYY-MM-DD HH:mm:ss} {level} {message}", + ) logger.add("data-transfer.log", level="INFO", retention=5) reset_all_tables(db) @@ -352,4 +360,4 @@ def transfer_data(db: Session | None) -> None: if __name__ == "__main__": - transfer_data(get_session(JsonFileConfigProvider("./secrets.json"))) + transfer_data(get_session("sqlite:///local-test-data.db")) diff --git a/tests/utils/data_transfer_test.py b/tests/utils/data_transfer_test.py index f2deb8d..351cf83 100644 --- a/tests/utils/data_transfer_test.py +++ b/tests/utils/data_transfer_test.py @@ -262,7 +262,7 @@ def company_generator(seed: int) -> dict[str, Any]: "zip_code": get_random_zip() if random.choice([True, False]) else None, "street": get_random_string(20) if random.choice([True, False]) else None, }, - "last_update": date(random.randint(2000, 2023), 1, 1), + "last_update": date(random.randint(2000, 2023), 1, 1).isoformat(), }