Ingest schedule (#391)

Includes a new "app" running the ingestion jobs (aka fetch_news and
find_missing_companies + enrich_company_financials) on a schedule.

This also fixes an issue with the previous schedule implementation by
persisting the schedule in a file that survives new deployment and
continues where it left off.
This commit is contained in:
Tristan Nolde
2023-12-04 19:05:52 +01:00
committed by GitHub
parent 011e169383
commit 92d14b5824
10 changed files with 280 additions and 82 deletions

View File

@ -62,9 +62,7 @@ def test_schedule(
mock_handelsblatt_rss.return_value = Mock(
get_news_for_category=Mock(return_value=mock_news_handelsblatt)
)
assert fetch_news.schedule(Mock()) == len(
mock_news_handelsblatt + mock_news_tagesschau
)
assert fetch_news.main(Mock()) == len(mock_news_handelsblatt + mock_news_tagesschau)
@patch("aki_prj23_transparenzregister.apps.fetch_news.MongoNewsService")
@ -90,4 +88,4 @@ def test_schedule_error(
mock_handelsblatt_rss.return_value = Mock(
get_news_for_category=Mock(return_value=mock_news_handelsblatt)
)
assert fetch_news.schedule(Mock()) == 0
assert fetch_news.main(Mock()) == 0

41
tests/apps/ingest_test.py Normal file
View File

@ -0,0 +1,41 @@
"""Testing apps/ingest.py."""
import json
import tempfile
from unittest.mock import Mock, patch
from aki_prj23_transparenzregister.apps import ingest
def test_import() -> None:
assert ingest
def test_load_schedule() -> None:
with tempfile.TemporaryDirectory() as temp_dir:
data = {"test": "test"}
path = f"{temp_dir}/schedule.json"
with open(path, "w") as file:
json.dump({"test": "test"}, file)
assert ingest.load_schedule(path) == data
def test_load_scheduler_no_result() -> None:
assert ingest.load_schedule("./hello_there.json") == {}
def test_save_schedule() -> None:
with tempfile.TemporaryDirectory() as temp_dir:
data = {"test": "test"}
path = f"{temp_dir}/schedule.json"
ingest.save_schedule(data, path)
with open(path) as file:
assert json.load(file) == data
@patch("aki_prj23_transparenzregister.apps.ingest.find_missing_companies_main")
@patch("aki_prj23_transparenzregister.apps.ingest.enrich_company_financials_main")
def test_main(mock_financials: Mock, mock_find_missing: Mock) -> None:
ingest.main(Mock())
assert mock_financials.called
assert mock_find_missing.called