mirror of
https://github.com/fhswf/aki_prj23_transparenzregister.git
synced 2025-06-21 19:33:55 +02:00
Ingest schedule (#391)
Includes a new "app" running the ingestion jobs (aka fetch_news and find_missing_companies + enrich_company_financials) on a schedule. This also fixes an issue with the previous schedule implementation by persisting the schedule in a file that survives new deployment and continues where it left off.
This commit is contained in:
@ -62,9 +62,7 @@ def test_schedule(
|
||||
mock_handelsblatt_rss.return_value = Mock(
|
||||
get_news_for_category=Mock(return_value=mock_news_handelsblatt)
|
||||
)
|
||||
assert fetch_news.schedule(Mock()) == len(
|
||||
mock_news_handelsblatt + mock_news_tagesschau
|
||||
)
|
||||
assert fetch_news.main(Mock()) == len(mock_news_handelsblatt + mock_news_tagesschau)
|
||||
|
||||
|
||||
@patch("aki_prj23_transparenzregister.apps.fetch_news.MongoNewsService")
|
||||
@ -90,4 +88,4 @@ def test_schedule_error(
|
||||
mock_handelsblatt_rss.return_value = Mock(
|
||||
get_news_for_category=Mock(return_value=mock_news_handelsblatt)
|
||||
)
|
||||
assert fetch_news.schedule(Mock()) == 0
|
||||
assert fetch_news.main(Mock()) == 0
|
||||
|
41
tests/apps/ingest_test.py
Normal file
41
tests/apps/ingest_test.py
Normal file
@ -0,0 +1,41 @@
|
||||
"""Testing apps/ingest.py."""
|
||||
import json
|
||||
import tempfile
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from aki_prj23_transparenzregister.apps import ingest
|
||||
|
||||
|
||||
def test_import() -> None:
|
||||
assert ingest
|
||||
|
||||
|
||||
def test_load_schedule() -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
data = {"test": "test"}
|
||||
path = f"{temp_dir}/schedule.json"
|
||||
with open(path, "w") as file:
|
||||
json.dump({"test": "test"}, file)
|
||||
assert ingest.load_schedule(path) == data
|
||||
|
||||
|
||||
def test_load_scheduler_no_result() -> None:
|
||||
assert ingest.load_schedule("./hello_there.json") == {}
|
||||
|
||||
|
||||
def test_save_schedule() -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
data = {"test": "test"}
|
||||
path = f"{temp_dir}/schedule.json"
|
||||
|
||||
ingest.save_schedule(data, path)
|
||||
with open(path) as file:
|
||||
assert json.load(file) == data
|
||||
|
||||
|
||||
@patch("aki_prj23_transparenzregister.apps.ingest.find_missing_companies_main")
|
||||
@patch("aki_prj23_transparenzregister.apps.ingest.enrich_company_financials_main")
|
||||
def test_main(mock_financials: Mock, mock_find_missing: Mock) -> None:
|
||||
ingest.main(Mock())
|
||||
assert mock_financials.called
|
||||
assert mock_find_missing.called
|
Reference in New Issue
Block a user