Ingest schedule (#391)

Includes a new "app" running the ingestion jobs (aka fetch_news and
find_missing_companies + enrich_company_financials) on a schedule.

This also fixes an issue with the previous schedule implementation by
persisting the schedule in a file that survives new deployment and
continues where it left off.
This commit is contained in:
Tristan Nolde
2023-12-04 19:05:52 +01:00
committed by GitHub
parent 011e169383
commit 92d14b5824
10 changed files with 280 additions and 82 deletions

View File

@ -17,6 +17,9 @@ services:
PYTHON_MONGO_PORT: ${PYTHON_MONGO_PORT:-27017}
PYTHON_MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister}
PYTHON_INGEST_SCHEDULE: ${PYTHON_INGEST_SCHEDULE:-4}
PYTHON_INGEST_SCHEDULE_FILE: ${PYTHON_INGEST_SCHEDULE_FILE:-/data/ingest_schedule.json}
volumes:
- ingest_data:/data
mongodb:
image: mongo:4.4.6
@ -91,6 +94,7 @@ services:
volumes:
postgres_data:
mongo_data:
ingest_data:
networks:
default: