diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2a7b81d --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +assets/*.json filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 70d36a0..8701434 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ **/*.xml **/*.json +# Keep assets +!assets/**/* + # LaTeX temp files **/*.aux **/*-blx.bib diff --git a/assets/mongo_import.sh b/assets/mongo_import.sh new file mode 100644 index 0000000..456c529 --- /dev/null +++ b/assets/mongo_import.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mongoimport --host ${MONGO_HOST}:${MONGO_PORT} -u ${MONGO_USERNAME} -p ${MONGO_PASSWORD} --authenticationDatabase admin --db ${MONGO_DATABASE} --collection companies --file /tmp/transparenzregister.companies.json --jsonArray +mongoimport --host ${MONGO_HOST}:${MONGO_PORT} -u ${MONGO_USERNAME} -p ${MONGO_PASSWORD} --authenticationDatabase admin --db ${MONGO_DATABASE} --collection news --file /tmp/transparenzregister.news.json --jsonArray diff --git a/assets/transparenzregister.companies.json b/assets/transparenzregister.companies.json new file mode 100644 index 0000000..c4a283b --- /dev/null +++ b/assets/transparenzregister.companies.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1525e684041f3a724d8e24a223d075d1793b69620eb84fc7689a085c2213b883 +size 11861626 diff --git a/assets/transparenzregister.news.json b/assets/transparenzregister.news.json new file mode 100644 index 0000000..6ba6531 --- /dev/null +++ b/assets/transparenzregister.news.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5a5c896684311b6b54a0f09c3e944c508ac18e838b0892956ce013fa9fc6561 +size 38692112 diff --git a/local-docker-compose.yml b/local-docker-compose.yml index ed3d5ec..9b1aa50 100644 --- a/local-docker-compose.yml +++ b/local-docker-compose.yml @@ -1,106 +1,112 @@ -version: '3.8' - -name: local-aki_prj23_transparenzregister - -services: - - ingest: - image: ingestor:local-build - build: - context: . - target: ingest - environment: - # mongo access - PYTHON_MONGO_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error} - PYTHON_MONGO_USERNAME: ${PYTHON_MONGO_USERNAME:-root} - PYTHON_MONGO_HOST: ${PYTHON_MONGO_HOST:-mongodb} - PYTHON_MONGO_PORT: ${PYTHON_MONGO_PORT:-27017} - PYTHON_MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister} - PYTHON_INGEST_SCHEDULE: ${PYTHON_INGEST_SCHEDULE:-4} - PYTHON_INGEST_SCHEDULE_FILE: ${PYTHON_INGEST_SCHEDULE_FILE:-/data/ingest_schedule.json} - volumes: - - ingest_data:/data - - mongodb: - image: mongo:4.4.6 - restart: unless-stopped - environment: - MONGO_INITDB_ROOT_USERNAME: ${PYTHON_MONGO_USERNAME:-root} - MONGO_INITDB_ROOT_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error} - MONGO_INITDB_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister} - ports: - - ${PYTHON_MONGO_PORT}:27017 - volumes: - - mongo_data:/data/db - deploy: - replicas: 1 # set to one to activate - - postgres: - image: postgres:15.4 - restart: always - environment: - POSTGRES_USER: ${PYTHON_POSTGRES_USERNAME:-postgres} - POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error} - POSTGRES_DB: ${PYTHON_POSTGRES_DATABASE:-db} - volumes: - - postgres_data:/var/lib/postgresql/data - ports: - - ${PYTHON_POSTGRES_PORT:-5432}:5432 - deploy: - replicas: 1 # set to one to activate - - data-transformation: - image: data-transformation:local-build - depends_on: - - postgres - - mongodb - build: - context: . - target: data-transformation - environment: - # postgres access - PYTHON_POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error} - PYTHON_POSTGRES_USERNAME: ${PYTHON_POSTGRES_USERNAME:-postgres} - PYTHON_POSTGRES_HOST: ${PYTHON_POSTGRES_HOST:-postgres} - PYTHON_POSTGRES_DATABASE: ${PYTHON_POSTGRES_DATABASE:-db} - PYTHON_POSTGRES_PORT: ${PYTHON_POSTGRES_PORT:-5432} - # mongo access - PYTHON_MONGO_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error} - PYTHON_MONGO_USERNAME: ${PYTHON_MONGO_USERNAME:-root} - PYTHON_MONGO_HOST: ${PYTHON_MONGO_HOST:-mongodb} - PYTHON_MONGO_PORT: ${PYTHON_MONGO_PORT:-27017} - PYTHON_MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister} - - web-server: - image: web-server:local-build - build: - context: . - target: web-server - depends_on: - - postgres - ports: - - 8888:8888 - environment: - PYTHON_POSTGRES_USERNAME: ${PYTHON_POSTGRES_USERNAME:-postgres} - PYTHON_POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error} - PYTHON_POSTGRES_HOST: ${PYTHON_POSTGRES_HOST:-postgres} - PYTHON_POSTGRES_DATABASE: ${PYTHON_POSTGRES_DATABASE:-db} - PYTHON_POSTGRES_PORT: ${PYTHON_POSTGRES_PORT:-5432} - PYTHON_DASH_LOGIN_PW: ${PYTHON_DASH_LOGIN_PW} - PYTHON_DASH_LOGIN_USERNAME: ${PYTHON_DASH_LOGIN_USERNAME} - PORT: 8888 - HOST: 0.0.0.0 - -volumes: - postgres_data: - mongo_data: - ingest_data: - -networks: - default: - driver: bridge - ipam: - driver: default - config: - - subnet: 192.168.7.0/24 - gateway: 192.168.7.1 +version: '3.8' + +name: local-aki_prj23_transparenzregister + +services: + + ingest: + image: ingestor:local-build + build: + context: . + target: ingest + environment: + # mongo access + PYTHON_MONGO_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error} + PYTHON_MONGO_USERNAME: ${PYTHON_MONGO_USERNAME:-root} + PYTHON_MONGO_HOST: ${PYTHON_MONGO_HOST:-mongodb} + PYTHON_MONGO_PORT: ${PYTHON_MONGO_PORT:-27017} + PYTHON_MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister} + PYTHON_INGEST_SCHEDULE: ${PYTHON_INGEST_SCHEDULE:-4} + PYTHON_INGEST_SCHEDULE_FILE: ${PYTHON_INGEST_SCHEDULE_FILE:-/data/ingest_schedule.json} + volumes: + - ingest_data:/data + + mongodb: + image: mongo:4.4.6 + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: ${PYTHON_MONGO_USERNAME:-root} + MONGO_INITDB_ROOT_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error} + MONGO_INITDB_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister} + ports: + - ${PYTHON_MONGO_PORT}:27017 + volumes: + - mongo_data:/data/db + deploy: + replicas: 1 # set to one to activate + + mongodb-init: + image: mongo:4.4.6 + depends_on: + - mongodb + restart: no + volumes: + - ./assets:/tmp + environment: + MONGO_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error} + MONGO_USERNAME: ${PYTHON_MONGO_USERNAME:-root} + MONGO_HOST: ${PYTHON_MONGO_HOST:-mongodb} + MONGO_PORT: ${PYTHON_MONGO_PORT:-27017} + MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister} + entrypoint: [/tmp/mongo_import.sh] + + postgres: + image: postgres:15.4 + restart: always + environment: + POSTGRES_USER: ${PYTHON_POSTGRES_USERNAME:-postgres} + POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error} + POSTGRES_DB: ${PYTHON_POSTGRES_DATABASE:-db} + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - ${PYTHON_POSTGRES_PORT:-5432}:5432 + deploy: + replicas: 1 # set to one to activate + + data-transformation: + image: data-transformation:local-build + depends_on: + - postgres + - mongodb + build: + context: . + target: data-transformation + environment: + # postgres access + PYTHON_POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error} + PYTHON_POSTGRES_USERNAME: ${PYTHON_POSTGRES_USERNAME:-postgres} + PYTHON_POSTGRES_HOST: ${PYTHON_POSTGRES_HOST:-postgres} + PYTHON_POSTGRES_DATABASE: ${PYTHON_POSTGRES_DATABASE:-db} + PYTHON_POSTGRES_PORT: ${PYTHON_POSTGRES_PORT:-5432} + # mongo access + PYTHON_MONGO_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error} + PYTHON_MONGO_USERNAME: ${PYTHON_MONGO_USERNAME:-root} + PYTHON_MONGO_HOST: ${PYTHON_MONGO_HOST:-mongodb} + PYTHON_MONGO_PORT: ${PYTHON_MONGO_PORT:-27017} + PYTHON_MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister} + + web-server: + image: web-server:local-build + build: + context: . + target: web-server + depends_on: + - postgres + ports: + - 8888:8888 + environment: + PYTHON_POSTGRES_USERNAME: ${PYTHON_POSTGRES_USERNAME:-postgres} + PYTHON_POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error} + PYTHON_POSTGRES_HOST: ${PYTHON_POSTGRES_HOST:-postgres} + PYTHON_POSTGRES_DATABASE: ${PYTHON_POSTGRES_DATABASE:-db} + PYTHON_POSTGRES_PORT: ${PYTHON_POSTGRES_PORT:-5432} + PYTHON_DASH_LOGIN_PW: ${PYTHON_DASH_LOGIN_PW} + PYTHON_DASH_LOGIN_USERNAME: ${PYTHON_DASH_LOGIN_USERNAME} + PORT: 8888 + HOST: 0.0.0.0 + +volumes: + postgres_data: + mongo_data: + ingest_data: