Philipp/aki_prj23_transparenzregister
1
0
Fork 0
mirror of https://github.com/fhswf/aki_prj23_transparenzregister.git synced 2025-06-21 11:13:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

build: Include Mongo dump with init in local deployment (#448)

Browse Source 
Added the latest data from the internet-facing staging DB as json files
to the repo (Git LFS) and added another container to the local
deployment that ingests the files upon creation.

---------

Co-authored-by: Philipp Horstenkamp <philipp@horstenkamp.de>
This commit is contained in:
Tristan Nolde
2023-12-10 11:43:15 +01:00
committed by GitHub
parent 043501a784
commit c917cac58a
6 changed files with 125 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitattributes vendored Normal file
View File

@ -0,0 +1 @@
assets/*.json filter=lfs diff=lfs merge=lfs -text

3
.gitignore vendored
View File

@ -2,6 +2,9 @@
**/*.xml **/*.xml
**/*.json **/*.json
# Keep assets
!assets/**/*
# LaTeX temp files # LaTeX temp files
**/*.aux **/*.aux
**/*-blx.bib **/*-blx.bib

3
assets/mongo_import.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/sh
mongoimport --host ${MONGO_HOST}:${MONGO_PORT} -u ${MONGO_USERNAME} -p ${MONGO_PASSWORD} --authenticationDatabase admin --db ${MONGO_DATABASE} --collection companies --file /tmp/transparenzregister.companies.json --jsonArray
mongoimport --host ${MONGO_HOST}:${MONGO_PORT} -u ${MONGO_USERNAME} -p ${MONGO_PASSWORD} --authenticationDatabase admin --db ${MONGO_DATABASE} --collection news --file /tmp/transparenzregister.news.json --jsonArray

BIN
assets/transparenzregister.companies.json (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
assets/transparenzregister.news.json (Stored with Git LFS) Normal file
View File

Binary file not shown.

218
local-docker-compose.yml
View File

@ -1,106 +1,112 @@
version: '3.8' version: '3.8'
name: local-aki_prj23_transparenzregister name: local-aki_prj23_transparenzregister
services: services:
ingest: ingest:
image: ingestor:local-build image: ingestor:local-build
build: build:
context: . context: .
target: ingest target: ingest
environment: environment:
# mongo access # mongo access
PYTHON_MONGO_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error} PYTHON_MONGO_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error}
PYTHON_MONGO_USERNAME: ${PYTHON_MONGO_USERNAME:-root} PYTHON_MONGO_USERNAME: ${PYTHON_MONGO_USERNAME:-root}
PYTHON_MONGO_HOST: ${PYTHON_MONGO_HOST:-mongodb} PYTHON_MONGO_HOST: ${PYTHON_MONGO_HOST:-mongodb}
PYTHON_MONGO_PORT: ${PYTHON_MONGO_PORT:-27017} PYTHON_MONGO_PORT: ${PYTHON_MONGO_PORT:-27017}
PYTHON_MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister} PYTHON_MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister}
PYTHON_INGEST_SCHEDULE: ${PYTHON_INGEST_SCHEDULE:-4} PYTHON_INGEST_SCHEDULE: ${PYTHON_INGEST_SCHEDULE:-4}
PYTHON_INGEST_SCHEDULE_FILE: ${PYTHON_INGEST_SCHEDULE_FILE:-/data/ingest_schedule.json} PYTHON_INGEST_SCHEDULE_FILE: ${PYTHON_INGEST_SCHEDULE_FILE:-/data/ingest_schedule.json}
volumes: volumes:
- ingest_data:/data - ingest_data:/data
mongodb: mongodb:
image: mongo:4.4.6 image: mongo:4.4.6
restart: unless-stopped restart: unless-stopped
environment: environment:
MONGO_INITDB_ROOT_USERNAME: ${PYTHON_MONGO_USERNAME:-root} MONGO_INITDB_ROOT_USERNAME: ${PYTHON_MONGO_USERNAME:-root}
MONGO_INITDB_ROOT_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error} MONGO_INITDB_ROOT_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error}
MONGO_INITDB_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister} MONGO_INITDB_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister}
ports: ports:
- ${PYTHON_MONGO_PORT}:27017 - ${PYTHON_MONGO_PORT}:27017
volumes: volumes:
- mongo_data:/data/db - mongo_data:/data/db
deploy: deploy:
replicas: 1 # set to one to activate replicas: 1 # set to one to activate
postgres: mongodb-init:
image: postgres:15.4 image: mongo:4.4.6
restart: always depends_on:
environment: - mongodb
POSTGRES_USER: ${PYTHON_POSTGRES_USERNAME:-postgres} restart: no
POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error} volumes:
POSTGRES_DB: ${PYTHON_POSTGRES_DATABASE:-db} - ./assets:/tmp
volumes: environment:
- postgres_data:/var/lib/postgresql/data MONGO_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error}
ports: MONGO_USERNAME: ${PYTHON_MONGO_USERNAME:-root}
- ${PYTHON_POSTGRES_PORT:-5432}:5432 MONGO_HOST: ${PYTHON_MONGO_HOST:-mongodb}
deploy: MONGO_PORT: ${PYTHON_MONGO_PORT:-27017}
replicas: 1 # set to one to activate MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister}
entrypoint: [/tmp/mongo_import.sh]
data-transformation:
image: data-transformation:local-build postgres:
depends_on: image: postgres:15.4
- postgres restart: always
- mongodb environment:
build: POSTGRES_USER: ${PYTHON_POSTGRES_USERNAME:-postgres}
context: . POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error}
target: data-transformation POSTGRES_DB: ${PYTHON_POSTGRES_DATABASE:-db}
environment: volumes:
# postgres access - postgres_data:/var/lib/postgresql/data
PYTHON_POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error} ports:
PYTHON_POSTGRES_USERNAME: ${PYTHON_POSTGRES_USERNAME:-postgres} - ${PYTHON_POSTGRES_PORT:-5432}:5432
PYTHON_POSTGRES_HOST: ${PYTHON_POSTGRES_HOST:-postgres} deploy:
PYTHON_POSTGRES_DATABASE: ${PYTHON_POSTGRES_DATABASE:-db} replicas: 1 # set to one to activate
PYTHON_POSTGRES_PORT: ${PYTHON_POSTGRES_PORT:-5432}
# mongo access data-transformation:
PYTHON_MONGO_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error} image: data-transformation:local-build
PYTHON_MONGO_USERNAME: ${PYTHON_MONGO_USERNAME:-root} depends_on:
PYTHON_MONGO_HOST: ${PYTHON_MONGO_HOST:-mongodb} - postgres
PYTHON_MONGO_PORT: ${PYTHON_MONGO_PORT:-27017} - mongodb
PYTHON_MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister} build:
context: .
web-server: target: data-transformation
image: web-server:local-build environment:
build: # postgres access
context: . PYTHON_POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error}
target: web-server PYTHON_POSTGRES_USERNAME: ${PYTHON_POSTGRES_USERNAME:-postgres}
depends_on: PYTHON_POSTGRES_HOST: ${PYTHON_POSTGRES_HOST:-postgres}
- postgres PYTHON_POSTGRES_DATABASE: ${PYTHON_POSTGRES_DATABASE:-db}
ports: PYTHON_POSTGRES_PORT: ${PYTHON_POSTGRES_PORT:-5432}
- 8888:8888 # mongo access
environment: PYTHON_MONGO_PASSWORD: ${PYTHON_MONGO_PASSWORD:?error}
PYTHON_POSTGRES_USERNAME: ${PYTHON_POSTGRES_USERNAME:-postgres} PYTHON_MONGO_USERNAME: ${PYTHON_MONGO_USERNAME:-root}
PYTHON_POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error} PYTHON_MONGO_HOST: ${PYTHON_MONGO_HOST:-mongodb}
PYTHON_POSTGRES_HOST: ${PYTHON_POSTGRES_HOST:-postgres} PYTHON_MONGO_PORT: ${PYTHON_MONGO_PORT:-27017}
PYTHON_POSTGRES_DATABASE: ${PYTHON_POSTGRES_DATABASE:-db} PYTHON_MONGO_DATABASE: ${PYTHON_MONGO_DATABASE:-transparenzregister}
PYTHON_POSTGRES_PORT: ${PYTHON_POSTGRES_PORT:-5432}
PYTHON_DASH_LOGIN_PW: ${PYTHON_DASH_LOGIN_PW} web-server:
PYTHON_DASH_LOGIN_USERNAME: ${PYTHON_DASH_LOGIN_USERNAME} image: web-server:local-build
PORT: 8888 build:
HOST: 0.0.0.0 context: .
target: web-server
volumes: depends_on:
postgres_data: - postgres
mongo_data: ports:
ingest_data: - 8888:8888
environment:
networks: PYTHON_POSTGRES_USERNAME: ${PYTHON_POSTGRES_USERNAME:-postgres}
default: PYTHON_POSTGRES_PASSWORD: ${PYTHON_POSTGRES_PASSWORD:?error}
driver: bridge PYTHON_POSTGRES_HOST: ${PYTHON_POSTGRES_HOST:-postgres}
ipam: PYTHON_POSTGRES_DATABASE: ${PYTHON_POSTGRES_DATABASE:-db}
driver: default PYTHON_POSTGRES_PORT: ${PYTHON_POSTGRES_PORT:-5432}
config: PYTHON_DASH_LOGIN_PW: ${PYTHON_DASH_LOGIN_PW}
- subnet: 192.168.7.0/24 PYTHON_DASH_LOGIN_USERNAME: ${PYTHON_DASH_LOGIN_USERNAME}
gateway: 192.168.7.1 PORT: 8888
HOST: 0.0.0.0
volumes:
postgres_data:
mongo_data:
ingest_data: