Philipp/aki_prj23_transparenzregister
1
0
Fork 0
mirror of https://github.com/fhswf/aki_prj23_transparenzregister.git synced 2025-06-21 15:13:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint(db): Refactor mongo utils, extract postgres entities from Juptyer

Browse Source
This commit is contained in:
TrisNol
2023-08-11 15:12:18 +02:00
parent d493fd0978
commit d565770b99
19 changed files with 866 additions and 441 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
Jupyter/mongoDB/01_How_to_connect_to_MongoDB_StagingDB.ipynb
View File

@ -8,7 +8,7 @@
"outputs": [],
"source": [
"import configuration as conf\n",
"import aki_prj23_transparenzregister.utils.mongo as mongo"
"import aki_prj23_transparenzregister.utils.mongo.connector as mongo"
]
},
{
@ -38,8 +38,10 @@
"metadata": {},
"outputs": [],
"source": [
"#Create Connection-Object\n",
"MongoObject=mongo.MongoConnection(conf.HOSTNAME,conf.DATABASE,conf.PORT,conf.USERNAME,conf.PASSWORD)"
"# Create Connection-Object\n",
"MongoObject = mongo.MongoConnection(\n",
" conf.HOSTNAME, conf.DATABASE, conf.PORT, conf.USERNAME, conf.PASSWORD\n",
")"
]
},
{
@ -59,8 +61,8 @@
"metadata": {},
"outputs": [],
"source": [
"#create Connection\n",
"MongoConnect=mongo.MongoConnector(MongoObject)"
"# create Connection\n",
"MongoConnect = mongo.MongoConnector(MongoObject)"
]
},
{
@ -79,8 +81,8 @@
"metadata": {},
"outputs": [],
"source": [
"#create Object for news-collection\n",
"CollNews=MongoConnect.database[\"news\"]"
"# create Object for news-collection\n",
"CollNews = MongoConnect.database[\"news\"]"
]
},
{
@ -104,8 +106,8 @@
"metadata": {},
"outputs": [],
"source": [
"cursor=CollNews.find({'date':'2023-06-16T13:29:18.901+02:00'})\n",
"NewsList=list(cursor)"
"cursor = CollNews.find({\"date\": \"2023-06-16T13:29:18.901+02:00\"})\n",
"NewsList = list(cursor)"
]
},
{
@ -169,9 +171,9 @@
"metadata": {},
"outputs": [],
"source": [
"#get all documents from collection\n",
"cursor=CollNews.find({})\n",
"NewsList=list(cursor)"
"# get all documents from collection\n",
"cursor = CollNews.find({})\n",
"NewsList = list(cursor)"
]
},
{
@ -4936,7 +4938,7 @@
}
],
"source": [
"#loop trough all documents\n",
"# loop trough all documents\n",
"for post in CollNews.find():\n",
" print(post)"
]

42
Jupyter/mongoDB/02_How_to_write_to_MongoDB_StagingDB.ipynb
View File

@ -7,8 +7,8 @@
"metadata": {},
"outputs": [],
"source": [
"import configurationFH as conf #hier wird die Configuration für das K8 cluster verwendet\n",
"import aki_prj23_transparenzregister.utils.mongo as mongo"
"import configurationFH as conf # hier wird die Configuration für das K8 cluster verwendet\n",
"import aki_prj23_transparenzregister.utils.mongo.connector as mongo"
]
},
{
@ -38,8 +38,10 @@
"metadata": {},
"outputs": [],
"source": [
"#Create Connection-Object\n",
"MongoObject=mongo.MongoConnection(conf.HOSTNAME,conf.DATABASE,conf.PORT,conf.USERNAME,conf.PASSWORD)"
"# Create Connection-Object\n",
"MongoObject = mongo.MongoConnection(\n",
" conf.HOSTNAME, conf.DATABASE, conf.PORT, conf.USERNAME, conf.PASSWORD\n",
")"
]
},
{
@ -59,8 +61,8 @@
"metadata": {},
"outputs": [],
"source": [
"#create Connection\n",
"MongoConnect=mongo.MongoConnector(MongoObject)"
"# create Connection\n",
"MongoConnect = mongo.MongoConnector(MongoObject)"
]
},
{
@ -79,8 +81,8 @@
"metadata": {},
"outputs": [],
"source": [
"#create Object for news-collection\n",
"CollNews=MongoConnect.database[\"news\"]"
"# create Object for news-collection\n",
"CollNews = MongoConnect.database[\"news\"]"
]
},
{
@ -104,9 +106,9 @@
"metadata": {},
"outputs": [],
"source": [
"#get all documents from collection\n",
"cursor=CollNews.find({})\n",
"NewsList=list(cursor)"
"# get all documents from collection\n",
"cursor = CollNews.find({})\n",
"NewsList = list(cursor)"
]
},
{
@ -190,11 +192,11 @@
"import json\n",
"\n",
"# select the news article\n",
"data=NewsList[212]\n",
"jsonID=data['_id']\n",
"data = NewsList[212]\n",
"jsonID = data[\"_id\"]\n",
"\n",
"#add the new attribute \n",
"data['company'] = '[Deutsche Bahn, GDL]'\n",
"# add the new attribute\n",
"data[\"company\"] = \"[Deutsche Bahn, GDL]\"\n",
"\n",
"print(data)"
]
@ -241,8 +243,8 @@
"CollNews.update_one(\n",
" {\"_id\": data[\"_id\"]},\n",
" {\"$set\": data},\n",
" upsert=True # Setze upsert=True, um das Dokument einzufügen, falls es noch nicht existiert\n",
")\n"
" upsert=True, # Setze upsert=True, um das Dokument einzufügen, falls es noch nicht existiert\n",
")"
]
},
{
@ -252,9 +254,9 @@
"metadata": {},
"outputs": [],
"source": [
"#get all documents from collection\n",
"cursor=CollNews.find({})\n",
"NewsList=list(cursor)"
"# get all documents from collection\n",
"cursor = CollNews.find({})\n",
"NewsList = list(cursor)"
]
},
{