Philipp/aki_prj23_transparenzregister
1
0
Fork 0
mirror of https://github.com/fhswf/aki_prj23_transparenzregister.git synced 2025-06-22 20:43:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

Executing black over all jupyter notebook (#190)

Browse Source 
Reverting black for the jupyter notebooks gets old. Can we just run
black over all of them?
This commit is contained in:
Philipp Horstenkamp
2023-10-04 20:03:47 +02:00
committed by GitHub
parent 030ad00c7d
commit 41f2c9f995
15 changed files with 658 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

393
documentations/seminararbeiten/Datenspeicherung/Jupyter/11_Create_Tables_with_SQL-Alchemy.ipynb
View File

@ -44,7 +44,7 @@
" username=\"postgres\",\n",
" password=\"postgres\",\n",
" host=\"localhost\",\n",
" database=\"postgres\"\n",
" database=\"postgres\",\n",
")\n",
"\n",
"engine = create_engine(url)"
@ -57,7 +57,7 @@
"metadata": {},
"outputs": [],
"source": [
"#connect to database\n",
"# connect to database\n",
"connection = engine.connect()"
]
},
@ -77,12 +77,13 @@
"metadata": {},
"outputs": [],
"source": [
"#create an object *district_court* which inherits attributes from Base-class\n",
"# create an object *district_court* which inherits attributes from Base-class\n",
"Base = declarative_base()\n",
"\n",
"\n",
"class DistrictCourt(Base):\n",
" __tablename__ = 'district_court'\n",
" \n",
" __tablename__ = \"district_court\"\n",
"\n",
" id = Column(Integer(), primary_key=True)\n",
" city = Column(String(100), nullable=False)\n",
" name = Column(String(100), nullable=False)"
@ -106,10 +107,12 @@
"outputs": [],
"source": [
"class Company(Base):\n",
" __tablename__ = 'company'\n",
" __tablename__ = \"company\"\n",
"\n",
" hr = Column(Integer(), nullable=False, primary_key=True)\n",
" court_id = Column(Integer, ForeignKey(\"district_court.id\"), nullable=False, primary_key=True)\n",
" court_id = Column(\n",
" Integer, ForeignKey(\"district_court.id\"), nullable=False, primary_key=True\n",
" )\n",
" name = Column(String(100), nullable=False)\n",
" street = Column(String(100), nullable=False)\n",
" zip = Column(Integer(), nullable=False)\n",
@ -117,7 +120,7 @@
" sector = Column(String(100), nullable=False)\n",
"\n",
" __table_args__ = (\n",
" PrimaryKeyConstraint('hr', 'court_id', name='pk_company_hr_court'),\n",
" PrimaryKeyConstraint(\"hr\", \"court_id\", name=\"pk_company_hr_court\"),\n",
" )"
]
},
@ -139,7 +142,7 @@
}
],
"source": [
"#check if table-object is created\n",
"# check if table-object is created\n",
"Company.__table__"
]
},
@ -151,7 +154,7 @@
"outputs": [],
"source": [
"class Finance(Base):\n",
" __tablename__ = 'finance'\n",
" __tablename__ = \"finance\"\n",
"\n",
" id = Column(Integer, primary_key=True)\n",
" company_hr = Column(Integer)\n",
@ -170,7 +173,9 @@
" company = relationship(\"Company\")\n",
"\n",
" __table_args__ = (\n",
" ForeignKeyConstraint([company_hr, company_court], [Company.hr, Company.court_id]),\n",
" ForeignKeyConstraint(\n",
" [company_hr, company_court], [Company.hr, Company.court_id]\n",
" ),\n",
" )"
]
},
@ -181,25 +186,35 @@
"metadata": {},
"outputs": [],
"source": [
"#create own enumeration type and sentiment object\n",
"sentiment_type=Enum(\"employee_voting\",\"sustainability\",\"environmental_aspects\",\"perception\", name=\"sentiment_type\", create_type=False)\n",
"# create own enumeration type and sentiment object\n",
"sentiment_type = Enum(\n",
" \"employee_voting\",\n",
" \"sustainability\",\n",
" \"environmental_aspects\",\n",
" \"perception\",\n",
" name=\"sentiment_type\",\n",
" create_type=False,\n",
")\n",
"\n",
"\n",
"class Sentiment(Base):\n",
" __tablename__ = 'sentiment'\n",
" __tablename__ = \"sentiment\"\n",
"\n",
" id = Column(Integer(), primary_key=True)\n",
" #company_hr = mapped_column(ForeignKey(\"company.hr\"))\n",
" #company_court = mapped_column(ForeignKey(\"company.court_id\"))\n",
" # company_hr = mapped_column(ForeignKey(\"company.hr\"))\n",
" # company_court = mapped_column(ForeignKey(\"company.court_id\"))\n",
" company_hr = Column(Integer)\n",
" company_court = Column(Integer)\n",
" date = Column(DateTime(), default=datetime.now)\n",
" type = Column(sentiment_type,nullable=False)\n",
" value =Column(Float(),nullable=False)\n",
" source=Column(String(100))\n",
" \n",
" sentiment = relationship('Company')\n",
" type = Column(sentiment_type, nullable=False)\n",
" value = Column(Float(), nullable=False)\n",
" source = Column(String(100))\n",
"\n",
" sentiment = relationship(\"Company\")\n",
" __table_args__ = (\n",
" ForeignKeyConstraint([company_hr, company_court], [Company.hr, Company.court_id]),\n",
" ForeignKeyConstraint(\n",
" [company_hr, company_court], [Company.hr, Company.court_id]\n",
" ),\n",
" )"
]
},
@ -210,14 +225,14 @@
"metadata": {},
"outputs": [],
"source": [
"#create person object\n",
"# create person object\n",
"class Person(Base):\n",
" __tablename__ = 'person'\n",
" __tablename__ = \"person\"\n",
"\n",
" id = Column(Integer(), primary_key=True)\n",
" name=Column(String(100), nullable=False)\n",
" surname=Column(String(100), nullable=False)\n",
" works_for=Column(String(100))"
" name = Column(String(100), nullable=False)\n",
" surname = Column(String(100), nullable=False)\n",
" works_for = Column(String(100))"
]
},
{
@ -227,27 +242,39 @@
"metadata": {},
"outputs": [],
"source": [
"#create own relation type and person_relation object\n",
"rel_type=Enum(\"Executive\",\"Auditor\",\"Supervisory_Board\",\"Managing_Director\",\"Authorized_Representive\",\"Final_Auditor\", name=\"rel_type\", create_type=False)\n",
"# create own relation type and person_relation object\n",
"rel_type = Enum(\n",
" \"Executive\",\n",
" \"Auditor\",\n",
" \"Supervisory_Board\",\n",
" \"Managing_Director\",\n",
" \"Authorized_Representive\",\n",
" \"Final_Auditor\",\n",
" name=\"rel_type\",\n",
" create_type=False,\n",
")\n",
"\n",
"\n",
"class Person_Relation(Base):\n",
" __tablename__ = 'person_relation'\n",
" __tablename__ = \"person_relation\"\n",
"\n",
" id = Column(Integer(), primary_key=True)\n",
" #company_hr = mapped_column(ForeignKey(\"company.hr\"))\n",
" #company_court = mapped_column(ForeignKey(\"company.court_id\"))\n",
" # company_hr = mapped_column(ForeignKey(\"company.hr\"))\n",
" # company_court = mapped_column(ForeignKey(\"company.court_id\"))\n",
" company_hr = Column(Integer)\n",
" company_court = Column(Integer)\n",
" person_id = mapped_column(ForeignKey(\"person.id\"))\n",
" date_from = Column(DateTime(), default=datetime.now)\n",
" date_to = Column(DateTime(), default=datetime.now) \n",
" relation=Column(rel_type, nullable=False)\n",
" \n",
" #company = relationship(\"Company\")\n",
" #person = relationship(\"Person\", foreign_keys=[person_id])\n",
" #company = relationship('Company', foreign_keys=[company_hr,company_court])\n",
" date_to = Column(DateTime(), default=datetime.now)\n",
" relation = Column(rel_type, nullable=False)\n",
"\n",
" # company = relationship(\"Company\")\n",
" # person = relationship(\"Person\", foreign_keys=[person_id])\n",
" # company = relationship('Company', foreign_keys=[company_hr,company_court])\n",
" __table_args__ = (\n",
" ForeignKeyConstraint([company_hr, company_court], [Company.hr, Company.court_id]),\n",
" ForeignKeyConstraint(\n",
" [company_hr, company_court], [Company.hr, Company.court_id]\n",
" ),\n",
" )"
]
},
@ -258,22 +285,30 @@
"metadata": {},
"outputs": [],
"source": [
"#create own relation type and company_relation object\n",
"rel_type_comp=Enum(\"participates_with\",\"has_shares_of\",\"is_supplied_by\",\"works_with\", name=\"rel_type_comp\", create_type=False)\n",
"# create own relation type and company_relation object\n",
"rel_type_comp = Enum(\n",
" \"participates_with\",\n",
" \"has_shares_of\",\n",
" \"is_supplied_by\",\n",
" \"works_with\",\n",
" name=\"rel_type_comp\",\n",
" create_type=False,\n",
")\n",
"\n",
"\n",
"class Company_Relation(Base):\n",
" __tablename__ = 'company_relation'\n",
" __tablename__ = \"company_relation\"\n",
"\n",
" id = Column(Integer(), primary_key=True)\n",
" company1_id = Column(Integer,nullable=False)\n",
" company2_id= Column(Integer,nullable=False)\n",
" company1_id = Column(Integer, nullable=False)\n",
" company2_id = Column(Integer, nullable=False)\n",
" date_from = Column(DateTime(), default=datetime.now)\n",
" date_to = Column(DateTime(), default=datetime.now) \n",
" relation=Column(rel_type_comp, nullable=False)\n",
" \n",
" #company = relationship(\"Company\")\n",
" date_to = Column(DateTime(), default=datetime.now)\n",
" relation = Column(rel_type_comp, nullable=False)\n",
"\n",
" __table_args__ = {'extend_existing': True}"
" # company = relationship(\"Company\")\n",
"\n",
" __table_args__ = {\"extend_existing\": True}"
]
},
{
@ -320,7 +355,7 @@
"metadata": {},
"outputs": [],
"source": [
"df=pd.read_csv('Amtsgerichte.csv', sep=';') \n"
"df = pd.read_csv(\"Amtsgerichte.csv\", sep=\";\")"
]
},
{
@ -331,11 +366,9 @@
"outputs": [],
"source": [
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" court=DistrictCourt( \n",
" city = str(df['Stadt'].iloc[i]),\n",
" name = str(df['Name'].iloc[i]))\n",
" \n",
" # get data from dataframe\n",
" court = DistrictCourt(city=str(df[\"Stadt\"].iloc[i]), name=str(df[\"Name\"].iloc[i]))\n",
"\n",
" session.add(court)\n",
" session.commit()"
]
@ -355,7 +388,7 @@
"metadata": {},
"outputs": [],
"source": [
"df=pd.read_csv('01_Stammdaten_Unternehmen_HR2.csv', sep=';',encoding=\"ISO-8859-1\")"
"df = pd.read_csv(\"01_Stammdaten_Unternehmen_HR2.csv\", sep=\";\", encoding=\"ISO-8859-1\")"
]
},
{
@ -406,15 +439,16 @@
],
"source": [
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" comp=Company( \n",
" hr= int(df['HR'].iloc[i]),\n",
" court_id= int(df['Amtsgericht'].iloc[i]),\n",
" name = str(df['Name'].iloc[i]),\n",
" street = str(df['Strasse'].iloc[i]),\n",
" zip = int(df['PLZ'].iloc[i]),\n",
" city = str(df['Stadt'].iloc[i]),\n",
" sector=str(df['Branche'].iloc[i]))\n",
" # get data from dataframe\n",
" comp = Company(\n",
" hr=int(df[\"HR\"].iloc[i]),\n",
" court_id=int(df[\"Amtsgericht\"].iloc[i]),\n",
" name=str(df[\"Name\"].iloc[i]),\n",
" street=str(df[\"Strasse\"].iloc[i]),\n",
" zip=int(df[\"PLZ\"].iloc[i]),\n",
" city=str(df[\"Stadt\"].iloc[i]),\n",
" sector=str(df[\"Branche\"].iloc[i]),\n",
" )\n",
" session.add(comp)\n",
" session.commit()"
]
@ -445,7 +479,7 @@
}
],
"source": [
"df=pd.read_csv('BASF_Data.csv', sep=';', decimal=\",\",encoding=\"ISO-8859-1\") \n",
"df = pd.read_csv(\"BASF_Data.csv\", sep=\";\", decimal=\",\", encoding=\"ISO-8859-1\")\n",
"df.columns"
]
},
@ -459,21 +493,22 @@
"from datetime import datetime\n",
"\n",
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" fin=Finance( \n",
" company_hr = int(df['Company_HR'].iloc[i]),\n",
" company_court = int(df['Company_Court'].iloc[i]),\n",
" date=datetime.strptime(str(df['Jahr'].iloc[i]), '%Y'),\n",
" total_volume=str(df['Umsatz'].iloc[i]),\n",
" ebit=str(df['Ebit'].iloc[i]) ,\n",
" ebitda=str(df['EBITDA'].iloc[i]),\n",
" ebit_margin=null(),\n",
" total_balance=null(),\n",
" equity=null(),\n",
" debt=null(),\n",
" return_on_equity=null(),\n",
" capital_turnover_rate=null())\n",
" \n",
" # get data from dataframe\n",
" fin = Finance(\n",
" company_hr=int(df[\"Company_HR\"].iloc[i]),\n",
" company_court=int(df[\"Company_Court\"].iloc[i]),\n",
" date=datetime.strptime(str(df[\"Jahr\"].iloc[i]), \"%Y\"),\n",
" total_volume=str(df[\"Umsatz\"].iloc[i]),\n",
" ebit=str(df[\"Ebit\"].iloc[i]),\n",
" ebitda=str(df[\"EBITDA\"].iloc[i]),\n",
" ebit_margin=null(),\n",
" total_balance=null(),\n",
" equity=null(),\n",
" debt=null(),\n",
" return_on_equity=null(),\n",
" capital_turnover_rate=null(),\n",
" )\n",
"\n",
" session.add(fin)\n",
" session.commit()"
]
@ -504,7 +539,7 @@
}
],
"source": [
"df=pd.read_csv('Telekom_Data.csv', sep=';',decimal=',',encoding=\"ISO-8859-1\") \n",
"df = pd.read_csv(\"Telekom_Data.csv\", sep=\";\", decimal=\",\", encoding=\"ISO-8859-1\")\n",
"df.columns"
]
},
@ -516,21 +551,22 @@
"outputs": [],
"source": [
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" fin=Finance( \n",
" company_hr = int(df['Company_HR'].iloc[i]),\n",
" company_court = int(df['Company_Court'].iloc[i]), \n",
" date=datetime.strptime(str(df['Jahr'].iloc[i]), '%Y'),\n",
" total_volume=str(df['Umsatz'].iloc[i]),\n",
" ebit=str(df['Ebit'].iloc[i]) ,\n",
" ebitda=str(df['EBITDA'].iloc[i]),\n",
" ebit_margin=null(),\n",
" total_balance=null(),\n",
" equity=null(),\n",
" debt=null(),\n",
" return_on_equity=null(),\n",
" capital_turnover_rate=null())\n",
" \n",
" # get data from dataframe\n",
" fin = Finance(\n",
" company_hr=int(df[\"Company_HR\"].iloc[i]),\n",
" company_court=int(df[\"Company_Court\"].iloc[i]),\n",
" date=datetime.strptime(str(df[\"Jahr\"].iloc[i]), \"%Y\"),\n",
" total_volume=str(df[\"Umsatz\"].iloc[i]),\n",
" ebit=str(df[\"Ebit\"].iloc[i]),\n",
" ebitda=str(df[\"EBITDA\"].iloc[i]),\n",
" ebit_margin=null(),\n",
" total_balance=null(),\n",
" equity=null(),\n",
" debt=null(),\n",
" return_on_equity=null(),\n",
" capital_turnover_rate=null(),\n",
" )\n",
"\n",
" session.add(fin)\n",
" session.commit()"
]
@ -561,7 +597,7 @@
}
],
"source": [
"df=pd.read_csv('EON_Data.csv', sep=';',decimal=',',encoding=\"ISO-8859-1\") \n",
"df = pd.read_csv(\"EON_Data.csv\", sep=\";\", decimal=\",\", encoding=\"ISO-8859-1\")\n",
"df.columns"
]
},
@ -575,21 +611,22 @@
"outputs": [],
"source": [
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" fin=Finance( \n",
" company_hr = int(df['Company_HR'].iloc[i]),\n",
" company_court = int(df['Company_Court'].iloc[i]),\n",
" date=datetime.strptime(str(df['Jahr'].iloc[i]), '%Y'),\n",
" total_volume=str(df['Umsatz'].iloc[i]),\n",
" ebit=str(df['Ebit'].iloc[i]) ,\n",
" ebitda=str(df['EBITDA'].iloc[i]),\n",
" ebit_margin=null(),\n",
" total_balance=null(),\n",
" equity=null(),\n",
" debt=null(),\n",
" return_on_equity=null(),\n",
" capital_turnover_rate=null())\n",
" \n",
" # get data from dataframe\n",
" fin = Finance(\n",
" company_hr=int(df[\"Company_HR\"].iloc[i]),\n",
" company_court=int(df[\"Company_Court\"].iloc[i]),\n",
" date=datetime.strptime(str(df[\"Jahr\"].iloc[i]), \"%Y\"),\n",
" total_volume=str(df[\"Umsatz\"].iloc[i]),\n",
" ebit=str(df[\"Ebit\"].iloc[i]),\n",
" ebitda=str(df[\"EBITDA\"].iloc[i]),\n",
" ebit_margin=null(),\n",
" total_balance=null(),\n",
" equity=null(),\n",
" debt=null(),\n",
" return_on_equity=null(),\n",
" capital_turnover_rate=null(),\n",
" )\n",
"\n",
" session.add(fin)\n",
" session.commit()"
]
@ -628,7 +665,7 @@
}
],
"source": [
"df=pd.read_csv('person1000.csv', sep=';',decimal=',',encoding=\"ISO-8859-1\") \n",
"df = pd.read_csv(\"person1000.csv\", sep=\";\", decimal=\",\", encoding=\"ISO-8859-1\")\n",
"df.columns"
]
},
@ -640,12 +677,9 @@
"outputs": [],
"source": [
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" per=Person( \n",
" name = str(df['Name'].iloc[i]),\n",
" surname = str(df['Surname'].iloc[i])\n",
")\n",
" \n",
" # get data from dataframe\n",
" per = Person(name=str(df[\"Name\"].iloc[i]), surname=str(df[\"Surname\"].iloc[i]))\n",
"\n",
" session.add(per)\n",
" session.commit()"
]
@ -667,53 +701,70 @@
"source": [
"import random\n",
"\n",
"relation=['Executive',\n",
"'Auditor',\n",
"'Supervisory_Board',\n",
"'Managing_Director',\n",
"'Authorized_Representive',\n",
"'Final_Auditor'\n",
"relation = [\n",
" \"Executive\",\n",
" \"Auditor\",\n",
" \"Supervisory_Board\",\n",
" \"Managing_Director\",\n",
" \"Authorized_Representive\",\n",
" \"Final_Auditor\",\n",
"]\n",
"\n",
"hr_court=[\n",
"(12334,2),\n",
"(64566,2),\n",
"(5433,3),\n",
"(12334,4),\n",
"(12336,5),\n",
"(555,6),\n",
"(555,7),\n",
"(12384,8),\n",
"(64345,9),\n",
"(4344,1),\n",
"(866,1),\n",
"(9875,1)\n",
"hr_court = [\n",
" (12334, 2),\n",
" (64566, 2),\n",
" (5433, 3),\n",
" (12334, 4),\n",
" (12336, 5),\n",
" (555, 6),\n",
" (555, 7),\n",
" (12384, 8),\n",
" (64345, 9),\n",
" (4344, 1),\n",
" (866, 1),\n",
" (9875, 1),\n",
"]\n",
"\n",
"edges=[]\n",
"edges = []\n",
"\n",
"#create amount of combinations\n",
"# create amount of combinations\n",
"for i in range(2000):\n",
" rand_comp=random.randint(0,11)\n",
" comp_hr=hr_court[rand_comp][0]\n",
" comp_court=hr_court[rand_comp][1]\n",
" \n",
" rand_person=random.randint(1,999)\n",
" rand_relation=random.randint(0,5)\n",
" rand_year_start=random.randint(2005,2023)\n",
" if rand_year_start<2023:\n",
" year_to=rand_year_start+1\n",
" rand_comp = random.randint(0, 11)\n",
" comp_hr = hr_court[rand_comp][0]\n",
" comp_court = hr_court[rand_comp][1]\n",
"\n",
" rand_person = random.randint(1, 999)\n",
" rand_relation = random.randint(0, 5)\n",
" rand_year_start = random.randint(2005, 2023)\n",
" if rand_year_start < 2023:\n",
" year_to = rand_year_start + 1\n",
" else:\n",
" pass\n",
" # year_to=None\n",
" \n",
" #edges.append((rand_company,df['Name'].iloc[rand_person],rand_year_start,year_to,relation[rand_relation])) \n",
" edges.append((comp_hr,comp_court,rand_person,int(rand_year_start),year_to,relation[rand_relation])) \n",
" \n",
"#edges.to_csv('edges.csv')\n",
"col=['Company_HR','Company_Court','Person_ID','Year_From','Year_To','Relation_Type']\n",
"dfEdges=pd.DataFrame(edges,columns=col)\n",
"dfEdges.to_csv('edges.csv')"
"\n",
" # edges.append((rand_company,df['Name'].iloc[rand_person],rand_year_start,year_to,relation[rand_relation]))\n",
" edges.append(\n",
" (\n",
" comp_hr,\n",
" comp_court,\n",
" rand_person,\n",
" int(rand_year_start),\n",
" year_to,\n",
" relation[rand_relation],\n",
" )\n",
" )\n",
"\n",
"# edges.to_csv('edges.csv')\n",
"col = [\n",
" \"Company_HR\",\n",
" \"Company_Court\",\n",
" \"Person_ID\",\n",
" \"Year_From\",\n",
" \"Year_To\",\n",
" \"Relation_Type\",\n",
"]\n",
"dfEdges = pd.DataFrame(edges, columns=col)\n",
"dfEdges.to_csv(\"edges.csv\")"
]
},
{
@ -912,7 +963,7 @@
}
],
"source": [
"df=pd.read_csv('edges.csv', sep=',') \n",
"df = pd.read_csv(\"edges.csv\", sep=\",\")\n",
"df"
]
},
@ -924,16 +975,16 @@
"outputs": [],
"source": [
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" edges=Person_Relation( \n",
" company_hr = int(df['Company_HR'].iloc[i]),\n",
" company_court = int(df['Company_Court'].iloc[i]),\n",
" person_id = int(df['Person_ID'].iloc[i]),\n",
" date_from=datetime.strptime(str(df['Year_From'].iloc[i]), '%Y'),\n",
" date_to=datetime.strptime(str(df['Year_To'].iloc[i]), '%Y'),\n",
" relation = str(df['Relation_Type'].iloc[i])\n",
")\n",
" \n",
" # get data from dataframe\n",
" edges = Person_Relation(\n",
" company_hr=int(df[\"Company_HR\"].iloc[i]),\n",
" company_court=int(df[\"Company_Court\"].iloc[i]),\n",
" person_id=int(df[\"Person_ID\"].iloc[i]),\n",
" date_from=datetime.strptime(str(df[\"Year_From\"].iloc[i]), \"%Y\"),\n",
" date_to=datetime.strptime(str(df[\"Year_To\"].iloc[i]), \"%Y\"),\n",
" relation=str(df[\"Relation_Type\"].iloc[i]),\n",
" )\n",
"\n",
" session.add(edges)\n",
" session.commit()"
]
@ -1084,7 +1135,7 @@
"\n",
"print(\"total_volume|ebit|ebitda\")\n",
"for n in Fin:\n",
" print(n.total_volume, n.ebit, n.ebitda)\n"
" print(n.total_volume, n.ebit, n.ebitda)"
]
},
{
@ -1094,7 +1145,7 @@
"metadata": {},
"outputs": [],
"source": [
"users = session.query(User).filter(User.name == 'John').all()"
"users = session.query(User).filter(User.name == \"John\").all()"
]
},
{

46
documentations/seminararbeiten/Datenspeicherung/Jupyter/Notebooks_with_SQL_and_preliminary_data/01_Connect_to_Database.ipynb
View File

@ -7,7 +7,7 @@
"metadata": {},
"outputs": [],
"source": [
"#pip install psycopg2"
"# pip install psycopg2"
]
},
{
@ -17,7 +17,7 @@
"metadata": {},
"outputs": [],
"source": [
"#pip install --user ipython-sql"
"# pip install --user ipython-sql"
]
},
{
@ -55,10 +55,8 @@
],
"source": [
"conn = psycopg2.connect(\n",
" host=\"localhost\",\n",
" database=\"transparenz\",\n",
" user=\"postgres\",\n",
" password=\"postgres\")\n",
" host=\"localhost\", database=\"transparenz\", user=\"postgres\", password=\"postgres\"\n",
")\n",
"\n",
"print(\"Database connected successfully\")"
]
@ -89,11 +87,12 @@
],
"source": [
"conn.autocommit = True\n",
"#create a table\n",
"# create a table\n",
"cur = conn.cursor() # creating a cursor\n",
" \n",
"\n",
"# executing queries to create table\n",
"cur.execute(\"\"\"\n",
"cur.execute(\n",
" \"\"\"\n",
"CREATE TABLE company\n",
"(\n",
" ID SERIAL PRIMARY KEY NOT NULL,\n",
@ -102,12 +101,13 @@
" ZIP INT NOT NULL,\n",
" CITY VARCHAR(100) NOT NULL,\n",
" SECTOR VARCHAR(200) NOT NULL)\n",
"\"\"\")\n",
" \n",
"\"\"\"\n",
")\n",
"\n",
"# commit the changes\n",
"conn.commit() # <--- makes sure the change is shown in the database\n",
"#conn.close()\n",
"#cur.close()\n",
"conn.commit() # <--- makes sure the change is shown in the database\n",
"# conn.close()\n",
"# cur.close()\n",
"print(\"Table Created successfully\")"
]
},
@ -135,11 +135,12 @@
],
"source": [
"conn.autocommit = True\n",
"#create a table\n",
"# create a table\n",
"cur = conn.cursor() # creating a cursor\n",
" \n",
"\n",
"# executing queries to create table\n",
"cur.execute(\"\"\"\n",
"cur.execute(\n",
" \"\"\"\n",
"CREATE TABLE finance\n",
"(\n",
" FINANCE_ID SERIAL PRIMARY KEY NOT NULL,\n",
@ -147,12 +148,13 @@
" KIND_OF VARCHAR(50) NOT NULL,\n",
" DATE DATE NOT NULL,\n",
" SUM FLOAT NOT NULL)\n",
"\"\"\")\n",
" \n",
"\"\"\"\n",
")\n",
"\n",
"# commit the changes\n",
"conn.commit() # <--- makes sure the change is shown in the database\n",
"#conn.close()\n",
"#cur.close()\n",
"conn.commit() # <--- makes sure the change is shown in the database\n",
"# conn.close()\n",
"# cur.close()\n",
"print(\"Table Created successfully\")"
]
},

33
documentations/seminararbeiten/Datenspeicherung/Jupyter/Notebooks_with_SQL_and_preliminary_data/02_Connect_to_Database_publish_Company_Data.ipynb
View File

@ -10,6 +10,7 @@
"import numpy as np\n",
"import pandas as pd\n",
"import ipywidgets as widgets\n",
"\n",
"pd.options.plotting.backend = \"plotly\""
]
},
@ -20,7 +21,7 @@
"metadata": {},
"outputs": [],
"source": [
"df=pd.read_csv('01_Stammdaten_Unternehmen.csv', sep=';') "
"df = pd.read_csv(\"01_Stammdaten_Unternehmen.csv\", sep=\";\")"
]
},
{
@ -240,10 +241,8 @@
],
"source": [
"conn = psycopg2.connect(\n",
" host=\"localhost\",\n",
" database=\"transparenz\",\n",
" user=\"postgres\",\n",
" password=\"postgres\")\n",
" host=\"localhost\", database=\"transparenz\", user=\"postgres\", password=\"postgres\"\n",
")\n",
"\n",
"print(\"Database connected successfully\")"
]
@ -267,18 +266,18 @@
"\n",
"\n",
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" name=str(df['Name'].iloc[i])\n",
" street=str(df['Straße'].iloc[i])\n",
" zipcode=int(df['PLZ'].iloc[i])\n",
" city=str(df['Stadt'].iloc[i])\n",
" sector=str(df['Branche'].iloc[i])\n",
" \n",
" postgres_insert_query = \"\"\" INSERT INTO company (NAME,STREET, ZIP, CITY,SECTOR) VALUES (%s,%s,%s,%s,%s)\"\"\" \n",
" \n",
" record_to_insert = (name,street,zipcode,city,sector)\n",
" cur.execute(postgres_insert_query, record_to_insert) \n",
" \n",
" # get data from dataframe\n",
" name = str(df[\"Name\"].iloc[i])\n",
" street = str(df[\"Straße\"].iloc[i])\n",
" zipcode = int(df[\"PLZ\"].iloc[i])\n",
" city = str(df[\"Stadt\"].iloc[i])\n",
" sector = str(df[\"Branche\"].iloc[i])\n",
"\n",
" postgres_insert_query = \"\"\" INSERT INTO company (NAME,STREET, ZIP, CITY,SECTOR) VALUES (%s,%s,%s,%s,%s)\"\"\"\n",
"\n",
" record_to_insert = (name, street, zipcode, city, sector)\n",
" cur.execute(postgres_insert_query, record_to_insert)\n",
"\n",
"conn.commit()\n",
"conn.close()"
]

33
documentations/seminararbeiten/Datenspeicherung/Jupyter/Notebooks_with_SQL_and_preliminary_data/03-1_Publish_Finance_Testdata_BASF.ipynb
View File

@ -23,6 +23,7 @@
"import numpy as np\n",
"import pandas as pd\n",
"import ipywidgets as widgets\n",
"\n",
"pd.options.plotting.backend = \"plotly\""
]
},
@ -33,7 +34,7 @@
"metadata": {},
"outputs": [],
"source": [
"df=pd.read_csv('BASF_Data_NewOrder.csv', sep=';', decimal=\",\") "
"df = pd.read_csv(\"BASF_Data_NewOrder.csv\", sep=\";\", decimal=\",\")"
]
},
{
@ -528,10 +529,8 @@
],
"source": [
"conn = psycopg2.connect(\n",
" host=\"localhost\",\n",
" database=\"transparenz\",\n",
" user=\"postgres\",\n",
" password=\"postgres\")\n",
" host=\"localhost\", database=\"transparenz\", user=\"postgres\", password=\"postgres\"\n",
")\n",
"\n",
"print(\"Database connected successfully\")"
]
@ -553,20 +552,22 @@
"source": [
"cur = conn.cursor()\n",
"\n",
"PK_ID=8 #BASF hat den PK 8, deshalb wird dieser manuell hinzugefügt\n",
"PK_ID = 8 # BASF hat den PK 8, deshalb wird dieser manuell hinzugefügt\n",
"\n",
"\n",
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" kind_of=str(df['Metrik'].iloc[i])\n",
" date=str(df['Datum'].iloc[i])\n",
" amount=float(df['Summe [Milliarden €]'].iloc[i])\n",
" \n",
" postgres_insert_query = \"\"\" INSERT INTO finance (company_id,kind_of, date, sum) VALUES (%s,%s,%s,%s)\"\"\" \n",
" record_to_insert = (PK_ID,kind_of,date,amount)\n",
" cur.execute(postgres_insert_query, record_to_insert) \n",
" #print(postgres_insert_query, record_to_insert)\n",
" \n",
" # get data from dataframe\n",
" kind_of = str(df[\"Metrik\"].iloc[i])\n",
" date = str(df[\"Datum\"].iloc[i])\n",
" amount = float(df[\"Summe [Milliarden €]\"].iloc[i])\n",
"\n",
" postgres_insert_query = (\n",
" \"\"\" INSERT INTO finance (company_id,kind_of, date, sum) VALUES (%s,%s,%s,%s)\"\"\"\n",
" )\n",
" record_to_insert = (PK_ID, kind_of, date, amount)\n",
" cur.execute(postgres_insert_query, record_to_insert)\n",
" # print(postgres_insert_query, record_to_insert)\n",
"\n",
"conn.commit()\n",
"conn.close()"
]

33
documentations/seminararbeiten/Datenspeicherung/Jupyter/Notebooks_with_SQL_and_preliminary_data/03-2_Publish-Finance_Testdata_Telekom.ipynb
View File

@ -10,6 +10,7 @@
"import numpy as np\n",
"import pandas as pd\n",
"import ipywidgets as widgets\n",
"\n",
"pd.options.plotting.backend = \"plotly\""
]
},
@ -20,7 +21,7 @@
"metadata": {},
"outputs": [],
"source": [
"df=pd.read_csv('Telekom_Data_NewOrder.csv', sep=';',decimal=',') "
"df = pd.read_csv(\"Telekom_Data_NewOrder.csv\", sep=\";\", decimal=\",\")"
]
},
{
@ -403,10 +404,8 @@
],
"source": [
"conn = psycopg2.connect(\n",
" host=\"localhost\",\n",
" database=\"transparenz\",\n",
" user=\"postgres\",\n",
" password=\"postgres\")\n",
" host=\"localhost\", database=\"transparenz\", user=\"postgres\", password=\"postgres\"\n",
")\n",
"\n",
"print(\"Database connected successfully\")"
]
@ -428,20 +427,22 @@
"source": [
"cur = conn.cursor()\n",
"\n",
"PK_ID=5 #BASF hat den PK 8, deshalb wird dieser manuell hinzugefügt\n",
"PK_ID = 5 # BASF hat den PK 8, deshalb wird dieser manuell hinzugefügt\n",
"\n",
"\n",
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" kind_of=str(df['Metrik'].iloc[i])\n",
" date=str(df['Datum'].iloc[i])\n",
" amount=float(df['Summe [Milliarden €]'].iloc[i])\n",
" \n",
" postgres_insert_query = \"\"\" INSERT INTO finance (company_id,kind_of, date, sum) VALUES (%s,%s,%s,%s)\"\"\" \n",
" record_to_insert = (PK_ID,kind_of,date,amount)\n",
" cur.execute(postgres_insert_query, record_to_insert) \n",
" #print(postgres_insert_query, record_to_insert)\n",
" \n",
" # get data from dataframe\n",
" kind_of = str(df[\"Metrik\"].iloc[i])\n",
" date = str(df[\"Datum\"].iloc[i])\n",
" amount = float(df[\"Summe [Milliarden €]\"].iloc[i])\n",
"\n",
" postgres_insert_query = (\n",
" \"\"\" INSERT INTO finance (company_id,kind_of, date, sum) VALUES (%s,%s,%s,%s)\"\"\"\n",
" )\n",
" record_to_insert = (PK_ID, kind_of, date, amount)\n",
" cur.execute(postgres_insert_query, record_to_insert)\n",
" # print(postgres_insert_query, record_to_insert)\n",
"\n",
"conn.commit()\n",
"conn.close()"
]

33
documentations/seminararbeiten/Datenspeicherung/Jupyter/Notebooks_with_SQL_and_preliminary_data/03-2_Publish_Finance_Testdata_EON.ipynb
View File

@ -10,6 +10,7 @@
"import numpy as np\n",
"import pandas as pd\n",
"import ipywidgets as widgets\n",
"\n",
"pd.options.plotting.backend = \"plotly\""
]
},
@ -20,7 +21,7 @@
"metadata": {},
"outputs": [],
"source": [
"df=pd.read_csv('EON_Data_NewOrder.csv', sep=';',decimal=',') "
"df = pd.read_csv(\"EON_Data_NewOrder.csv\", sep=\";\", decimal=\",\")"
]
},
{
@ -340,10 +341,8 @@
],
"source": [
"conn = psycopg2.connect(\n",
" host=\"localhost\",\n",
" database=\"transparenz\",\n",
" user=\"postgres\",\n",
" password=\"postgres\")\n",
" host=\"localhost\", database=\"transparenz\", user=\"postgres\", password=\"postgres\"\n",
")\n",
"\n",
"print(\"Database connected successfully\")"
]
@ -365,20 +364,22 @@
"source": [
"cur = conn.cursor()\n",
"\n",
"PK_ID=9 #BASF hat den PK 8, deshalb wird dieser manuell hinzugefügt\n",
"PK_ID = 9 # BASF hat den PK 8, deshalb wird dieser manuell hinzugefügt\n",
"\n",
"\n",
"for i in range(len(df)):\n",
" #get data from dataframe\n",
" kind_of=str(df['Metrik'].iloc[i])\n",
" date=str(df['Datum'].iloc[i])\n",
" amount=float(df['Summe [Milliarden €]'].iloc[i])\n",
" \n",
" postgres_insert_query = \"\"\" INSERT INTO finance (company_id,kind_of, date, sum) VALUES (%s,%s,%s,%s)\"\"\" \n",
" record_to_insert = (PK_ID,kind_of,date,amount)\n",
" cur.execute(postgres_insert_query, record_to_insert) \n",
" #print(postgres_insert_query, record_to_insert)\n",
" \n",
" # get data from dataframe\n",
" kind_of = str(df[\"Metrik\"].iloc[i])\n",
" date = str(df[\"Datum\"].iloc[i])\n",
" amount = float(df[\"Summe [Milliarden €]\"].iloc[i])\n",
"\n",
" postgres_insert_query = (\n",
" \"\"\" INSERT INTO finance (company_id,kind_of, date, sum) VALUES (%s,%s,%s,%s)\"\"\"\n",
" )\n",
" record_to_insert = (PK_ID, kind_of, date, amount)\n",
" cur.execute(postgres_insert_query, record_to_insert)\n",
" # print(postgres_insert_query, record_to_insert)\n",
"\n",
"conn.commit()\n",
"conn.close()"
]

3
documentations/seminararbeiten/Datenspeicherung/Jupyter/Notebooks_with_SQL_and_preliminary_data/04_First_Query.ipynb
View File

@ -17,6 +17,7 @@
"import numpy as np\n",
"import pandas as pd\n",
"import ipywidgets as widgets\n",
"\n",
"pd.options.plotting.backend = \"plotly\""
]
},
@ -48,7 +49,7 @@
"metadata": {},
"outputs": [],
"source": [
"#load sql extension\n",
"# load sql extension\n",
"%load_ext sql"
]
},