54 get data from multiple tables postgres (#72)

Create first dash board including header, basic company information and
finance plot

---------

Co-authored-by: Philipp Horstenkamp <philipp@horstenkamp.de>

documentations/meeting-notes/Meeting_2023-08-31.md

@@ -0,0 +1,37 @@
+# Weekly *9*: 17.08.2023
+
+## Teilnehmer
+- Prof. Arinir
+- Tristan Nolde
+- Philipp Horstenkamp
+- Sebastian Zeleny
+- Kim Mesewinkel-Risse (Protokoll)
+
+## Themen
+
+- Rückmeldung von Herrn Gawron bzgl. mehr Ressourcen steht noch aus, ggfs. persönliche Absprache nächste Woche möglich
+
+- Rückfrage von Herrn Arinir bezüglich Aufbau der Software und Architektur
+    - Gerade werden einzelne Funktionen erstellt, Daten werden ungefiltert in die Mongo DB geschrieben, anschließend Bereinigung und Übertragung in die Postgres
+    - Vorstellung aktueller Repo-Struktur durch Tristan, relevanter Code befindet sich im src-Ordner
+
+- Wie kann sichergestellt werden, dass unsere Ziele erreicht werden?
+    - Zeitplan/Meilensteinplan gewünscht
+    - Wann soll was erreicht werden?
+    - Burndown-Diagramm
+    -> Umsetzung durch Team beim Präsenzmeeting am 09.09.2023
+
+- Kurze Vorstellung der bearbeiteten Themen: NER + Sentiment (Sebastian), Finanzdaten (Tristan), UI (Kim), Datentransfer (Philipp)
+
+## Abgeleitete Action Items
+
+| Action Item | Verantwortlicher | Deadline        |
+|-------------|------------------|-----------------|
+| Festlegung Zeitplan für Präsenztreffen | Alle  | 07.09.2023 |
+| Zeitplan bzw. Meilensteinplan | Alle  | nächstes Weekly |
+| Erstellen einer Übersicht aller bestehenden Services | Alle  | nächstes Weekly |
+| Update bzgl. Cluster und Ressourcen | Herr Arinir | nächstes Weekly |
+| Finanzdaten finalisieren | Tristan  | nächstes Weekly |
+| NER/Sentiment Aggregation | Sebastian  | nächstes Weekly |
+| Füllen der UI mit Echtdaten | Kim | nächstes Weekly |
+| Teststruktur für SQL hinzufügen | Philipp  | nächstes Weekly |

poetry.lock

@@ -781,6 +781,24 @@ dev = ["PyYAML (>=5.4.1)", "coloredlogs (>=15.0.1)", "fire (>=0.4.0)"]
 diskcache = ["diskcache (>=5.2.1)", "multiprocess (>=0.70.12)", "psutil (>=5.8.0)"]
 testing = ["beautifulsoup4 (>=4.8.2)", "cryptography (<3.4)", "dash-testing-stub (>=0.0.2)", "lxml (>=4.6.2)", "multiprocess (>=0.70.12)", "percy (>=2.0.2)", "psutil (>=5.8.0)", "pytest (>=6.0.2)", "requests[security] (>=2.21.0)", "selenium (>=3.141.0,<=4.2.0)", "waitress (>=1.4.4)"]
 
+[[package]]
+name = "dash-bootstrap-components"
+version = "1.4.2"
+description = "Bootstrap themed components for use in Plotly Dash"
+category = "main"
+optional = false
+python-versions = ">=3.7, <4"
+files = [
+    {file = "dash-bootstrap-components-1.4.2.tar.gz", hash = "sha256:b7514be30e229a1701db5010a47d275882a94d1efff4c803ac42a9d222ed86e0"},
+    {file = "dash_bootstrap_components-1.4.2-py3-none-any.whl", hash = "sha256:4f59352a2f81cb0c41ae75dd3e0814f64049a4520f935397298e9a093ace727c"},
+]
+
+[package.dependencies]
+dash = ">=2.0.0"
+
+[package.extras]
+pandas = ["numpy", "pandas"]
+
 [[package]]
 name = "dash-core-components"
 version = "2.0.0"
@@ -4802,4 +4820,4 @@ h11 = ">=0.9.0,<1"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.11"
-content-hash = "946f6b81f3072e2a3d93482405f7bbe6f02218a3c2097c1f3f8246363c7e15c2"
+content-hash = "d3b9d0efd28ad07060618d55c49a8ba6d3c716fa5caea7e35e6c5975a243e85e"

pyproject.toml

@@ -37,6 +37,7 @@ version = "0.1.0"
 [tool.poetry.dependencies]
 SQLAlchemy = {version = "^1.4.46", extras = ["mypy"]}
 dash = "^2.11.1"
+dash-bootstrap-components = "^1.4.2"
 loguru = "^0.7.0"
 matplotlib = "^3.7.1"
 plotly = "^5.14.1"

src/aki_prj23_transparenzregister/ui/assets/typography.css

@@ -0,0 +1,3 @@
+body {
+    margin: 0;
+}

src/aki_prj23_transparenzregister/ui/company_finance_dash.py

@@ -0,0 +1,392 @@
+"""Dash."""
+
+import dash_bootstrap_components as dbc
+import pandas as pd
+import plotly.graph_objs as go
+from dash import Dash, Input, Output, callback, dash_table, dcc, html
+from dash.exceptions import PreventUpdate
+from sqlalchemy.engine import Engine
+
+from aki_prj23_transparenzregister.utils.postgres import entities
+from aki_prj23_transparenzregister.utils.postgres.connector import (
+    get_session,
+)
+
+if __name__ == "__main__":
+    session = get_session()
+    query_finance = session.query(
+        entities.AnnualFinanceStatement, entities.Company.name, entities.Company.id
+    ).join(entities.Company)
+
+    query_company = session.query(entities.Company, entities.DistrictCourt.name).join(
+        entities.DistrictCourt
+    )
+    engine = session.bind
+    if not isinstance(engine, Engine):
+        raise TypeError
+
+    finance_df: pd.DataFrame = pd.read_sql(str(query_finance), engine)
+    company_df: pd.DataFrame = pd.read_sql(str(query_company), engine)
+
+    select_company_df = company_df[["company_id", "company_name"]]
+    select_company_dropdown = select_company_df.to_dict("records")
+    options = [
+        {"label": i["company_name"], "value": i["company_id"]}
+        for i in select_company_dropdown
+    ]
+
+    colors = {
+        "light": "#edefef",
+        "lavender-blush": "#f3e8ee",
+        "ash-gray": "#bacdb0",
+        "cambridge-blue": "#729b79",
+        "paynes-gray": "#475b63",
+        "raisin-black": "#2e2c2f",
+    }
+
+    def financials_figure(
+        finance_df: pd.DataFrame, company: str, metric: str
+    ) -> go.Figure:
+        """Creates plotly line chart for a specific company and a metric."""
+        finance_df = finance_df.loc[finance_df["company_name"] == company]
+        # create figure
+        fig_line = go.Figure()
+        # add trace for company 1
+        fig_line.add_trace(
+            go.Scatter(
+                x=finance_df["annual_finance_statement_date"],
+                y=finance_df[metric],
+                name=company,
+                line_color=colors["raisin-black"],
+                marker_color=colors["raisin-black"],
+            )
+        )
+        # set title and labels
+        fig_line.update_layout(
+            title=metric,
+            xaxis_title="Jahr",
+            yaxis_title="in Mio.€",
+            plot_bgcolor=colors["light"],
+        )
+        return fig_line
+
+    tab_style = {
+        "borderBottom": "1px solid #d6d6d6",
+        "padding": "6px",
+        "backgroundColor": "white",
+        "color": colors["paynes-gray"],
+        "fontWeight": "bold",
+    }
+
+    tab_selected_style = {
+        "borderTop": "1px solid #d6d6d6",
+        "borderBottom": "1px solid #d6d6d6",
+        "padding": "6px",
+        "backgroundColor": colors["paynes-gray"],
+        "color": "white",
+        "fontWeight": "bold",
+    }
+
+    # TBD: get data from database instead of mock data
+    company = 1  # selected company id
+    selected_company = company_df.loc[company_df["company_id"] == company]
+
+    turnover = 123456
+    stock = "1,23"
+    company_data = {
+        "col1": ["Unternehmen", "Straße", "Stadt"],
+        "col2": [
+            selected_company["company_name"][0],
+            selected_company["company_street"][0],
+            str(
+                selected_company["company_zip_code"][0]
+                + " "
+                + selected_company["company_city"][0]
+            ),
+        ],
+        "col3": ["Branche", "Amtsgericht", "Gründungsjahr"],
+        "col4": [
+            selected_company["company_sector"][0],
+            selected_company["district_court_name"][0],
+            "xxx",
+        ],
+    }
+    df_company_data = pd.DataFrame(data=company_data)
+
+    app = Dash(
+        __name__, external_stylesheets=[dbc.icons.BOOTSTRAP]
+    )  # use dbc for icons
+
+    kennzahlen_layout = html.Div(
+        [
+            dcc.Graph(
+                figure=financials_figure(
+                    finance_df, str(company), "annual_finance_statement_ebit"
+                )
+            )
+        ]
+    )
+
+    app.layout = html.Div(
+        [
+            # title header of page
+            html.Div(
+                style={
+                    "backgroundColor": colors["raisin-black"],
+                    "border": "1px solid",
+                },
+                children=[
+                    html.I(
+                        className="bi bi-house-door-fill",
+                        style={
+                            "fontSize": 24,
+                            "paddingLeft": "10px",
+                            "color": "white",
+                            "display": "inline-block",
+                            "verticalAlign": "middle",
+                        },
+                    ),
+                    html.H1(
+                        children="Transparenzregister für Kapitalgesellschaften",
+                        style={
+                            "color": "white",
+                            "textAlign": "left",
+                            "margin": "0",
+                            "paddingLeft": "10px",
+                            "paddingBottom": "20px",
+                            "paddingTop": "20px",
+                            "display": "inline-block",
+                            "verticalAlign": "middle",
+                        },
+                    ),
+                    html.Div(
+                        dcc.Dropdown(
+                            id="select_company",
+                            placeholder="Suche nach Unternehmen oder Person",
+                        ),
+                        style={
+                            "float": "right",
+                            "width": "30%",
+                            "margin": "0",
+                            "paddingRight": "10px",
+                            "paddingBottom": "20px",
+                            "paddingTop": "20px",
+                            "display": "inline-block",
+                            "verticalAlign": "middle",
+                        },
+                    ),
+                ],
+            ),
+            # header company name
+            html.Div(
+                style={"backgroundColor": colors["paynes-gray"], "border": "1px solid"},
+                children=[
+                    html.H1(
+                        children=selected_company["company_name"][0],
+                        style={
+                            "color": "white",
+                            "fontSize": 30,
+                            "textAlign": "left",
+                            "margin": "0",
+                            "paddingLeft": "20px",
+                            "paddingBottom": "20px",
+                            "paddingTop": "20px",
+                        },
+                    )
+                ],
+            ),
+            html.Div(style={"height": "20px"}),
+            html.Div(style={"width": "2%", "display": "inline-block"}),
+            # table basic company information
+            html.Div(
+                style={
+                    "backgroundColor": colors["ash-gray"],
+                    "border": "1px solid",
+                    "border-radius": 10,
+                    "width": "45%",
+                    "height": "150px",
+                    "display": "inline-block",
+                    "vertical-align": "top",
+                },
+                children=[
+                    html.H5(
+                        children="Stammdaten",
+                        style={
+                            "color": colors["raisin-black"],
+                            "fontSize": 16,
+                            "textAlign": "center",
+                            "margin": "0",
+                            "paddingBottom": "10px",
+                            "paddingTop": "10px",
+                        },
+                    ),
+                    dash_table.DataTable(
+                        df_company_data.to_dict("records"),
+                        [{"name": i, "id": i} for i in df_company_data.columns],
+                        style_table={
+                            "width": "80%",
+                            "overflowX": "auto",
+                            "marginLeft": "auto",
+                            "marginRight": "auto",
+                            "paddingBottom": "20px",
+                            "color": colors["raisin-black"],
+                        },
+                        # hide header of table
+                        css=[
+                            {
+                                "selector": "tr:first-child",
+                                "rule": "display: none",
+                            },
+                        ],
+                        style_cell={"textAlign": "center"},
+                        style_cell_conditional=[
+                            {"if": {"column_id": c}, "fontWeight": "bold"}
+                            for c in ["col1", "col3"]
+                        ],
+                    ),
+                ],
+            ),
+            html.Div(style={"width": "2%", "display": "inline-block"}),
+            html.Div(
+                style={
+                    "backgroundColor": colors["ash-gray"],
+                    "border": "1px solid",
+                    "border-radius": 10,
+                    "width": "15%",
+                    "height": "150px",
+                    "display": "inline-block",
+                    "vertical-align": "top",
+                },
+                children=[
+                    html.H5(
+                        children="Stimmung",
+                        style={
+                            "color": colors["raisin-black"],
+                            "fontSize": 16,
+                            "textAlign": "center",
+                            "margin": "0",
+                            "paddingBottom": "10px",
+                            "paddingTop": "10px",
+                        },
+                    ),
+                ],
+            ),
+            html.Div(style={"width": "2%", "display": "inline-block"}),
+            html.Div(
+                style={
+                    "backgroundColor": colors["ash-gray"],
+                    "border": "1px solid",
+                    "border-radius": 10,
+                    "width": "15%",
+                    "height": "150px",
+                    "display": "inline-block",
+                    "vertical-align": "top",
+                },
+                children=[
+                    html.H5(
+                        children="Aktienkurs",
+                        style={
+                            "color": colors["raisin-black"],
+                            "fontSize": 16,
+                            "textAlign": "center",
+                            "margin": "0",
+                            "paddingBottom": "10px",
+                            "paddingTop": "10px",
+                        },
+                    ),
+                    html.H1(
+                        children=stock,
+                        style={
+                            "color": colors["raisin-black"],
+                            "textAlign": "center",
+                        },
+                    ),
+                ],
+            ),
+            html.Div(style={"width": "2%", "display": "inline-block"}),
+            html.Div(
+                style={
+                    "backgroundColor": colors["ash-gray"],
+                    "border": "1px solid",
+                    "border-radius": 10,
+                    "width": "15%",
+                    "height": "150px",
+                    "display": "inline-block",
+                    "vertical-align": "top",
+                },
+                children=[
+                    html.H5(
+                        children="Umsatz",
+                        style={
+                            "color": colors["raisin-black"],
+                            "fontSize": 16,
+                            "textAlign": "center",
+                            "margin": "0",
+                            "paddingBottom": "10px",
+                            "paddingTop": "10px",
+                        },
+                    ),
+                    html.H1(
+                        children=turnover,
+                        style={
+                            "color": colors["raisin-black"],
+                            "textAlign": "center",
+                        },
+                    ),
+                ],
+            ),
+            html.Div(style={"width": "2%", "display": "inline-block"}),
+            # ]),
+            html.Div(
+                style={
+                    "marginTop": "20px",
+                    "border": "1px solid",
+                },
+                children=[
+                    dcc.Tabs(
+                        id="tabs",
+                        value="tab-1",
+                        children=[
+                            dcc.Tab(
+                                label="Kennzahlen",
+                                value="tab-1",
+                                style=tab_style,
+                                selected_style=tab_selected_style,
+                                children=[kennzahlen_layout],
+                            ),
+                            dcc.Tab(
+                                label="Beteiligte Personen",
+                                value="tab-2",
+                                style=tab_style,
+                                selected_style=tab_selected_style,
+                            ),
+                            dcc.Tab(
+                                label="Stimmung",
+                                value="tab-3",
+                                style=tab_style,
+                                selected_style=tab_selected_style,
+                            ),
+                            dcc.Tab(
+                                label="Verflechtungen",
+                                value="tab-4",
+                                style=tab_style,
+                                selected_style=tab_selected_style,
+                            ),
+                        ],
+                    ),
+                    html.Div(id="tabs-example-content-1"),
+                ],
+            ),
+        ]
+    )
+
+    @callback(
+        Output("select_company", "options"), Input("select_company", "search_value")
+    )
+    def update_options(search_value: str) -> list:
+        """Update page based on selected company."""
+        if not search_value:
+            raise PreventUpdate
+        return [o for o in options if search_value in o["label"]]
+
+    app.run_server(debug=True)

src/aki_prj23_transparenzregister/utils/postgres/entities.py

@@ -28,22 +28,38 @@ class Company(Base):
     __tablename__ = "company"
 
     __table_args__ = (
-        sa.PrimaryKeyConstraint("id"),
         sa.UniqueConstraint("hr", "court_id"),
+        sa.UniqueConstraint("name", "city"),
+        sa.UniqueConstraint("name", "zip_code"),
+        sa.UniqueConstraint("name"),
     )
 
     id = sa.Column(sa.Integer, primary_key=True, autoincrement=True)
-    hr = sa.Column(sa.Integer, nullable=False)
+    hr = sa.Column(sa.String, nullable=False)
     court_id = sa.Column(
         sa.Integer,
         sa.ForeignKey("district_court.id"),
         nullable=False,
     )
+    name = sa.Column(sa.String(150), nullable=False)
+    street = sa.Column(sa.String(100), nullable=True)
+    zip_code = sa.Column(sa.String(5), nullable=True)
+    city = sa.Column(sa.String(100), nullable=True)
+    last_update = sa.Column(sa.Date, nullable=False)
+    sector = sa.Column(sa.String(100), nullable=True)
+
+
+class Person(Base):
+    """Person."""
+
+    __tablename__ = "person"
+    __table_args__ = (sa.UniqueConstraint("name", "surname", "date_of_birth"),)
+
+    id = sa.Column(sa.Integer, primary_key=True)
     name = sa.Column(sa.String(100), nullable=False)
-    street = sa.Column(sa.String(100), nullable=False)
-    zip_code = sa.Column(sa.String(5), nullable=False)
-    city = sa.Column(sa.String(100), nullable=False)
-    sector = sa.Column(sa.String(100), nullable=False)
+    surname = sa.Column(sa.String(100), nullable=False)
+    date_of_birth = sa.Column(sa.Date, nullable=False)
+    works_for = sa.Column(sa.String(100), nullable=True)
 
 
 class AnnualFinanceStatement(Base):
@@ -52,7 +68,7 @@ class AnnualFinanceStatement(Base):
     __tablename__ = "annual_finance_statement"
 
     id = sa.Column(sa.Integer, primary_key=True)
-    company_id = sa.Column(sa.String, sa.ForeignKey("company.id"))
+    company_id = sa.Column(sa.Integer, sa.ForeignKey("company.id"))
     date = sa.Column(sa.DateTime(timezone=True), nullable=False)
     total_volume = sa.Column(sa.Float)
     ebit = sa.Column(sa.Float)
@@ -63,7 +79,6 @@ class AnnualFinanceStatement(Base):
     debt = sa.Column(sa.Float)
     return_on_equity = sa.Column(sa.Float)
     capital_turnover_rate = sa.Column(sa.Float)
-
     # company: Mapped[Company] = relationship(Company)
 
 
@@ -74,7 +89,7 @@ class Sentiment(Base):
     __tablename__ = "sentiment"
 
     id = sa.Column(sa.Integer, primary_key=True)
-    company_id = sa.Column(sa.String, sa.ForeignKey("company.id"))
+    company_id = sa.Column(sa.Integer, sa.ForeignKey("company.id"))
     date = sa.Column(sa.DateTime(timezone=True), default=datetime.now)
     sentiment_type = sa.Column(
         sa.Enum(SentimentTypeEnum),
@@ -82,20 +97,10 @@ class Sentiment(Base):
     )
     value = sa.Column(sa.Float(), nullable=False)
     source = sa.Column(sa.String(100))
-
     # sentiment = relationship(Company)
 
 
 # create person object
-class Person(Base):
-    """Person."""
-
-    __tablename__ = "person"
-
-    id = sa.Column(sa.Integer, primary_key=True)
-    name = sa.Column(sa.String(100), nullable=False)
-    surname = sa.Column(sa.String(100), nullable=False)
-    works_for = sa.Column(sa.String(100))
 
 
 class Relation(Base):
@@ -103,7 +108,7 @@ class Relation(Base):
 
     __tablename__ = "relation"
     id = sa.Column(sa.Integer, primary_key=True)
-    company_id = sa.Column(sa.String, sa.ForeignKey("company.id"))
+    company_id = sa.Column(sa.Integer, sa.ForeignKey("company.id"))
 
     date_from = sa.Column(sa.DateTime(timezone=True), nullable=True)
     date_to = sa.Column(sa.DateTime(timezone=True), nullable=True)
@@ -133,8 +138,7 @@ class CompanyRelation(Relation):
     __tablename__ = "company_relation"
 
     id = sa.Column(sa.Integer, sa.ForeignKey("relation.id"), primary_key=True)
-    company2_id = sa.Column(sa.String, sa.ForeignKey("company.id"), nullable=False)
+    company2_id = sa.Column(sa.Integer, sa.ForeignKey("company.id"), nullable=False)
 
     # company = relationship("Company")
-
     __table_args__ = {"extend_existing": True}

tests/ui/company_finance_dash_test.py

@@ -0,0 +1,7 @@
+"""Test for the company stats dashboard."""
+from aki_prj23_transparenzregister.ui import company_finance_dash
+
+
+def test_import() -> None:
+    """Checks if an import co company_stats_dash can be made."""
+    assert company_finance_dash is not None