300 ner service crash (#301)

- Bugfixing: Wenn das Dokument aus der Mongo keinen Eintrag für Text
oder Titel hatte, ist die Pipeline gecrasht. -->Eine Abfrage wurde
hinzugefügt.
- Überschreiben des Datenbanknamens wurde entfernt
- ein Attribut des Transformers hat sich geändert

---------

Co-authored-by: Philipp Horstenkamp <philipp@horstenkamp.de>

src/aki_prj23_transparenzregister/utils/mongo/ner_pipeline.py

@@ -22,7 +22,6 @@ class EntityPipeline:
     def __init__(self, conn_string: conn.MongoConnection) -> None:
         """Method to connect to StagingDB."""
         self.connect_string = conn_string
-        self.connect_string.database = "transparenzregister_ner"
         self.connector = conn.MongoConnector(self.connect_string)
         self.news_obj = news.MongoNewsService(self.connector)
 

src/aki_prj23_transparenzregister/utils/mongo/ner_service.py

@@ -44,7 +44,7 @@ class NerAnalysisService:
         self.classifier = pipeline(
             "ner",
             model="fhswf/bert_de_ner",
-            grouped_entities=True,
+            aggregation_strategy="simple",
             tokenizer="dbmdz/bert-base-german-cased",
         )
 
@@ -72,8 +72,9 @@ class NerAnalysisService:
         # init list for entities
         entities = []
 
-        text = doc[doc_attrib]
-
+        text = doc[doc_attrib].strip()
+        # check if text is a string and not empty
+        if isinstance(text, str) and text:
             # get entities
             doc_nlp = self.nlp(text)
 
@@ -104,7 +105,9 @@ class NerAnalysisService:
         entities = []
 
         # Search the text for company names
-        text = doc[doc_attrib]
+        text = doc[doc_attrib].strip()
+        # check if text is a string and not empty
+        if isinstance(text, str) and text:
             # Convert title to lowercase
             text = text.lower()
 
@@ -136,6 +139,8 @@ class NerAnalysisService:
         # init list for entities
         entities = []
         text = doc[doc_attrib]
+        # check if text is a string and not empty
+        if isinstance(text, str) and text:
             sentences = text.split(". ")  # Split text into sentences based on '. '
 
             # Process each sentence separately
@@ -144,7 +149,8 @@ class NerAnalysisService:
                     sentence
                 )  # Assuming 'classifier' processes a single sentence at a time
 
-            for i in range(len(res)):
-                if res[i]["entity_group"] == ent_type:
-                    entities.append(res[i]["word"])
+                for _ in res:
+                    if _["entity_group"] == ent_type:
+                        entities.append(_["word"])
+
         return dict(Counter(entities))