Philipp/aki_prj23_transparenzregister
Philipp/aki_prj23_transparenzregister
1
0
Fork 0
mirror of https://github.com/fhswf/aki_prj23_transparenzregister.git synced 2025-04-25 12:42:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

Simplified some functions. (#414)

Browse Source
This commit is contained in:
Philipp Horstenkamp 2023-11-26 15:01:12 +01:00 committed by GitHub
parent 6a584f5c10
commit 6890562a18
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 32 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/aki_prj23_transparenzregister/ai/ner_pipeline.py
View File

@ -67,15 +67,15 @@ class EntityPipeline:
ner_service_func = ner_service_instance.ner_transformer ner_service_func = ner_service_instance.ner_transformer
else: else:
raise ValueError raise ValueError
if len(documents) > 0: if not documents:
for document in tqdm(documents):
ents = ner_service_func(document, "ORG", doc_attrib)
self.news_obj.collection.update_one(
{"_id": document["_id"]},
{"$set": {"companies": ents}},
)
else:
logger.info("No documents found.") logger.info("No documents found.")
return
for document in tqdm(documents):
ents = ner_service_func(document, "ORG", doc_attrib)
self.news_obj.collection.update_one(
{"_id": document["_id"]},
{"$set": {"companies": ents}},
)
def execute_ner(config_provider: ConfigProvider) -> None: def execute_ner(config_provider: ConfigProvider) -> None:

50
src/aki_prj23_transparenzregister/ai/sentiment_pipeline.py
View File

@ -43,33 +43,31 @@ class SentimentPipeline:
{"sentiment": {"$exists": False}} {"sentiment": {"$exists": False}}
) )
documents = list(cursor_unprocessed) documents = list(cursor_unprocessed)
logger.info(f"Documents to be processed: {cursor_unprocessed}") if not documents:
if len(documents) > 0:
for document in tqdm(documents):
text = document[doc_attrib]
# Determine sentiment analysis service based on config
if sentiment_method == "spacy":
selected_service = sentiment_service.SentimentAnalysisService(
use_spacy=True, use_transformer=False
)
sentiment_service_func = selected_service.sentiment_spacy
elif sentiment_method == "transformer":
selected_service = sentiment_service.SentimentAnalysisService(
use_spacy=False, use_transformer=True
)
sentiment_service_func = selected_service.sentiment_transformer
sents = sentiment_service_func(text)
sentiment = {"label": sents[0], "score": sents[1]}
self.news_obj.collection.update_one(
{"_id": document["_id"]},
{"$set": {"sentiment": sentiment}},
)
else:
logger.info("No documents found.") logger.info("No documents found.")
return
for document in tqdm(documents):
text = document[doc_attrib]
# Determine sentiment analysis service based on config
if sentiment_method == "spacy":
selected_service = sentiment_service.SentimentAnalysisService(
use_spacy=True, use_transformer=False
)
sentiment_service_func = selected_service.sentiment_spacy
elif sentiment_method == "transformer":
selected_service = sentiment_service.SentimentAnalysisService(
use_spacy=False, use_transformer=True
)
sentiment_service_func = selected_service.sentiment_transformer
sents = sentiment_service_func(text)
sentiment = {"label": sents[0], "score": sents[1]}
self.news_obj.collection.update_one(
{"_id": document["_id"]},
{"$set": {"sentiment": sentiment}},
)
def execute_sentiment(config_provider: ConfigProvider) -> None: def execute_sentiment(config_provider: ConfigProvider) -> None: