From 8d9981d9670c0b7f1723a5ab0eefba40d3a0a3a4 Mon Sep 17 00:00:00 2001 From: Philipp Horstenkamp Date: Thu, 2 Nov 2023 20:30:04 +0100 Subject: [PATCH] Moved AI files in the AI module. (#308) --- .github/workflows/test-and-build-action.yaml | 2 +- .pre-commit-config.yaml | 1 + .../{utils/mongo => ai}/ner_pipeline.py | 2 +- .../ai/ner_sentiment_config.json | 16 +++++++++++++ .../{utils/mongo => ai}/ner_service.py | 0 .../{utils/mongo => ai}/sentiment_pipeline.py | 2 +- .../{utils/mongo => ai}/sentiment_service.py | 0 .../utils/mongo/ner_sentiment_config.json | 17 ------------- tests/utils/mongo/ner_pipeline_test.py | 24 +++++++------------ tests/utils/mongo/ner_service_test.py | 2 +- tests/utils/mongo/sentiment_pipeline_test.py | 14 +++++------ tests/utils/mongo/sentiment_service_test.py | 2 +- 12 files changed, 37 insertions(+), 45 deletions(-) rename src/aki_prj23_transparenzregister/{utils/mongo => ai}/ner_pipeline.py (98%) create mode 100644 src/aki_prj23_transparenzregister/ai/ner_sentiment_config.json rename src/aki_prj23_transparenzregister/{utils/mongo => ai}/ner_service.py (100%) rename src/aki_prj23_transparenzregister/{utils/mongo => ai}/sentiment_pipeline.py (98%) rename src/aki_prj23_transparenzregister/{utils/mongo => ai}/sentiment_service.py (100%) delete mode 100644 src/aki_prj23_transparenzregister/utils/mongo/ner_sentiment_config.json diff --git a/.github/workflows/test-and-build-action.yaml b/.github/workflows/test-and-build-action.yaml index d059dad..50a1213 100644 --- a/.github/workflows/test-and-build-action.yaml +++ b/.github/workflows/test-and-build-action.yaml @@ -27,7 +27,7 @@ jobs: - run: poetry install --without develop,doc,lint --all-extras - name: Run test suite run: | - poetry run pytest --junit-xml=unit-test-results.xml --cov-report "xml:coverage.xml" --cov=src tests/ + poetry run pytest --junit-xml=unit-test-results.xml --cov-report "xml:coverage.xml" --cov=src --disable-warnings tests/ - name: Archive code coverage results uses: actions/upload-artifact@v3 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3656af8..6d6345c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,6 +22,7 @@ repos: - id: destroyed-symlinks - id: debug-statements - id: pretty-format-json + args: [--autofix] exclude: (\.ipynb$) - id: no-commit-to-branch args: [--branch, main] diff --git a/src/aki_prj23_transparenzregister/utils/mongo/ner_pipeline.py b/src/aki_prj23_transparenzregister/ai/ner_pipeline.py similarity index 98% rename from src/aki_prj23_transparenzregister/utils/mongo/ner_pipeline.py rename to src/aki_prj23_transparenzregister/ai/ner_pipeline.py index 2402865..f123976 100644 --- a/src/aki_prj23_transparenzregister/utils/mongo/ner_pipeline.py +++ b/src/aki_prj23_transparenzregister/ai/ner_pipeline.py @@ -8,10 +8,10 @@ from tqdm import tqdm import aki_prj23_transparenzregister.utils.mongo.connector as conn import aki_prj23_transparenzregister.utils.mongo.news_mongo_service as news +from aki_prj23_transparenzregister.ai import ner_service from aki_prj23_transparenzregister.config.config_providers import ( JsonFileConfigProvider, ) -from aki_prj23_transparenzregister.utils.mongo import ner_service logger.add(sys.stdout, colorize=True) diff --git a/src/aki_prj23_transparenzregister/ai/ner_sentiment_config.json b/src/aki_prj23_transparenzregister/ai/ner_sentiment_config.json new file mode 100644 index 0000000..b710acb --- /dev/null +++ b/src/aki_prj23_transparenzregister/ai/ner_sentiment_config.json @@ -0,0 +1,16 @@ +{ + "ner_service": { + "comment": "Select only one service by setting true and deselect the other with false. Valid doc_attrib: text, title", + "doc_attrib": "text", + "entity": "ORG", + "use_companylist_ner": false, + "use_spacy_ner": false, + "use_transformer_ner": true + }, + "sentiment_service": { + "comment": "Select only one service by setting true and deselect the other with false. Valid doc_attrib: text, title", + "doc_attrib": "text", + "use_spacy": false, + "use_transformer": true + } +} diff --git a/src/aki_prj23_transparenzregister/utils/mongo/ner_service.py b/src/aki_prj23_transparenzregister/ai/ner_service.py similarity index 100% rename from src/aki_prj23_transparenzregister/utils/mongo/ner_service.py rename to src/aki_prj23_transparenzregister/ai/ner_service.py diff --git a/src/aki_prj23_transparenzregister/utils/mongo/sentiment_pipeline.py b/src/aki_prj23_transparenzregister/ai/sentiment_pipeline.py similarity index 98% rename from src/aki_prj23_transparenzregister/utils/mongo/sentiment_pipeline.py rename to src/aki_prj23_transparenzregister/ai/sentiment_pipeline.py index 5b21c46..ec071f7 100644 --- a/src/aki_prj23_transparenzregister/utils/mongo/sentiment_pipeline.py +++ b/src/aki_prj23_transparenzregister/ai/sentiment_pipeline.py @@ -8,9 +8,9 @@ from tqdm import tqdm import aki_prj23_transparenzregister.utils.mongo.connector as conn import aki_prj23_transparenzregister.utils.mongo.news_mongo_service as news +from aki_prj23_transparenzregister.ai import sentiment_service from aki_prj23_transparenzregister.config.config_providers import JsonFileConfigProvider from aki_prj23_transparenzregister.config.config_template import MongoConnection -from aki_prj23_transparenzregister.utils.mongo import sentiment_service class SentimentPipeline: diff --git a/src/aki_prj23_transparenzregister/utils/mongo/sentiment_service.py b/src/aki_prj23_transparenzregister/ai/sentiment_service.py similarity index 100% rename from src/aki_prj23_transparenzregister/utils/mongo/sentiment_service.py rename to src/aki_prj23_transparenzregister/ai/sentiment_service.py diff --git a/src/aki_prj23_transparenzregister/utils/mongo/ner_sentiment_config.json b/src/aki_prj23_transparenzregister/utils/mongo/ner_sentiment_config.json deleted file mode 100644 index a41a2b1..0000000 --- a/src/aki_prj23_transparenzregister/utils/mongo/ner_sentiment_config.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "sentiment_service": { - "comment": "Select only one service by setting true and deselect the other with false. Valid doc_attrib: text, title", - "use_spacy": false, - "use_transformer": true, - "doc_attrib": "text" - }, - - "ner_service": { - "comment": "Select only one service by setting true and deselect the other with false. Valid doc_attrib: text, title", - "use_spacy_ner": false, - "use_transformer_ner": true, - "use_companylist_ner":false, - "doc_attrib": "text", - "entity":"ORG" - } -} diff --git a/tests/utils/mongo/ner_pipeline_test.py b/tests/utils/mongo/ner_pipeline_test.py index bd603c8..df815d6 100644 --- a/tests/utils/mongo/ner_pipeline_test.py +++ b/tests/utils/mongo/ner_pipeline_test.py @@ -4,8 +4,8 @@ from unittest.mock import Mock, patch import pytest +from aki_prj23_transparenzregister.ai.ner_pipeline import EntityPipeline from aki_prj23_transparenzregister.config.config_template import MongoConnection -from aki_prj23_transparenzregister.utils.mongo.ner_pipeline import EntityPipeline @pytest.fixture() @@ -52,16 +52,14 @@ def mock_spacy(mocker: Mock) -> Mock: """ mock = Mock() mocker.patch( - "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.init_spacy", + "aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.init_spacy", return_value=mock, ) return mock # Mocking the NerAnalysisService methods -@patch( - "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_spacy" -) +@patch("aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_spacy") def test_entity_pipeline_with_spacy( mock_ner_spacy: Mock, mock_mongo_connector: Mock, @@ -99,9 +97,7 @@ def test_entity_pipeline_with_spacy( ) -@patch( - "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_spacy" -) +@patch("aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_spacy") def test_entity_pipeline_with_spacy_no_docs( mock_ner_spacy: Mock, mock_mongo_connector: Mock, @@ -137,7 +133,7 @@ def test_entity_pipeline_with_spacy_no_docs( @patch( - "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_company_list" + "aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_company_list" ) def test_entity_pipeline_with_companylist_ner( mock_ner_companylist: Mock, @@ -178,7 +174,7 @@ def test_entity_pipeline_with_companylist_ner( @patch( - "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_company_list" + "aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_company_list" ) def test_entity_pipeline_with_companylist_ner_no_docs( mock_ner_companylist: Mock, @@ -214,9 +210,7 @@ def test_entity_pipeline_with_companylist_ner_no_docs( # Add more test cases for other NER methods (e.g., use_companylist_ner, use_transformer_ner) following a similar pattern. -@patch( - "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_spacy" -) +@patch("aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_spacy") def test_entity_pipeline_with_transformer( mock_ner_transformer: Mock, mock_mongo_connector: Mock, @@ -254,9 +248,7 @@ def test_entity_pipeline_with_transformer( ) -@patch( - "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_spacy" -) +@patch("aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_spacy") def test_entity_pipeline_with_transformer_no_docs( mock_ner_transformer: Mock, mock_mongo_connector: Mock, diff --git a/tests/utils/mongo/ner_service_test.py b/tests/utils/mongo/ner_service_test.py index 45de69a..235ac37 100644 --- a/tests/utils/mongo/ner_service_test.py +++ b/tests/utils/mongo/ner_service_test.py @@ -1,6 +1,6 @@ """Tests for checking NER Services.""" -from aki_prj23_transparenzregister.utils.mongo.ner_service import NerAnalysisService +from aki_prj23_transparenzregister.ai.ner_service import NerAnalysisService def test_ner_spacy() -> None: diff --git a/tests/utils/mongo/sentiment_pipeline_test.py b/tests/utils/mongo/sentiment_pipeline_test.py index 05eeaea..99fce35 100644 --- a/tests/utils/mongo/sentiment_pipeline_test.py +++ b/tests/utils/mongo/sentiment_pipeline_test.py @@ -4,10 +4,10 @@ from unittest.mock import Mock, patch import pytest -from aki_prj23_transparenzregister.config.config_template import MongoConnection -from aki_prj23_transparenzregister.utils.mongo.sentiment_pipeline import ( +from aki_prj23_transparenzregister.ai.sentiment_pipeline import ( SentimentPipeline, ) +from aki_prj23_transparenzregister.config.config_template import MongoConnection @pytest.fixture() @@ -54,14 +54,14 @@ def mock_spacy(mocker: Mock) -> Mock: """ mock = Mock() mocker.patch( - "aki_prj23_transparenzregister.utils.mongo.sentiment_service.SentimentAnalysisService.init_spacy", + "aki_prj23_transparenzregister.ai.sentiment_service.SentimentAnalysisService.init_spacy", return_value=mock, ) return mock @patch( - "aki_prj23_transparenzregister.utils.mongo.sentiment_service.SentimentAnalysisService.sentiment_spacy" + "aki_prj23_transparenzregister.ai.sentiment_service.SentimentAnalysisService.sentiment_spacy" ) def test_sentiment_pipeline_existing_sentiment( mock_sentiment_spacy: Mock, @@ -102,7 +102,7 @@ def test_sentiment_pipeline_existing_sentiment( @patch( - "aki_prj23_transparenzregister.utils.mongo.sentiment_service.SentimentAnalysisService.sentiment_spacy" + "aki_prj23_transparenzregister.ai.sentiment_service.SentimentAnalysisService.sentiment_spacy" ) def test_sentiment_pipeline_no_documents( mock_sentiment_spacy: Mock, @@ -134,7 +134,7 @@ def test_sentiment_pipeline_no_documents( @patch( - "aki_prj23_transparenzregister.utils.mongo.sentiment_service.SentimentAnalysisService.sentiment_spacy" + "aki_prj23_transparenzregister.ai.sentiment_service.SentimentAnalysisService.sentiment_spacy" ) def test_sentiment_pipeline_with_spacy( mock_sentiment_spacy: Mock, @@ -173,7 +173,7 @@ def test_sentiment_pipeline_with_spacy( # Mocking the SentimentAnalysisService methods @patch( - "aki_prj23_transparenzregister.utils.mongo.sentiment_service.SentimentAnalysisService.sentiment_transformer" + "aki_prj23_transparenzregister.ai.sentiment_service.SentimentAnalysisService.sentiment_transformer" ) def test_sentiment_pipeline_with_transformer( mock_sentiment_transformer: Mock, diff --git a/tests/utils/mongo/sentiment_service_test.py b/tests/utils/mongo/sentiment_service_test.py index 3c4faa5..62e4780 100644 --- a/tests/utils/mongo/sentiment_service_test.py +++ b/tests/utils/mongo/sentiment_service_test.py @@ -1,7 +1,7 @@ """Tests for checking Sentiment Services.""" -from aki_prj23_transparenzregister.utils.mongo.sentiment_service import ( +from aki_prj23_transparenzregister.ai.sentiment_service import ( SentimentAnalysisService, )