Moved AI files in the AI module. (#308)

.github/workflows/test-and-build-action.yaml

@@ -27,7 +27,7 @@ jobs:
     - run: poetry install --without develop,doc,lint --all-extras
     - name: Run test suite
       run: |
-        poetry run pytest --junit-xml=unit-test-results.xml --cov-report "xml:coverage.xml" --cov=src tests/
+        poetry run pytest --junit-xml=unit-test-results.xml --cov-report "xml:coverage.xml" --cov=src --disable-warnings tests/
     - name: Archive code coverage results
       uses: actions/upload-artifact@v3
       with:

.pre-commit-config.yaml

@@ -22,6 +22,7 @@ repos:
   - id: destroyed-symlinks
   - id: debug-statements
   - id: pretty-format-json
+    args: [--autofix]
     exclude: (\.ipynb$)
   - id: no-commit-to-branch
     args: [--branch, main]

src/aki_prj23_transparenzregister/ai/ner_pipeline.py

@@ -8,10 +8,10 @@ from tqdm import tqdm
 
 import aki_prj23_transparenzregister.utils.mongo.connector as conn
 import aki_prj23_transparenzregister.utils.mongo.news_mongo_service as news
+from aki_prj23_transparenzregister.ai import ner_service
 from aki_prj23_transparenzregister.config.config_providers import (
     JsonFileConfigProvider,
 )
-from aki_prj23_transparenzregister.utils.mongo import ner_service
 
 logger.add(sys.stdout, colorize=True)
 

src/aki_prj23_transparenzregister/ai/ner_sentiment_config.json

@@ -0,0 +1,16 @@
+{
+  "ner_service": {
+    "comment": "Select only one service by setting true and deselect the other with false. Valid doc_attrib: text, title",
+    "doc_attrib": "text",
+    "entity": "ORG",
+    "use_companylist_ner": false,
+    "use_spacy_ner": false,
+    "use_transformer_ner": true
+  },
+  "sentiment_service": {
+    "comment": "Select only one service by setting true and deselect the other with false. Valid doc_attrib: text, title",
+    "doc_attrib": "text",
+    "use_spacy": false,
+    "use_transformer": true
+  }
+}

src/aki_prj23_transparenzregister/ai/sentiment_pipeline.py

@@ -8,9 +8,9 @@ from tqdm import tqdm
 
 import aki_prj23_transparenzregister.utils.mongo.connector as conn
 import aki_prj23_transparenzregister.utils.mongo.news_mongo_service as news
+from aki_prj23_transparenzregister.ai import sentiment_service
 from aki_prj23_transparenzregister.config.config_providers import JsonFileConfigProvider
 from aki_prj23_transparenzregister.config.config_template import MongoConnection
-from aki_prj23_transparenzregister.utils.mongo import sentiment_service
 
 
 class SentimentPipeline:

src/aki_prj23_transparenzregister/utils/mongo/ner_sentiment_config.json

@@ -1,17 +0,0 @@
-{
-    "sentiment_service": {
-        "comment": "Select only one service by setting true and deselect the other with false. Valid doc_attrib: text, title",
-        "use_spacy": false,
-        "use_transformer": true,
-        "doc_attrib": "text"
-    },
-
-    "ner_service": {
-        "comment": "Select only one service by setting true and deselect the other with false. Valid doc_attrib: text, title",
-        "use_spacy_ner": false,
-        "use_transformer_ner": true,
-        "use_companylist_ner":false,
-        "doc_attrib": "text",
-        "entity":"ORG"
-    }
-}

tests/utils/mongo/ner_pipeline_test.py

@@ -4,8 +4,8 @@ from unittest.mock import Mock, patch
 
 import pytest
 
+from aki_prj23_transparenzregister.ai.ner_pipeline import EntityPipeline
 from aki_prj23_transparenzregister.config.config_template import MongoConnection
-from aki_prj23_transparenzregister.utils.mongo.ner_pipeline import EntityPipeline
 
 
 @pytest.fixture()
@@ -52,16 +52,14 @@ def mock_spacy(mocker: Mock) -> Mock:
     """
     mock = Mock()
     mocker.patch(
-        "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.init_spacy",
+        "aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.init_spacy",
         return_value=mock,
     )
     return mock
 
 
 # Mocking the NerAnalysisService methods
-@patch(
-    "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_spacy"
-)
+@patch("aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_spacy")
 def test_entity_pipeline_with_spacy(
     mock_ner_spacy: Mock,
     mock_mongo_connector: Mock,
@@ -99,9 +97,7 @@ def test_entity_pipeline_with_spacy(
     )
 
 
-@patch(
-    "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_spacy"
-)
+@patch("aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_spacy")
 def test_entity_pipeline_with_spacy_no_docs(
     mock_ner_spacy: Mock,
     mock_mongo_connector: Mock,
@@ -137,7 +133,7 @@ def test_entity_pipeline_with_spacy_no_docs(
 
 
 @patch(
-    "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_company_list"
+    "aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_company_list"
 )
 def test_entity_pipeline_with_companylist_ner(
     mock_ner_companylist: Mock,
@@ -178,7 +174,7 @@ def test_entity_pipeline_with_companylist_ner(
 
 
 @patch(
-    "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_company_list"
+    "aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_company_list"
 )
 def test_entity_pipeline_with_companylist_ner_no_docs(
     mock_ner_companylist: Mock,
@@ -214,9 +210,7 @@ def test_entity_pipeline_with_companylist_ner_no_docs(
 
 
 # Add more test cases for other NER methods (e.g., use_companylist_ner, use_transformer_ner) following a similar pattern.
-@patch(
-    "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_spacy"
-)
+@patch("aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_spacy")
 def test_entity_pipeline_with_transformer(
     mock_ner_transformer: Mock,
     mock_mongo_connector: Mock,
@@ -254,9 +248,7 @@ def test_entity_pipeline_with_transformer(
     )
 
 
-@patch(
-    "aki_prj23_transparenzregister.utils.mongo.ner_service.NerAnalysisService.ner_spacy"
-)
+@patch("aki_prj23_transparenzregister.ai.ner_service.NerAnalysisService.ner_spacy")
 def test_entity_pipeline_with_transformer_no_docs(
     mock_ner_transformer: Mock,
     mock_mongo_connector: Mock,

tests/utils/mongo/ner_service_test.py

@@ -1,6 +1,6 @@
 """Tests for checking NER Services."""
 
-from aki_prj23_transparenzregister.utils.mongo.ner_service import NerAnalysisService
+from aki_prj23_transparenzregister.ai.ner_service import NerAnalysisService
 
 
 def test_ner_spacy() -> None:

tests/utils/mongo/sentiment_pipeline_test.py

@@ -4,10 +4,10 @@ from unittest.mock import Mock, patch
 
 import pytest
 
-from aki_prj23_transparenzregister.config.config_template import MongoConnection
-from aki_prj23_transparenzregister.utils.mongo.sentiment_pipeline import (
+from aki_prj23_transparenzregister.ai.sentiment_pipeline import (
     SentimentPipeline,
 )
+from aki_prj23_transparenzregister.config.config_template import MongoConnection
 
 
 @pytest.fixture()
@@ -54,14 +54,14 @@ def mock_spacy(mocker: Mock) -> Mock:
     """
     mock = Mock()
     mocker.patch(
-        "aki_prj23_transparenzregister.utils.mongo.sentiment_service.SentimentAnalysisService.init_spacy",
+        "aki_prj23_transparenzregister.ai.sentiment_service.SentimentAnalysisService.init_spacy",
         return_value=mock,
     )
     return mock
 
 
 @patch(
-    "aki_prj23_transparenzregister.utils.mongo.sentiment_service.SentimentAnalysisService.sentiment_spacy"
+    "aki_prj23_transparenzregister.ai.sentiment_service.SentimentAnalysisService.sentiment_spacy"
 )
 def test_sentiment_pipeline_existing_sentiment(
     mock_sentiment_spacy: Mock,
@@ -102,7 +102,7 @@ def test_sentiment_pipeline_existing_sentiment(
 
 
 @patch(
-    "aki_prj23_transparenzregister.utils.mongo.sentiment_service.SentimentAnalysisService.sentiment_spacy"
+    "aki_prj23_transparenzregister.ai.sentiment_service.SentimentAnalysisService.sentiment_spacy"
 )
 def test_sentiment_pipeline_no_documents(
     mock_sentiment_spacy: Mock,
@@ -134,7 +134,7 @@ def test_sentiment_pipeline_no_documents(
 
 
 @patch(
-    "aki_prj23_transparenzregister.utils.mongo.sentiment_service.SentimentAnalysisService.sentiment_spacy"
+    "aki_prj23_transparenzregister.ai.sentiment_service.SentimentAnalysisService.sentiment_spacy"
 )
 def test_sentiment_pipeline_with_spacy(
     mock_sentiment_spacy: Mock,
@@ -173,7 +173,7 @@ def test_sentiment_pipeline_with_spacy(
 
 # Mocking the SentimentAnalysisService methods
 @patch(
-    "aki_prj23_transparenzregister.utils.mongo.sentiment_service.SentimentAnalysisService.sentiment_transformer"
+    "aki_prj23_transparenzregister.ai.sentiment_service.SentimentAnalysisService.sentiment_transformer"
 )
 def test_sentiment_pipeline_with_transformer(
     mock_sentiment_transformer: Mock,

tests/utils/mongo/sentiment_service_test.py

@@ -1,7 +1,7 @@
 """Tests for checking Sentiment Services."""
 
 
-from aki_prj23_transparenzregister.utils.mongo.sentiment_service import (
+from aki_prj23_transparenzregister.ai.sentiment_service import (
     SentimentAnalysisService,
 )