checkpoint(data-ingestion): Replace print statements with loguru

This commit is contained in:
TrisNol 2023-09-06 17:19:19 +02:00
parent 00a5e9ec25
commit d34bb1f3be
2 changed files with 12 additions and 11 deletions

View File

@ -34,7 +34,7 @@ def work(company: typing.Any, company_service: CompanyMongoService) -> None:
if __name__ == "__main__":
import concurrent.futures
from tqdm import tqdm
from loguru import logger
config_provider = JsonFileConfigProvider("./secrets.json")
@ -49,14 +49,15 @@ if __name__ == "__main__":
executor.submit(work, entry, company_service): entry for entry in companies
}
with tqdm(total=len(companies)) as pbar:
# Wait for all tasks to complete
for future in concurrent.futures.as_completed(future_to_entry):
entry = future_to_entry[future]
# try:
# with tqdm(total=len(companies)) as pbar:
# Wait for all tasks to complete
for future in concurrent.futures.as_completed(future_to_entry):
entry = future_to_entry[future]
logger.info(entry["name"])
try:
# Get the result of the completed task (if needed)
result = future.result()
pbar.set_description(entry["name"])
pbar.update(1)
# except Exception as e:
# print(f"Error processing entry {entry}: {e}")
# pbar.set_description(entry["name"])
# pbar.update(1)
except Exception as e:
logger.error(f"Error processing entry {e}")

View File

@ -8,7 +8,7 @@ from deutschland.bundesanzeiger import Bundesanzeiger as Ba
from aki_prj23_transparenzregister.models.auditor import Auditor
from aki_prj23_transparenzregister.models.company import FinancialKPIEnum
pd.options.mode.chained_assignment = ""
pd.options.mode.chained_assignment = None # type: ignore
class Bundesanzeiger: