More tweaks to new logging implementation.

Author: Kevin Armengol

.gitignore

@@ -9,6 +9,8 @@
 /dist/
 /docs
 /notebooks/.ipynb_checkpoints
+/notebooks/logger.log
 __pycache__/
-/ddcuimap/**/logging.txt
-/logging.txt
+/ddcuimap/**/logger.log
+/ddcuimap-workspace.code-workspace
+/.vscode

ddcuimap/__init__.py

@@ -1,5 +1,2 @@
 # from get_version import get_version
 # __version__ = get_version(__file__)
-
-# import logging.config
-# logging.config.fileConfig('/configs/logging/logging.yaml')

ddcuimap/curation/__init__.py

@@ -1,8 +1,8 @@
 import logging
-from ddcuimap.utils.setup_logging import log_setup
-
+from ddcuimap.utils.logger.config_logging import setup_log, log, copy_log
 
 # CREATE LOGGER
-log_setup()
-logger = logging.getLogger("curation_logger")
-logger.propagate = False
+setup_log()
+cur_logger = logging.getLogger("curation_logger")
+# logger.propagate = False
+cur_logger.info("Initiating ddcuimap.curation logger.")

ddcuimap/curation/check_cuis.py

@@ -8,8 +8,7 @@
 import numpy as np
 
 from ddcuimap.utils import helper as helper
-from ddcuimap.utils.decorators import log
-from ddcuimap.curation import logger
+from ddcuimap.curation import cur_logger, log, copy_log
 from ddcuimap.curation.utils import dictionary_functions as dd
 
 
@@ -95,12 +94,13 @@ def check_cuis(cfg):
         df_multi_cui = df_multi_cui.add_suffix("_multi_cui")
         df_check = df_check.join(df_multi_cui, how="outer")
 
-        logger.info("Done checking CUIs for " + check)
+        cur_logger.info("Done checking CUIs for " + check)
 
-    # Save file
+    # SAVE FILE AND MOVE LOG
     fp_check = os.path.join(dir_check, "dictionary-import-file-check.csv")
     df_check.to_csv(fp_check, index=False)
-    logger.info("Saved file to " + fp_check)
+    cur_logger.info("Saved file to " + fp_check)
+    copy_log(cur_logger, dir_check, "dictionary-import-file-check.log")
 
     return df_check
 

ddcuimap/curation/create_dictionary_import_file.py

@@ -7,8 +7,7 @@
 from pathlib import Path
 
 from ddcuimap.utils import helper as helper
-from ddcuimap.utils.decorators import log
-from ddcuimap.curation import logger
+from ddcuimap.curation import cur_logger, log, copy_log
 from ddcuimap.curation.utils import curation_functions as cur
 
 
@@ -70,12 +69,15 @@ def create_dd_file(cfg):
         .pipe(cur.override_cols, cfg.custom.create_dictionary_import_settings.override)
     )
 
-    # SAVE FINALIZED IMPORT TEMPLATE
+    # SAVE FINALIZED IMPORT
     fp_step2 = f"{dir_step2}/{cfg.custom.curation_settings.file_settings.file_prefix}_Step-2_dictionary-import-file.csv"
     cfg.custom.create_dictionary_import_settings.dict_file_path = fp_step2
     df_final.to_csv(fp_step2, index=False)  # output df_final dataframe to csv
-    logger.info(f"Saved {fp_step2}")
+    cur_logger.info(f"Saved {fp_step2}")
+
+    # SAVE CONFIG AND MOVE LOG
     helper.save_config(cfg, dir_step2)
+    copy_log(cur_logger, dir_step2, "cur_logger.log")
 
     return df_final
 

ddcuimap/curation/utils/curation_functions.py

@@ -9,9 +9,8 @@
 import numpy as np
 import pandas as pd
 
+from ddcuimap.curation import cur_logger, log, copy_log
 from ddcuimap.utils import helper as helper
-from ddcuimap.utils.decorators import log
-from ddcuimap.curation import logger
 from ddcuimap.curation.utils import xlsx_formatting as xlsx
 
 
@@ -166,7 +165,7 @@ def filter_keep_col(df):
 def order_keep_col(df):
     """Orders rows in keep column by number and letter e.g., 1a, 1b, 2a, 2b, 3a, 3b"""
 
-    # TODO: need to fix issue where 1a,1b,2,2c puts 2 first.
+    # TODO: need to fix issue where 1a,1b,2,2c puts 2 first and also treats 2 and 2b as separate (2|2b instead of 2/2b)
     df["keep"] = df["keep"].astype(str)
     df["keep_num"] = [x[0] for x in df["keep"]]
     df["keep_letter"] = [x[1:] if len(x) > 1 else "" for x in df["keep"]]
@@ -241,7 +240,7 @@ def keep_existing_cols(df_cols, cols_to_check: list):
     )  # TODO: check why I wrote this
     cols_excl = list(set(cols_to_check).difference(df_cols))
     cols = [x for x in df_cols if x not in cols_excl]
-    logger.warning(
+    cur_logger.warning(
         f"The following columns were not found and will be excluded: {cols_excl}"
     )
     return cols

ddcuimap/curation/utils/process_data_dictionary.py

@@ -4,12 +4,10 @@
 
 """
 
-
 import pandas as pd
 
+from ddcuimap.curation import cur_logger, log, copy_log
 from ddcuimap.utils import helper as helper
-from ddcuimap.utils.decorators import log
-from ddcuimap.curation import logger
 from ddcuimap.curation.utils import text_processing as tp
 
 
@@ -20,13 +18,13 @@ def load_data_dictionary(cfg):
     if not cfg.custom.data_dictionary_settings.filepath:
         fp_dd = helper.choose_file("Select data dictionary csv input file")
         df_dd = pd.read_csv(fp_dd)
-        logger.info(f"Data Dictionary shape is: {df_dd.shape}")
+        cur_logger.info(f"Data Dictionary shape is: {df_dd.shape}")
         cfg.custom.data_dictionary_settings.filepath = fp_dd
     else:
         fp_dd = cfg.custom.data_dictionary_settings.filepath
-        logger.warning(f"Loading data dictionary from filepath in configs.")
+        cur_logger.warning(f"Loading data dictionary from filepath in configs.")
         df_dd = pd.read_csv(fp_dd)
-        logger.info(f"Data Dictionary shape is: {df_dd.shape}")
+        cur_logger.info(f"Data Dictionary shape is: {df_dd.shape}")
     return df_dd, fp_dd
 
 
@@ -75,5 +73,5 @@ def process_data_dictionary(df_dd, cfg):
             tp.remove_stopwords_cols, cols_extracted, cfg.custom.preprocessing_settings
         )
     )
-    logger.info(f"Processed Data Dictionary shape is: {df_dd_preprocessed.shape}")
+    cur_logger.info(f"Processed Data Dictionary shape is: {df_dd_preprocessed.shape}")
     return df_dd_preprocessed

ddcuimap/curation/utils/text_processing.py

@@ -14,8 +14,7 @@
 # import cchardet # TODO: may be useful in future
 
 from ddcuimap.utils import helper
-from ddcuimap.utils.decorators import log
-from ddcuimap.curation import logger
+from ddcuimap.curation import cur_logger, log, copy_log
 
 
 # TEXT PROCESSING FUNCTIONS
@@ -74,10 +73,10 @@ def remove_stopwords_cols(df, columns, preprocessing_settings):
     cols_query_terms = []
     if preprocessing_settings.remove_stopwords:
         if preprocessing_settings.stopwords_filepath:
-            logger.warning("Loading stopwords file from configs")
+            cur_logger.warning("Loading stopwords file from configs")
             fp_stopwords = preprocessing_settings.stopwords_filepath
         else:
-            logger.warning("Opening dialog box to choose stopwords file")
+            cur_logger.warning("Opening dialog box to choose stopwords file")
             fp_stopwords = helper.choose_file("Select Stopwords csv file")
         df_stopwords = pd.read_csv(fp_stopwords)
         ls_stopwords = list(
@@ -106,18 +105,18 @@ def remove_vars_cheatsheet(df, preprocessing_settings):  # TODO: not yet impleme
 
     if preprocessing_settings.use_cheatsheet:
         if preprocessing_settings.cheatsheet_filepath:
-            logger.warning("Loading cheatsheet file from configs")
+            cur_logger.warning("Loading cheatsheet file from configs")
             fp_cheatsheet = preprocessing_settings.cheatsheet_filepath
         else:
-            logger.warning("Opening dialog box to choose cheatsheet file")
+            cur_logger.warning("Opening dialog box to choose cheatsheet file")
             fp_cheatsheet = helper.choose_file(title="Select Cheatsheet csv file")
         df_cheatsheet = pd.read_csv(fp_cheatsheet)
         curated_vars = df_cheatsheet[
             "variable name"
         ]  # TODO: need to add consistent formatting for use of a cheatsheet
         df = df[~df["variable name"].isin(curated_vars)]
     else:
-        logger.warning("Cheatsheet not used")
+        cur_logger.warning("Cheatsheet not used")
         pass
     return df
 

ddcuimap/curation/utils/xlsx_formatting.py

@@ -1,6 +1,12 @@
+"""
+
+Functions for formatting Excel curation file.
+
+"""
+
 from openpyxl.utils import get_column_letter
 
-from ddcuimap.utils.decorators import log
+from ddcuimap.curation import log
 
 
 # EXCEL FORMATTING

ddcuimap/hydra_search/__init__.py

@@ -1,9 +1,7 @@
 import logging
-from ddcuimap.utils.setup_logging import log_setup
-
+from ddcuimap.utils.logger.config_logging import setup_log, log, copy_log
 
 # CREATE LOGGER
-log_setup()
-logger = logging.getLogger("hydra_search_logger")
-logger.propagate = False
-logger.info("Initiating ddcuimap.hydra_search logging.")
+setup_log()
+hydra_logger = logging.getLogger("hydra_search_logger")
+hydra_logger.info("Initiating ddcuimap.hydra_search logger.")