From 55b6990228d409e98f80fe13d432d51fac0657be Mon Sep 17 00:00:00 2001 From: gwen Date: Thu, 6 Jul 2023 15:28:04 +0200 Subject: [PATCH] get catalog --- actes-princiers/src/actes_princiers/customcontext.py | 6 ++++++ .../pipelines/xml_processing/nodes.py | 2 +- .../pipelines/xml_processing/pipeline.py | 12 ++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/actes-princiers/src/actes_princiers/customcontext.py b/actes-princiers/src/actes_princiers/customcontext.py index 6238cb7..49763a3 100644 --- a/actes-princiers/src/actes_princiers/customcontext.py +++ b/actes-princiers/src/actes_princiers/customcontext.py @@ -22,6 +22,12 @@ class ProjectContext(KedroContext): houses = self.config_loader.get("houses*") return houses['raw_datapath'] + def get_catalog(self): + "catalog loader entry point" + # loading yaml defined catalogs + catalog = self.config_loader.get('catalog*') + return catalog + # def _get_catalog(self, *args, **kwargs): # "catalog loader entry point" # # loading yaml defined catalogs diff --git a/actes-princiers/src/actes_princiers/pipelines/xml_processing/nodes.py b/actes-princiers/src/actes_princiers/pipelines/xml_processing/nodes.py index 64fd0c4..5904028 100755 --- a/actes-princiers/src/actes_princiers/pipelines/xml_processing/nodes.py +++ b/actes-princiers/src/actes_princiers/pipelines/xml_processing/nodes.py @@ -18,7 +18,7 @@ def parse_xml_collection(datasetcollection: XMLDataSetCollection) -> Dict[str, E # the dataset **is not** registered in kedro's catalog dataset._load() descr = dataset._describe() - logger.info(f"dataset {descr} loaded") +# logger.info(f"dataset {descr} loaded") output_source_doc = dataset.transform() # set dataset's output filepath # output_filepath = dataset.filepath.replace("01_raw", "02_intermediate") diff --git a/actes-princiers/src/actes_princiers/pipelines/xml_processing/pipeline.py b/actes-princiers/src/actes_princiers/pipelines/xml_processing/pipeline.py index bc04674..614daf3 100755 --- a/actes-princiers/src/actes_princiers/pipelines/xml_processing/pipeline.py +++ b/actes-princiers/src/actes_princiers/pipelines/xml_processing/pipeline.py @@ -1,7 +1,19 @@ +import logging + from kedro.pipeline import Pipeline, node, pipeline from .nodes import parse_xml_collection +from kedro.framework.session import KedroSession + +logger = logging.getLogger(__name__) + +with KedroSession.create() as session: + context = session.load_context() + catalog = context.get_catalog() + bourbon = catalog['bourbon_xmlcontent'] + logger.info("+++++++++++++++++++" + bourbon['folderpath']) + def create_pipeline(**kwargs) -> Pipeline: return pipeline(