"
+ "\u001b[1m<\u001b[0m\u001b[1;95mkedro.io.data_catalog.DataCatalog\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f2d5be6e740\u001b[0m\u001b[1m>\u001b[0m\n"
]
},
- "execution_count": 4,
"metadata": {},
- "output_type": "execute_result"
+ "output_type": "display_data"
}
],
"source": [
"catalog"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "40417f25",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "[06/30/23 17:50:49] INFO Loading data from 'xmlreflector' (XMLHousesReflector)... data_catalog.py:345\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[2;36m[06/30/23 17:50:49]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[32m'xmlreflector'\u001b[0m \u001b[1m(\u001b[0mXMLHousesReflector\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=287074;file:///home/gwen/.local/lib/python3.10/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=134334;file:///home/gwen/.local/lib/python3.10/site-packages/kedro/io/data_catalog.py#345\u001b\\\u001b[2m345\u001b[0m\u001b]8;;\u001b\\\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+ "│ /home/gwen/.local/lib/python3.10/site-packages/kedro/io/core.py:187 in load │\n",
+ "│ │\n",
+ "│ 184 │ │ self._logger.debug(\"Loading %s\", str(self)) │\n",
+ "│ 185 │ │ │\n",
+ "│ 186 │ │ try: │\n",
+ "│ ❱ 187 │ │ │ return self._load() │\n",
+ "│ 188 │ │ except DataSetError: │\n",
+ "│ 189 │ │ │ raise │\n",
+ "│ 190 │ │ except Exception as exc: │\n",
+ "│ │\n",
+ "│ /media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/actes-princiers/src/actesdataset │\n",
+ "│ .py:62 in _load │\n",
+ "│ │\n",
+ "│ 59 │ │ self.filepath = filepath │\n",
+ "│ 60 │ │\n",
+ "│ 61 │ def _load(self): │\n",
+ "│ ❱ 62 │ │ raise \"C'est chargé!\" │\n",
+ "│ 63 │ │\n",
+ "│ 64 │ def _save(self): │\n",
+ "│ 65 │ │ raise NotImplementedError(\"Attention : dataset en lecture seule !\") │\n",
+ "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+ "TypeError: exceptions must derive from BaseException\n",
+ "\n",
+ "The above exception was the direct cause of the following exception:\n",
+ "\n",
+ "╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+ "│ /tmp/ipykernel_28884/4226322454.py:1 in <module> │\n",
+ "│ │\n",
+ "│ [Errno 2] No such file or directory: '/tmp/ipykernel_28884/4226322454.py' │\n",
+ "│ │\n",
+ "│ /home/gwen/.local/lib/python3.10/site-packages/kedro/io/data_catalog.py:349 in load │\n",
+ "│ │\n",
+ "│ 346 │ │ │ \"Loading data from '%s' (%s)...\", name, type(dataset).__name__ │\n",
+ "│ 347 │ │ ) │\n",
+ "│ 348 │ │ │\n",
+ "│ ❱ 349 │ │ result = dataset.load() │\n",
+ "│ 350 │ │ │\n",
+ "│ 351 │ │ return result │\n",
+ "│ 352 │\n",
+ "│ │\n",
+ "│ /home/gwen/.local/lib/python3.10/site-packages/kedro/io/core.py:196 in load │\n",
+ "│ │\n",
+ "│ 193 │ │ │ message = ( │\n",
+ "│ 194 │ │ │ │ f\"Failed while loading data from data set {str(self)}.\\n{str(exc)}\" │\n",
+ "│ 195 │ │ │ ) │\n",
+ "│ ❱ 196 │ │ │ raise DataSetError(message) from exc │\n",
+ "│ 197 │ │\n",
+ "│ 198 │ def save(self, data: _DI) -> None: │\n",
+ "│ 199 │ │ \"\"\"Saves data by delegation to the provided save method. │\n",
+ "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+ "DataSetError: Failed while loading data from data set XMLHousesReflector(name=my own dataset).\n",
+ "exceptions must derive from BaseException\n",
+ "\n"
+ ],
+ "text/plain": [
+ "\u001b[31m╭─\u001b[0m\u001b[31m────────────────────────────── \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m ───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/gwen/.local/lib/python3.10/site-packages/kedro/io/\u001b[0m\u001b[1;33mcore.py\u001b[0m:\u001b[94m187\u001b[0m in \u001b[92mload\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m184 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._logger.debug(\u001b[33m\"\u001b[0m\u001b[33mLoading \u001b[0m\u001b[33m%s\u001b[0m\u001b[33m\"\u001b[0m, \u001b[96mstr\u001b[0m(\u001b[96mself\u001b[0m)) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m185 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m186 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m187 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._load() \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m188 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mexcept\u001b[0m DataSetError: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m189 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m190 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m exc: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/actes-princiers/src/\u001b[0m\u001b[1;33mactesdataset\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[1;33m.py\u001b[0m:\u001b[94m62\u001b[0m in \u001b[92m_load\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m 59 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.filepath = filepath \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m 60 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m 61 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_load\u001b[0m(\u001b[96mself\u001b[0m): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 62 \u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mC\u001b[0m\u001b[33m'\u001b[0m\u001b[33mest chargé!\u001b[0m\u001b[33m\"\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m 63 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m 64 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_save\u001b[0m(\u001b[96mself\u001b[0m): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m 65 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mNotImplementedError\u001b[0m(\u001b[33m\"\u001b[0m\u001b[33mAttention : dataset en lecture seule !\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
+ "\u001b[1;91mTypeError: \u001b[0mexceptions must derive from BaseException\n",
+ "\n",
+ "\u001b[3mThe above exception was the direct cause of the following exception:\u001b[0m\n",
+ "\n",
+ "\u001b[31m╭─\u001b[0m\u001b[31m────────────────────────────── \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m ───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/tmp/ipykernel_28884/\u001b[0m\u001b[1;33m4226322454.py\u001b[0m:\u001b[94m1\u001b[0m in \u001b[92m\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[3;31m[Errno 2] No such file or directory: '/tmp/ipykernel_28884/4226322454.py'\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/gwen/.local/lib/python3.10/site-packages/kedro/io/\u001b[0m\u001b[1;33mdata_catalog.py\u001b[0m:\u001b[94m349\u001b[0m in \u001b[92mload\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m346 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[33m\"\u001b[0m\u001b[33mLoading data from \u001b[0m\u001b[33m'\u001b[0m\u001b[33m%s\u001b[0m\u001b[33m'\u001b[0m\u001b[33m (\u001b[0m\u001b[33m%s\u001b[0m\u001b[33m)...\u001b[0m\u001b[33m\"\u001b[0m, name, \u001b[96mtype\u001b[0m(dataset).\u001b[91m__name__\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m347 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m348 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m349 \u001b[2m│ │ \u001b[0mresult = dataset.load() \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m350 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m351 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m result \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m352 \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/gwen/.local/lib/python3.10/site-packages/kedro/io/\u001b[0m\u001b[1;33mcore.py\u001b[0m:\u001b[94m196\u001b[0m in \u001b[92mload\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m193 \u001b[0m\u001b[2m│ │ │ \u001b[0mmessage = ( \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m194 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mFailed while loading data from data set \u001b[0m\u001b[33m{\u001b[0m\u001b[96mstr\u001b[0m(\u001b[96mself\u001b[0m)\u001b[33m}\u001b[0m\u001b[33m.\u001b[0m\u001b[33m\\n\u001b[0m\u001b[33m{\u001b[0m\u001b[96mstr\u001b[0m(exc)\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m195 \u001b[0m\u001b[2m│ │ │ \u001b[0m) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m196 \u001b[2m│ │ │ \u001b[0m\u001b[94mraise\u001b[0m DataSetError(message) \u001b[94mfrom\u001b[0m \u001b[4;96mexc\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m197 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m198 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92msave\u001b[0m(\u001b[96mself\u001b[0m, data: _DI) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m199 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[33m\"\"\"Saves data by delegation to the provided save method.\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
+ "\u001b[1;91mDataSetError: \u001b[0mFailed while loading data from data set \u001b[1;35mXMLHousesReflector\u001b[0m\u001b[1m(\u001b[0m\u001b[33mname\u001b[0m=\u001b[35mmy\u001b[0m own dataset\u001b[1m)\u001b[0m.\n",
+ "exceptions must derive from BaseException\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "catalog.load(\"xmlreflector\")"
+ ]
+ },
{
"cell_type": "code",
"execution_count": 38,
@@ -453,7 +597,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.2"
+ "version": "3.10.6"
}
},
"nbformat": 4,
diff --git a/actes-princiers/src/actes_princiers/customcontext.py b/actes-princiers/src/actes_princiers/customcontext.py
index 852435d..c75b341 100644
--- a/actes-princiers/src/actes_princiers/customcontext.py
+++ b/actes-princiers/src/actes_princiers/customcontext.py
@@ -6,6 +6,8 @@ from kedro.framework.context import KedroContext
from kedro.pipeline import Pipeline, node, pipeline
from actesdataset import XMLDataSet
+
+# FIXME : move in an utils.py module
def tree(directory, relative_to=None):
"helper that returns a directory tree structure"
trees = dict()
@@ -22,14 +24,13 @@ class ProjectContext(KedroContext):
def get_houses(self):
"""loading from generic configuration file
(that is, the global houses `houses.yaml`)"""
- houses_file = self.config_loader.get("houses*")
- return houses_file['houses']
+ houses = self.config_loader.get("houses*")
+ return houses['houses']
def get_houses_datapath(self):
"""loading from generic configuration file"""
- config = self.config_loader.get("config*")
- return config['houses_datapath']
-
+ houses = self.config_loader.get("houses*")
+ return houses['raw_datapath']
def _get_catalog(self, *args, **kwargs):
"catalog loader entry point"
diff --git a/actes-princiers/src/actesdataset.py b/actes-princiers/src/actesdataset.py
index 6b0eb16..9448287 100644
--- a/actes-princiers/src/actesdataset.py
+++ b/actes-princiers/src/actesdataset.py
@@ -5,6 +5,8 @@ from lxml import etree
from kedro.io import AbstractDataSet, DataSetError
+
+# FIXME: supprimer l'héritage
class XMLDataSet(AbstractDataSet):
"lxml.etree._ElementTree loader"
# FIXME set the typing signature !!!!
@@ -23,24 +25,113 @@ class XMLDataSet(AbstractDataSet):
return source_doc
def _save(self, data:str) -> None:
-# raise NotImplementedError("pas encore implemente !!!!")
with open(self._filepath, 'w') as fhandle:
fhandle.write(data)
def _describe(self) -> Dict[str, Any]:
return dict(filepath=self._filepath)
-class JSONDataSet(AbstractDataSet):
- def __init__(self, filepath: str):
- self._filepath = filepath
+class XMLHousesReflector(AbstractDataSet):
+ """``XMLHousesReflector`` stores instances of ``XMLDataSet``
+ implementations to provide ``load`` and ``save`` capabilities.
+ anywhere in the program. To use a ``DataCatalog``, you need to
+ instantiate it with a file system folder path, it "reflects"
+ this file system of XML files.
+ It loads a dictionary of XML data sets.
+
+ Args:
+ data_sets: A dictionary of data set names and data set instances.
- def _load(self) -> Dict:
- with open(self._filepath, 'r') as f:
- return json.load(f)
+ Example::
- def _save(self, data: Dict) -> None:
- with open(self._filepath, 'w') as f:
- json.dump(data, f)
+ >>> from .actesdatasets import XMLDataSet, XMLCatalogReflector
+ >>>
+ >>> cars = XMLDataSet(filepath="cars.xml")
+ >>> io = XMLCatalogReflector(housename='bourbon', folderpath='/tmp/mydir', data_sets={'cars': cars})
+# filepath, load_args=None, save_args=None):
+ """
+ def __init__(self,
+ housename: str,
+ folderpath: str,
+ data_sets: dict[str, XMLDataSet] = None):
+ self._housename = housename
+ self._folderpath = folderpath
+ self._datasets = data_sets
+# self.filepath = filepath
- def _describe(self) -> Dict[str, Any]:
- return dict(filepath=self._filepath)
+ def _load(self):
+ return "C'est chargé!"
+
+ def _save(self):
+ raise NotImplementedError("Attention : dataset en lecture seule !")
+
+ def _exists(self) -> bool:
+ return True
+
+ def _describe(self):
+ return dict(name="my own dataset")
+
+# def load(self, name: str) -> Any:
+# """Loads a registered data set.
+
+# Args:
+# name: A data set to be loaded.
+# version: Optional argument for concrete data version to be loaded.
+# Works only with versioned datasets.
+
+# Returns:
+# The loaded data as configured.
+# """
+# return result
+#
+## def save(self, name: str, data: Any) -> None:
+## """Save data to a registered data set.
+
+## Args:
+## name: A data set to be saved to.
+## data: A data object to be saved as configured in the registered
+## data set.
+
+## Raises:
+## DatasetNotFoundError: When a data set with the given name
+## has not yet been registered.
+
+## Example:
+## ::
+
+## >>> import pandas as pd
+## >>>
+## >>> from kedro.extras.datasets.pandas import CSVDataSet
+## >>>
+## >>> cars = CSVDataSet(filepath="cars.csv",
+## >>> load_args=None,
+## >>> save_args={"index": False})
+## >>> io = DataCatalog(data_sets={'cars': cars})
+## >>>
+## >>> df = pd.DataFrame({'col1': [1, 2],
+## >>> 'col2': [4, 5],
+## >>> 'col3': [5, 6]})
+## >>> io.save("cars", df)
+## """
+## dataset = self._get_dataset(name)
+### self._logger.info("Saving data to '%s' (%s)...", name, type(dataset).__name__)
+## dataset.save(data)
+
+# def _describe(self) -> Dict[str, Any]:
+# return dict(filepath=self._housename)
+
+
+#class JSONDataSet(AbstractDataSet):
+# def __init__(self, filepath: str):
+# self._filepath = filepath
+
+# def _load(self) -> Dict:
+# with open(self._filepath, 'r') as fp:
+# return json.load(fp)
+
+# def _save(self, data: Dict) -> None:
+# with open(self._filepath, 'w') as fp:
+# json.dump(data, fp, sort_keys=True, indent=4)
+
+# def _describe(self) -> Dict[str, Any]:
+# return dict(filepath=self._filepath)