princely-acts
/
datascience
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
develop
main
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '42d90def46'
${ noResults }
datascience/actes-princiers/notebooks/LoadDataCatalog.ipynb

621 lines
61 KiB
Plaintext
Raw Normal View History Unescape Escape

add data
3 years ago
{
"cells": [
{
"cell_type": "markdown",
essai pipeline
3 years ago
"id": "aeacd24e",
add data
3 years ago
"metadata": {},
"source": [
"# Catalogs\n",
"\n",
"## Chargement des actors"
]
},
{
"cell_type": "code",
source_doc ok
3 years ago
"execution_count": 2,
add data
3 years ago
"id": "ae9bc24c",
"metadata": {},
"outputs": [
{
"data": {
debut de traitement par lots
3 years ago
"text/html": [
source_doc ok
3 years ago
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">&lt;module&gt;</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1 catalog <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2 </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">dir</span>(catalog) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">3 </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
"<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">NameError: </span>name <span style=\"color: #008000; text-decoration-color: #008000\">'catalog'</span> is not defined\n",
debut de traitement par lots
3 years ago
"</pre>\n"
],
add data
3 years ago
"text/plain": [
source_doc ok
3 years ago
"\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
"\u001b[31m│\u001b[0m in \u001b[92m<module>\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 catalog \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m\u001b[96mdir\u001b[0m(catalog) \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
"\u001b[1;91mNameError: \u001b[0mname \u001b[32m'catalog'\u001b[0m is not defined\n"
add data
3 years ago
]
},
"metadata": {},
debut de traitement par lots
3 years ago
"output_type": "display_data"
add data
3 years ago
}
],
"source": [
source_doc ok
3 years ago
"catalog\n",
"dir(catalog)"
add data
3 years ago
]
},
debut de traitement par lots
3 years ago
{
"cell_type": "code",
"execution_count": 1,
"id": "40417f25",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">[06/30/23 17:50:49] </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO </span> Loading data from <span style=\"color: #008000; text-decoration-color: #008000\">'xmlreflector'</span> <span style=\"font-weight: bold\">(</span>XMLHousesReflector<span style=\"font-weight: bold\">)</span><span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///home/gwen/.local/lib/python3.10/site-packages/kedro/io/data_catalog.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">data_catalog.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///home/gwen/.local/lib/python3.10/site-packages/kedro/io/data_catalog.py#345\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">345</span></a>\n",
"</pre>\n"
],
"text/plain": [
"\u001b[2;36m[06/30/23 17:50:49]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[32m'xmlreflector'\u001b[0m \u001b[1m(\u001b[0mXMLHousesReflector\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=287074;file:///home/gwen/.local/lib/python3.10/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=134334;file:///home/gwen/.local/lib/python3.10/site-packages/kedro/io/data_catalog.py#345\u001b\\\u001b[2m345\u001b[0m\u001b]8;;\u001b\\\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/home/gwen/.local/lib/python3.10/site-packages/kedro/io/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">core.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">187</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">load</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">184 │ │ </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._logger.debug(<span style=\"color: #808000; text-decoration-color: #808000\">\"Loading %s\"</span>, <span style=\"color: #00ffff; text-decoration-color: #00ffff\">str</span>(<span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>)) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">185 │ │ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">186 │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">try</span>: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>187 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._load() <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">188 │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">except</span> DataSetError: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">189 │ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">190 │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">except</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">Exception</span> <span style=\"color: #0000ff; text-decoration-color: #0000ff\">as</span> exc: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/actes-princiers/src/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">actesdataset</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">62</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_load</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 59 │ │ </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.filepath = filepath <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 60 │ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 61 │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">def</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_load</span>(<span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>): <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span> 62 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> <span style=\"color: #808000; text-decoration-color: #808000\">\"C'est chargé!\"</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 63 │ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 64 │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">def</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_save</span>(<span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>): <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 65 │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">NotImplementedError</span>(<span style=\"color: #808000; text-decoration-color: #808000\">\"Attention : dataset en lecture seule !\"</span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
"<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">TypeError: </span>exceptions must derive from BaseException\n",
"\n",
"<span style=\"font-style: italic\">The above exception was the direct cause of the following exception:</span>\n",
"\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/tmp/ipykernel_28884/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">4226322454.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">&lt;module&gt;</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000; font-style: italic\">[Errno 2] No such file or directory: '/tmp/ipykernel_28884/4226322454.py'</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/home/gwen/.local/lib/python3.10/site-packages/kedro/io/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">data_catalog.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">349</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">load</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">346 │ │ │ </span><span style=\"color: #808000; text-decoration-color: #808000\">\"Loading data from '%s' (%s)...\"</span>, name, <span style=\"color: #00ffff; text-decoration-color: #00ffff\">type</span>(dataset).<span style=\"color: #ff0000; text-decoration-color: #ff0000\">__name__</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">347 │ │ </span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">348 │ │ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>349 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span>result = dataset.load() <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">350 │ │ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">351 │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> result <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">352 </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/home/gwen/.local/lib/python3.10/site-packages/kedro/io/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">core.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">196</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">load</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">193 │ │ │ </span>message = ( <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">194 │ │ │ │ </span><span style=\"color: #808000; text-decoration-color: #808000\">f\"Failed while loading data from data set {</span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">str</span>(<span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>)<span style=\"color: #808000; text-decoration-color: #808000\">}.\\n{</span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">str</span>(exc)<span style=\"color: #808000; text-decoration-color: #808000\">}\"</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">195 │ │ │ </span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>196 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> DataSetError(message) <span style=\"color: #0000ff; text-decoration-color: #0000ff\">from</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff; text-decoration: underline\">exc</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">197 │ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">198 │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">def</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">save</span>(<span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>, data: _DI) -&gt; <span style=\"color: #0000ff; text-decoration-color: #0000ff\">None</span>: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">199 │ │ </span><span style=\"color: #808000; text-decoration-color: #808000\">\"\"\"Saves data by delegation to the provided save method.</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
"<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
"<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">DataSetError: </span>Failed while loading data from data set <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">XMLHousesReflector</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">name</span>=<span style=\"color: #800080; text-decoration-color: #800080\">my</span> own dataset<span style=\"font-weight: bold\">)</span>.\n",
"exceptions must derive from BaseException\n",
"</pre>\n"
],
"text/plain": [
"\u001b[31m╭─\u001b[0m\u001b[31m────────────────────────────── \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m ───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2;33m/home/gwen/.local/lib/python3.10/site-packages/kedro/io/\u001b[0m\u001b[1;33mcore.py\u001b[0m:\u001b[94m187\u001b[0m in \u001b[92mload\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m184 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._logger.debug(\u001b[33m\"\u001b[0m\u001b[33mLoading \u001b[0m\u001b[33m%s\u001b[0m\u001b[33m\"\u001b[0m, \u001b[96mstr\u001b[0m(\u001b[96mself\u001b[0m)) \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m185 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m186 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m187 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._load() \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m188 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mexcept\u001b[0m DataSetError: \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m189 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m190 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m exc: \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2;33m/media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/actes-princiers/src/\u001b[0m\u001b[1;33mactesdataset\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[1;33m.py\u001b[0m:\u001b[94m62\u001b[0m in \u001b[92m_load\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m 59 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.filepath = filepath \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m 60 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m 61 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_load\u001b[0m(\u001b[96mself\u001b[0m): \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 62 \u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mC\u001b[0m\u001b[33m'\u001b[0m\u001b[33mest chargé!\u001b[0m\u001b[33m\"\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m 63 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m 64 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_save\u001b[0m(\u001b[96mself\u001b[0m): \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m 65 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mNotImplementedError\u001b[0m(\u001b[33m\"\u001b[0m\u001b[33mAttention : dataset en lecture seule !\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n",
"\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
"\u001b[1;91mTypeError: \u001b[0mexceptions must derive from BaseException\n",
"\n",
"\u001b[3mThe above exception was the direct cause of the following exception:\u001b[0m\n",
"\n",
"\u001b[31m╭─\u001b[0m\u001b[31m────────────────────────────── \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m ───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2;33m/tmp/ipykernel_28884/\u001b[0m\u001b[1;33m4226322454.py\u001b[0m:\u001b[94m1\u001b[0m in \u001b[92m<module>\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[3;31m[Errno 2] No such file or directory: '/tmp/ipykernel_28884/4226322454.py'\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2;33m/home/gwen/.local/lib/python3.10/site-packages/kedro/io/\u001b[0m\u001b[1;33mdata_catalog.py\u001b[0m:\u001b[94m349\u001b[0m in \u001b[92mload\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m346 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[33m\"\u001b[0m\u001b[33mLoading data from \u001b[0m\u001b[33m'\u001b[0m\u001b[33m%s\u001b[0m\u001b[33m'\u001b[0m\u001b[33m (\u001b[0m\u001b[33m%s\u001b[0m\u001b[33m)...\u001b[0m\u001b[33m\"\u001b[0m, name, \u001b[96mtype\u001b[0m(dataset).\u001b[91m__name__\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m347 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m348 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m349 \u001b[2m│ │ \u001b[0mresult = dataset.load() \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m350 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m351 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m result \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m352 \u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2;33m/home/gwen/.local/lib/python3.10/site-packages/kedro/io/\u001b[0m\u001b[1;33mcore.py\u001b[0m:\u001b[94m196\u001b[0m in \u001b[92mload\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m193 \u001b[0m\u001b[2m│ │ │ \u001b[0mmessage = ( \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m194 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mFailed while loading data from data set \u001b[0m\u001b[33m{\u001b[0m\u001b[96mstr\u001b[0m(\u001b[96mself\u001b[0m)\u001b[33m}\u001b[0m\u001b[33m.\u001b[0m\u001b[33m\\n\u001b[0m\u001b[33m{\u001b[0m\u001b[96mstr\u001b[0m(exc)\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m195 \u001b[0m\u001b[2m│ │ │ \u001b[0m) \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m196 \u001b[2m│ │ │ \u001b[0m\u001b[94mraise\u001b[0m DataSetError(message) \u001b[94mfrom\u001b[0m \u001b[4;96mexc\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m197 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m198 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92msave\u001b[0m(\u001b[96mself\u001b[0m, data: _DI) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n",
"\u001b[31m│\u001b[0m \u001b[2m199 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[33m\"\"\"Saves data by delegation to the provided save method.\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
"\u001b[1;91mDataSetError: \u001b[0mFailed while loading data from data set \u001b[1;35mXMLHousesReflector\u001b[0m\u001b[1m(\u001b[0m\u001b[33mname\u001b[0m=\u001b[35mmy\u001b[0m own dataset\u001b[1m)\u001b[0m.\n",
"exceptions must derive from BaseException\n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"catalog.load(\"xmlreflector\")"
]
},
add data
3 years ago
{
"cell_type": "code",
essai pipeline
3 years ago
"execution_count": 38,
add data
3 years ago
"id": "dc290e93",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
essai pipeline
3 years ago
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">[06/16/23 15:56:44] </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO </span> Loading data from <span style=\"color: #008000; text-decoration-color: #008000\">'actors'</span> <span style=\"font-weight: bold\">(</span>CSVDataSet<span style=\"font-weight: bold\">)</span><span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">data_catalog.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py#345\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">345</span></a>\n",
add data
3 years ago
"</pre>\n"
],
"text/plain": [
essai pipeline
3 years ago
"\u001b[2;36m[06/16/23 15:56:44]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[32m'actors'\u001b[0m \u001b[1m(\u001b[0mCSVDataSet\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=858812;file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=44255;file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py#345\u001b\\\u001b[2m345\u001b[0m\u001b]8;;\u001b\\\n"
add data
3 years ago
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>NAME</th>\n",
" <th>ROLE</th>\n",
" <th>HOUSE</th>\n",
" <th>DATE1</th>\n",
" <th>DATE2</th>\n",
" <th>DATE3</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Charles Ier de Bourbon</td>\n",
" <td>prince</td>\n",
" <td>Bourbon</td>\n",
" <td>1400</td>\n",
essai pipeline
3 years ago
" <td>1434.0</td>\n",
" <td>1456.0</td>\n",
add data
3 years ago
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Gort, Étienne</td>\n",
" <td>secret</td>\n",
" <td>Bourbon</td>\n",
" <td>1425</td>\n",
essai pipeline
3 years ago
" <td>1440.0</td>\n",
add data
3 years ago
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Erart</td>\n",
" <td>secret</td>\n",
" <td>Berry</td>\n",
" <td>1404</td>\n",
essai pipeline
3 years ago
" <td>1405.0</td>\n",
add data
3 years ago
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Jean de Berry</td>\n",
" <td>prince</td>\n",
" <td>Berry</td>\n",
" <td>1337</td>\n",
essai pipeline
3 years ago
" <td>1360.0</td>\n",
" <td>1416.0</td>\n",
add data
3 years ago
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Agnès de Bourgogne</td>\n",
" <td>prince</td>\n",
" <td>Bourbon</td>\n",
" <td>1407</td>\n",
essai pipeline
3 years ago
" <td>1434.0</td>\n",
" <td>1476.0</td>\n",
add data
3 years ago
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
essai pipeline
3 years ago
" NAME ROLE HOUSE DATE1 DATE2 DATE3\n",
"0 Charles Ier de Bourbon prince Bourbon 1400 1434.0 1456.0\n",
"1 Gort, Étienne secret Bourbon 1425 1440.0 NaN\n",
"2 Erart secret Berry 1404 1405.0 NaN\n",
"3 Jean de Berry prince Berry 1337 1360.0 1416.0\n",
"4 Agnès de Bourgogne prince Bourbon 1407 1434.0 1476.0"
add data
3 years ago
]
},
essai pipeline
3 years ago
"execution_count": 38,
add data
3 years ago
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"catalog.load(\"actors\").head()"
]
},
{
"cell_type": "code",
"execution_count": 5,
essai pipeline
3 years ago
"id": "eedbc7fb",
add data
3 years ago
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['actors', 'corpus-agnes-bourgogne', 'corpus-charles-i', 'parameters']"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"catalog.list()"
]
},
{
"cell_type": "code",
"execution_count": 20,
essai pipeline
3 years ago
"id": "3168935f",
add data
3 years ago
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">[06/16/23 14:58:30] </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO </span> Loading data from <span style=\"color: #008000; text-decoration-color: #008000\">'actors'</span> <span style=\"font-weight: bold\">(</span>CSVDataSet<span style=\"font-weight: bold\">)</span><span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">data_catalog.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py#345\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">345</span></a>\n",
"</pre>\n"
],
"text/plain": [
"\u001b[2;36m[06/16/23 14:58:30]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[32m'actors'\u001b[0m \u001b[1m(\u001b[0mCSVDataSet\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=659228;file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=160900;file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py#345\u001b\\\u001b[2m345\u001b[0m\u001b]8;;\u001b\\\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"['NAME', 'ROLE', 'HOUSE', 'DATE1', 'DATE2', 'DATE3']"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"actors = catalog.load(\"actors\")\n",
"actors.columns.tolist()"
]
},
{
"cell_type": "markdown",
essai pipeline
3 years ago
"id": "902dd387",
add data
3 years ago
"metadata": {},
"source": [
"## Nettoyage des valeurs non renseignées\n",
"\n",
"Ligne d'origine (ligne 9) : \n",
"`\"René d'Anjou\";\"prince\";\"Anjou\";\"XXXX\";\"XXXX\";\"XXXX\"`\n"
]
},
{
"cell_type": "code",
"execution_count": 37,
essai pipeline
3 years ago
"id": "24fc62ce",
add data
3 years ago
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"NAME Bernard d'Armagnac\n",
"ROLE prince\n",
"HOUSE Armagnac\n",
"DATE1 NaN\n",
"DATE2 NaN\n",
"DATE3 NaN\n",
"Name: 9, dtype: object"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#actors.values\n",
"import numpy as np\n",
"cleaned_actors = actors.replace(\"XXXX\", np.NaN)\n",
"actors.head()\n",
"#actors.values\n",
"cleaned_actors.iloc[9]"
]
traitement xslt sur un fichier
3 years ago
},
PartitionedDataSet utilisé pour transformer un lot de datas
3 years ago
{
"cell_type": "markdown",
"id": "ee287f62",
"metadata": {},
"source": [
"## Autres catalogues"
]
},
traitement xslt sur un fichier
3 years ago
{
"cell_type": "code",
"execution_count": 1,
"id": "053ed17c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['actors',\n",
" 'corpus-agnes-bourgogne',\n",
" 'corpus-charles-i',\n",
" 'dataset_test',\n",
" 'preprocessed_dataset_test',\n",
" 'load_xml',\n",
" 'preprocess_html',\n",
PartitionedDataSet utilisé pour transformer un lot de datas
3 years ago
" 'load_full_xml_catalog',\n",
" 'preprocess_full_catalog_html',\n",
traitement xslt sur un fichier
3 years ago
" 'preprocessed_actors',\n",
PartitionedDataSet utilisé pour transformer un lot de datas
3 years ago
" 'parameters',\n",
" 'params:xlststylesheet']"
traitement xslt sur un fichier
3 years ago
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"catalog.list()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "660b898c",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">[06/20/23 16:44:19] </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO </span> Loading data from <span style=\"color: #008000; text-decoration-color: #008000\">'load_xml'</span> <span style=\"font-weight: bold\">(</span>XMLDataSet<span style=\"font-weight: bold\">)</span><span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">data_catalog.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py#345\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">345</span></a>\n",
"</pre>\n"
],
"text/plain": [
"\u001b[2;36m[06/20/23 16:44:19]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[32m'load_xml'\u001b[0m \u001b[1m(\u001b[0mXMLDataSet\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=813727;file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=696103;file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py#345\u001b\\\u001b[2m345\u001b[0m\u001b]8;;\u001b\\\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"<lxml.etree._ElementTree at 0x7f3e4c3b99c0>"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"catalog.load(\"load_xml\")"
]
PartitionedDataSet utilisé pour transformer un lot de datas
3 years ago
},
{
"cell_type": "markdown",
"id": "a46ddef9",
"metadata": {},
"source": [
"## PartitionedDataset catalogs"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "96a60999",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">[06/22/23 15:01:39] </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO </span> Loading data from <span style=\"color: #008000; text-decoration-color: #008000\">'load_full_xml_catalog'</span> <span style=\"font-weight: bold\">(</span>PartitionedDataSet<span style=\"font-weight: bold\">)</span><span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">data_catalog.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py#345\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">345</span></a>\n",
"</pre>\n"
],
"text/plain": [
"\u001b[2;36m[06/22/23 15:01:39]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[32m'load_full_xml_catalog'\u001b[0m \u001b[1m(\u001b[0mPartitionedDataSet\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=663642;file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=709654;file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py#345\u001b\\\u001b[2m345\u001b[0m\u001b]8;;\u001b\\\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\"> </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO </span> Loading data from <span style=\"color: #008000; text-decoration-color: #008000\">'load_full_xml_catalog'</span> <span style=\"font-weight: bold\">(</span>PartitionedDataSet<span style=\"font-weight: bold\">)</span><span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">data_catalog.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py#345\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">345</span></a>\n",
"</pre>\n"
],
"text/plain": [
"\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[32m'load_full_xml_catalog'\u001b[0m \u001b[1m(\u001b[0mPartitionedDataSet\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=916916;file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=129179;file:///media/gwen/maxtor/gwen/entrepot/cnrs/nicolas/depot/datascience/.venv/lib/python3.9/site-packages/kedro/io/data_catalog.py#345\u001b\\\u001b[2m345\u001b[0m\u001b]8;;\u001b\\\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"{'anj_is_i_1441_08_05a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7e16df0>>,\n",
" 'anj_lo_i_1360_08a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9700>>,\n",
" 'anj_lo_i_1371_07_08a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd96a0>>,\n",
" 'anj_lo_ii_1401_04_28a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9430>>,\n",
" 'anj_lo_ii_1402_11_07a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd92b0>>,\n",
" 'anj_lo_ii_1405_05_02a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9340>>,\n",
" 'anj_lo_ii_1406_01_26a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd93d0>>,\n",
" 'anj_lo_ii_1406_04_15a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd94c0>>,\n",
" 'anj_lo_ii_1409_08_07a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd94f0>>,\n",
" 'anj_lo_ii_1409_12_12a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9520>>,\n",
" 'anj_lo_ii_1413_03_01a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9610>>,\n",
" 'anj_lo_iii_1420_11_04a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9580>>,\n",
" 'anj_lo_iii_1422_02_09a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd95b0>>,\n",
" 'anj_lo_iii_1424_03_31a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9d00>>,\n",
" 'anj_lo_iii_1424_03_31b': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9c40>>,\n",
" 'anj_lo_iii_1428_06_07a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9970>>,\n",
" 'anj_lo_iii_1428_06_07b': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9940>>,\n",
" 'anj_lo_iii_1432_10_27a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9880>>,\n",
" 'anj_ma_i_1370_12_10a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9a90>>,\n",
" 'anj_re_i_1437_09_16a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9a30>>,\n",
" 'anj_re_i_1439_11_22a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9af0>>,\n",
" 'anj_re_i_1440_01_20a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9730>>,\n",
" 'anj_re_i_1445a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9b20>>,\n",
" 'anj_re_i_1450_11_07a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd99d0>>,\n",
" 'anj_re_i_1454_01_14a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd99a0>>,\n",
" 'anj_re_i_1454_02_09a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd97f0>>,\n",
" 'anj_re_i_1454_06_17a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9d30>>,\n",
" 'anj_re_i_1454_09_01a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9d90>>,\n",
" 'anj_re_i_1455_11_13a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9df0>>,\n",
" 'anj_re_i_1456_11_29a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9e50>>,\n",
" 'anj_re_i_1457_01_04a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9eb0>>,\n",
" 'anj_re_i_1459_03_17a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9f10>>,\n",
" 'anj_re_i_1459_04_16a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9f70>>,\n",
" 'anj_re_i_1463_07_21a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7dd9fd0>>,\n",
" 'anj_re_i_1466_12_16a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7ddb070>>,\n",
" 'anj_re_i_1474_02_01a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7ddb0d0>>,\n",
" 'anj_re_i_1475_05_26a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7ddb130>>,\n",
" 'anj_yo_i_1418_12_20a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7ddb190>>,\n",
" 'anj_yo_i_1421_06_28a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7ddb1f0>>,\n",
" 'anj_yo_i_1442_02_24a': <bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7fa3f7ddb250>>}"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"partitions = catalog.load('load_full_xml_catalog')\n",
"catalog.load('load_full_xml_catalog')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "bdc37079",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<bound method AbstractDataSet.load of <actesdataset.XMLDataSet object at 0x7faad403c550>>"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"partitions['anj_is_i_1441_08_05a']"
]
add data
3 years ago
}
],
"metadata": {
"kernelspec": {
"display_name": "Kedro (actes_princiers)",
"language": "python",
"name": "kedro_actes_princiers"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
source_doc ok
3 years ago
"version": "3.9.2"
add data
3 years ago
}
},
"nbformat": 4,
"nbformat_minor": 5
}