|
|
|
@ -41,13 +41,17 @@ def make_json_collection(datasetcol: BsXMLDataSetCollection) -> JSONDataSetColle
|
|
|
|
"node function entry point, performs batch processing"
|
|
|
|
"node function entry point, performs batch processing"
|
|
|
|
datasets = datasetcol.datasets
|
|
|
|
datasets = datasetcol.datasets
|
|
|
|
housename = datasetcol._housename
|
|
|
|
housename = datasetcol._housename
|
|
|
|
|
|
|
|
|
|
|
|
output_datasets = context.catalog.load(housename + '_jsonoutput')
|
|
|
|
output_datasets = context.catalog.load(housename + '_jsonoutput')
|
|
|
|
outputfolderpath = output_datasets._folderpath
|
|
|
|
outputfolderpath = output_datasets._folderpath
|
|
|
|
for dataset_filenamestem, dataset in datasets.items():
|
|
|
|
for dataset_filenamestem, dataset in datasets.items():
|
|
|
|
|
|
|
|
logger.info("filestem:" + dataset_filenamestem)
|
|
|
|
# a manual load is required here, because
|
|
|
|
# a manual load is required here, because
|
|
|
|
# the dataset **is not** registered in kedro's catalog
|
|
|
|
# the dataset **is not** registered in kedro's catalog
|
|
|
|
dataset._load()
|
|
|
|
dataset._load()
|
|
|
|
output_source_doc = dataset.transform()
|
|
|
|
output_source_doc = dataset.transform()
|
|
|
|
|
|
|
|
# let's add the house into the JSONDataSet, could be usefull
|
|
|
|
|
|
|
|
output_source_doc['house'] = housename
|
|
|
|
# set dataset's output filepath
|
|
|
|
# set dataset's output filepath
|
|
|
|
output_filepath = outputfolderpath / Path(dataset_filenamestem).with_suffix(".json")
|
|
|
|
output_filepath = outputfolderpath / Path(dataset_filenamestem).with_suffix(".json")
|
|
|
|
output_xmldataset = JSONDataSet(str(output_filepath))
|
|
|
|
output_xmldataset = JSONDataSet(str(output_filepath))
|
|
|
|
|