diff --git a/README.md b/README.md index 6bb3650..57c1ba0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Actes princiers -- data transformations +1. kedro run --tags="etl_transform" +2. run --tags="populate_database" + + ## Project Name human readable name : `Actes Princiers` diff --git a/actes-princiers/src/actes_princiers/pipelines/populate_mongo/nodes.py b/actes-princiers/src/actes_princiers/pipelines/populate_mongo/nodes.py index 2713d54..c0b8c53 100755 --- a/actes-princiers/src/actes_princiers/pipelines/populate_mongo/nodes.py +++ b/actes-princiers/src/actes_princiers/pipelines/populate_mongo/nodes.py @@ -1,26 +1,31 @@ import logging +import urllib.parse + from pathlib import Path from typing import Dict from kedro.framework.session import KedroSession - from actesdataset import JSONDataSetCollection import pymongo - logger = logging.getLogger(__name__) -def populate_mongo(jsondoc: JSONDataSetCollection, storage_ip: str, db_name: str, db_collection_name: str) -> None: +def populate_mongo(jsondoc: JSONDataSetCollection, storage_ip: str, db_name: str, db_collection_name: str, mongodb_admin: str, mongodb_password: str) -> None: #logger.info(storage_ip) #logger.info(db_name) #logger.info(db_collection_name) jsondatasets = jsondoc.datasets housename = jsondoc._housename - mongodb_url = "mongodb://{}:27017/".format(storage_ip) + #mongodb://%s:%s@149.202.41.75:27017' % (username, password) + # FIXME passer en parametres + username = urllib.parse.quote_plus(mongodb_admin) + password = urllib.parse.quote_plus(mongodb_password) + mongodb_url = f"mongodb://{username}:{password}@{storage_ip}:27017/" + #mongodb_url = "mongodb://{}:27017/".format(storage_ip) logger.info("connection to the mongodb server: " + mongodb_url) # pymongo settings diff --git a/actes-princiers/src/actes_princiers/pipelines/populate_mongo/pipeline.py b/actes-princiers/src/actes_princiers/pipelines/populate_mongo/pipeline.py index d8fde8d..fc983b8 100755 --- a/actes-princiers/src/actes_princiers/pipelines/populate_mongo/pipeline.py +++ b/actes-princiers/src/actes_princiers/pipelines/populate_mongo/pipeline.py @@ -9,7 +9,8 @@ def create_pipeline(**kwargs) -> Pipeline: node( func=populate_mongo, inputs=["bourbon_fulljsonoutput", "params:storage_ip", "params:db_name", - "params:db_collection_name"], + "params:db_collection_name", "params:mongodb_admin", + "params:mongodb_password"], outputs=None, name="populate_mongo", tags="populate_database",