|
|
|
|
@ -44,6 +44,7 @@ class Acte(Document):
|
|
|
|
|
ref_acte = StringField(required=True, max_length=100)
|
|
|
|
|
xmlcontent = StringField(required=True) # no max_length
|
|
|
|
|
place = DictField()
|
|
|
|
|
folium = StringField(required=False) # no max_length
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def db_connect(storage_ip, db_name, mongodb_admin, mongodb_password):
|
|
|
|
|
@ -52,7 +53,7 @@ def db_connect(storage_ip, db_name, mongodb_admin, mongodb_password):
|
|
|
|
|
password = urllib.parse.quote_plus(mongodb_password)
|
|
|
|
|
mongodb_url = f"mongodb://{username}:{password}@{storage_ip}:27017/"
|
|
|
|
|
#mongodb_url = "mongodb://{}:27017/".format(storage_ip)
|
|
|
|
|
logger.info("connection to the mongodb server")
|
|
|
|
|
#logger.info("connection to the mongodb server")
|
|
|
|
|
# pymongo settings
|
|
|
|
|
# myclient = pymongo.MongoClient(mongodb_url)
|
|
|
|
|
myclient = connect(db=db_name, host=mongodb_url, authentication_source='admin', alias="default")
|
|
|
|
|
@ -61,15 +62,10 @@ def db_connect(storage_ip, db_name, mongodb_admin, mongodb_password):
|
|
|
|
|
# pipeline functions
|
|
|
|
|
def populate_mongo(jsondoc: JSONDataSetCollection, storage_ip: str, db_name: str, db_collection_name: str, mongodb_admin: str, mongodb_password: str) -> None:
|
|
|
|
|
|
|
|
|
|
#logger.info(storage_ip)
|
|
|
|
|
#logger.info(db_name)
|
|
|
|
|
#logger.info(db_collection_name)
|
|
|
|
|
jsondatasets = jsondoc.datasets
|
|
|
|
|
housename = jsondoc._housename
|
|
|
|
|
|
|
|
|
|
db_connect(storage_ip, db_name, mongodb_admin, mongodb_password)
|
|
|
|
|
#actesdb = myclient[db_name]
|
|
|
|
|
#actes_collection = actesdb[db_collection_name]
|
|
|
|
|
|
|
|
|
|
for dataset_filenamestem, dataset in jsondatasets.items():
|
|
|
|
|
# a manual load is required here, because
|
|
|
|
|
@ -77,7 +73,7 @@ def populate_mongo(jsondoc: JSONDataSetCollection, storage_ip: str, db_name: str
|
|
|
|
|
json_document = dataset._load()
|
|
|
|
|
json_document["_id"] = json_document["filename"]
|
|
|
|
|
acte_entry = Acte(**json_document)
|
|
|
|
|
logger.info("adding entry: " + json_document["filename"])
|
|
|
|
|
#logger.info("adding entry: " + json_document["filename"])
|
|
|
|
|
acte_entry.save()
|
|
|
|
|
#res = actes_collection.insert_one(document)
|
|
|
|
|
#logger.info(res.inserted_id)
|
|
|
|
|
@ -92,7 +88,7 @@ def load_houses(yamldoc: YAMLDataSet, storage_ip: str, db_name: str, mongodb_adm
|
|
|
|
|
|
|
|
|
|
for house_dict in yamldoc['houses'].values():
|
|
|
|
|
house_dict['_id'] = house_dict['name']
|
|
|
|
|
logger.info("inserting: " + str(house_dict))
|
|
|
|
|
#logger.info("inserting: " + str(house_dict))
|
|
|
|
|
#houses_col.insert_one(house_dict)
|
|
|
|
|
house_entry = House(**house_dict)
|
|
|
|
|
house_entry.save()
|
|
|
|
|
|