diff --git a/app/cmd/db.py b/app/cmd/db.py index 72335c2..511d9c4 100644 --- a/app/cmd/db.py +++ b/app/cmd/db.py @@ -10,7 +10,7 @@ from tqdm import tqdm from app.app import APPPATH, db from app.data_actes import diplomatic_type, institution, state, houses, interventions -from app.modeles import Institution, State, House, Intervention_type, Production_place, Diplo_type, Document, Acte, Individual, Produced_by +from app.modeles import Institution, State, House, Intervention_type, Production_place, Diplo_type, Document, Acte, Individual, Involved_in db_cli = AppGroup("db") @@ -171,50 +171,66 @@ def _create_indiv(list_csv): for data in tqdm(individuals, desc="Populating Individual..."): Individual.create(**data) -def _create_produced_by(xml_file: str, folder: str): - princes_actes = [] - for acte in os.listdir(folder): - acte_q = [t.id_acte for t in Acte.select().where( - Acte.numb_acte == acte.replace(".xml", ""))] - # print("----") - # print(acte, "==", acte_q[0]) - soup = make_soup(os.path.join(folder, acte)) - princes = soup.sourceDesc.find_all("listPerson", {"type": "prince"}) - for prince in princes: - dukes = prince.find_all("person") - for duke in dukes: - prince = duke.text.replace("\n", "") +def __grape_indiv(list_person, role: str): + for persons in list_person: + for person_tag in persons.find_all("person"): + person_text = person_tag.text.replace("\n", "") + if person_text != "None": prince_q = [t.id_indiv for t in Individual.select().where( - Individual.name_indiv == duke.text.replace("\n", ""))] + Individual.name_indiv == person_text)] # duke_q = [t.id_duke for t in Duke.select().where( # Duke.indiv_duke == prince_q[0])] - # print(prince, "==", prince_q[0]) - princes_actes.append({"produced_by_acte": acte_q[0], - "produced_by_prince": prince_q[0]}) - for data in tqdm(princes_actes, desc="Populating Produced_by..."): - Produced_by.create(**data) - -def _create_signed_by(xml_file: str, folder: str): - sign_actes = [] + print(person_text, "==", prince_q[0]) + + + +def _create_involved_in(xml_file: str, folder: str): + princes_actes = [] for acte in os.listdir(folder): acte_q = [t.id_acte for t in Acte.select().where( Acte.numb_acte == acte.replace(".xml", ""))] # print(acte, "==", acte_q[0]) soup = make_soup(os.path.join(folder, acte)) - signatures = soup.sourceDesc.find_all("listPerson", {"type": "signatory"}) - for signature in signatures: - signatories = signature.find_all("person") - for signatory in signatories: - signator = signatory.text.replace("\n", "") - if "None" != signator: - # signator = signatory.text.replace("\n", "") - sign_q = [t.id_indiv for t in Individual.select().where( - Individual.name_indiv == signator)] - # print(signator, "==", sign_q) - sign_actes.append({"signed_by_acte": acte_q[0], - "signed_by_prince": sign_q[0]}) - for data in tqdm(sign_actes, desc="Populating Signed_by..."): - Signed_by.create(**data) + for persons in soup.sourceDesc.find_all("listPerson", {"type": "prince"}): + for person_tag in persons.find_all("person"): + person_text = person_tag.text.replace("\n", "") + if person_text != "None": + prince_q = [t.id_indiv for t in Individual.select().where( + Individual.name_indiv == person_text)] + interv_q = [t.id_intev for t in Intervention_type.select().where( + Intervention_type.interv_label == "producer")] + # print(person_text, "==", prince_q[0]) + + try: + prince_q[0] + except IndexError: + print("!! name " + person_text + " (prince) not found in /app/static/csv/actors.csv") + continue + + princes_actes.append({"involved_in_acte": acte_q[0], + "involved_in_prince": prince_q[0], + "invol_in_interv": interv_q[0]}) + for persons in soup.sourceDesc.find_all("listPerson", {"type": "signatory"}): + for person_tag in persons.find_all("person"): + person_text = person_tag.text.replace("\n", "") + if person_text != "None": + prince_q = [t.id_indiv for t in Individual.select().where( + Individual.name_indiv == person_text)] + interv_q = [t.id_intev for t in Intervention_type.select().where( + Intervention_type.interv_label == "signatory")] + # print(person_text, "==", prince_q[0]) + + try: + prince_q[0] + except IndexError: + print("!! name " + person_text + " (signatory) not found in /app/static/csv/actors.csv") + continue + + princes_actes.append({"involved_in_acte": acte_q[0], + "involved_in_prince": prince_q[0], + "invol_in_interv": interv_q[0]}) + for data in tqdm(princes_actes, desc="Populating involved_in..."): + Involved_in.create(**data) @db_cli.command() @@ -228,10 +244,10 @@ def init() -> None: print("Dropping existing DB...") db.drop_tables([Institution, State, House, Intervention_type, Production_place, Diplo_type, Document, Acte, Individual, - Produced_by]) + Involved_in]) print("Re-creating schema...") db.create_tables([Institution, State, House, Intervention_type, - Production_place, Diplo_type, Document, Acte, Individual, Produced_by]) + Production_place, Diplo_type, Document, Acte, Individual, Involved_in]) _create_institution(institution) _create_state(state) _create_house(houses) @@ -242,9 +258,9 @@ def init() -> None: _create_acte(xml, xml_folder) # check which names need to be add to the actors.csv actors = [*__csv_indiv_infos("secret"), *__csv_indiv_infos("prince")] - names_in_csv = [actor[0] for actor in actors] - names_in_xml = __find_indiv(xml_folder, "signatory") - for name in [x for x in names_in_xml if x not in names_in_csv]: - print("!! name " + name + " not found in /app/static/csv/actors.csv") + # names_in_csv = [actor[0] for actor in actors] + # names_in_xml = __find_indiv(xml_folder, "signatory") + # for name in [x for x in names_in_xml if x not in names_in_csv]: + # print("!! name " + name + " not found in /app/static/csv/actors.csv") _create_indiv(actors) - _create_produced_by(xml, xml_folder) + _create_involved_in(xml, xml_folder) diff --git a/app/modeles/__init__.py b/app/modeles/__init__.py index 5891a1c..cfc9f2d 100644 --- a/app/modeles/__init__.py +++ b/app/modeles/__init__.py @@ -1,3 +1,3 @@ -from .data import Institution, State, House, Intervention_type, Production_place, Diplo_type, Document, Acte, Individual, Produced_by +from .data import Institution, State, House, Intervention_type, Production_place, Diplo_type, Document, Acte, Individual, Involved_in -__all__ = ["Institution", "State", "House", "Intervention_type", "Production_place", "Diplo_type", "Document", "Acte", "Individual", "Produced_by"] +__all__ = ["Institution", "State", "House", "Intervention_type", "Production_place", "Diplo_type", "Document", "Acte", "Individual", "involved_in"] diff --git a/app/modeles/data.py b/app/modeles/data.py index e513d8d..cb4ede3 100644 --- a/app/modeles/data.py +++ b/app/modeles/data.py @@ -55,7 +55,7 @@ class House(BaseModel): db_table = 'House' class Intervention_type(BaseModel): - id_house = peewee.AutoField() + id_intev = peewee.AutoField() interv_label = peewee.TextField() class Meta: @@ -119,20 +119,12 @@ class Individual(BaseModel): database = db db_table = 'Individual' -class Produced_by(BaseModel): - id_produced_by = peewee.AutoField() - produced_by_acte = peewee.ForeignKeyField(Acte, backref='produced_bys') - produced_by_prince = peewee.ForeignKeyField(Individual, backref='produced_bys') - - class Meta: - database = db - db_table = 'Produced_by' - -class Signed_by(BaseModel): - id_signed_by = peewee.AutoField() - signed_by_acte = peewee.ForeignKeyField(Acte, backref='signed_bys') - signed_by_prince = peewee.ForeignKeyField(Individual, backref='signed_bys') +class Involved_in(BaseModel): + id_involved_in = peewee.AutoField() + involved_in_acte = peewee.ForeignKeyField(Acte, backref='involved_ins') + involved_in_prince = peewee.ForeignKeyField(Individual, backref='involved_ins') + invol_in_interv = peewee.ForeignKeyField(Intervention_type, backref='involved_ins') class Meta: database = db - db_table = 'Signed_by' + db_table = 'involved_in' diff --git a/app/routes.py b/app/routes.py index c993a20..c8969a3 100644 --- a/app/routes.py +++ b/app/routes.py @@ -7,7 +7,7 @@ from flask import Blueprint, abort, render_template, request, send_from_director from playhouse.flask_utils import PaginatedQuery from .app import APPPATH -from .modeles import Institution, State, House, Intervention_type, Production_place, Diplo_type, Document, Acte, Individual, Produced_by +from .modeles import Institution, State, House, Intervention_type, Production_place, Diplo_type, Document, Acte, Individual, Involved_in RESULT_PAR_PAGES = 5