first rev
gwen
2 years ago
commit
f2a6754021
@ -0,0 +1,2 @@
|
||||
.venv
|
||||
roles/archive/files/*
|
||||
@ -0,0 +1,4 @@
|
||||
[defaults]
|
||||
host_key_checking = false
|
||||
inventory = hosts
|
||||
|
||||
@ -0,0 +1,9 @@
|
||||
# development configuration
|
||||
domain_name: toto.site
|
||||
mail_address: toto@free.fr
|
||||
server_ip: XXXX
|
||||
dbadmin: XXXX
|
||||
dbpassword: XXXXXXXX
|
||||
linux_user: ubuntu
|
||||
application_release_tag: v0.20beta
|
||||
datascience_release_tag: v0.1pre-alpha
|
||||
@ -0,0 +1,5 @@
|
||||
# set the vps ip address or domain name
|
||||
|
||||
server1 ansible_host="{{ server_ip }}" ansible_ssh_user="{{ linux_user }}" ansible_python_interpreter="/usr/bin/python3"
|
||||
|
||||
|
||||
@ -0,0 +1,14 @@
|
||||
- hosts: server1
|
||||
#remote_user: debian
|
||||
become: true
|
||||
become_method: sudo
|
||||
|
||||
roles:
|
||||
- common
|
||||
- nginx
|
||||
- certbot
|
||||
- archive
|
||||
- pip
|
||||
- mongodb
|
||||
- datascience
|
||||
- run
|
||||
@ -0,0 +1,82 @@
|
||||
VPS installation procedure
|
||||
============================
|
||||
|
||||
Prerequisites
|
||||
------------------
|
||||
|
||||
You must have working copy repositories of the
|
||||
|
||||
- `deployment`
|
||||
- `webapp`
|
||||
|
||||
projects on your control node machine.
|
||||
|
||||
::
|
||||
|
||||
─ repositories
|
||||
├── deployment
|
||||
└── webapp
|
||||
|
||||
|
||||
.. important::
|
||||
|
||||
In the webapp project, before launching the installation procedure,
|
||||
make a `git pull --tags` to retrieve all the tags in the local
|
||||
working copy webapp repository.
|
||||
|
||||
|
||||
Before launching the installation
|
||||
-------------------------------------
|
||||
|
||||
You must have a `group_vars/all/main.yml` configuration file, wich is NOT
|
||||
in the working copy repository. Have a look at the `.gitignore` file.
|
||||
|
||||
Installation configuration
|
||||
-----------------------------------------------
|
||||
|
||||
You need to verify and set some variables before launching the playbook:
|
||||
|
||||
The `group_vars/all/main.yml` shall have these variables set :
|
||||
|
||||
- domain_name
|
||||
- mail_address
|
||||
- server_ip
|
||||
- dbadmin
|
||||
- dbpassword
|
||||
- application_release_tag
|
||||
|
||||
|
||||
Installation procedure
|
||||
-----------------------------------
|
||||
|
||||
From this `deployment` project, launch the script::
|
||||
|
||||
./install.sh
|
||||
|
||||
The script `install.sh` installs:
|
||||
|
||||
- nginx as a webserver
|
||||
- https (with a let's encrypt acme challenge)
|
||||
- usefull python librairies (flask, for example)
|
||||
- mongodb storage
|
||||
|
||||
Then go to the `datascience` repository and populate the database.
|
||||
When the database is populated, you can run the app service with::
|
||||
|
||||
./launch_application.sh
|
||||
|
||||
which lauches the webapp application service on the remote server.
|
||||
|
||||
Installation method
|
||||
----------------------
|
||||
|
||||
we use `ansible <https://www.ansible.com/>`_
|
||||
|
||||
The target is a VPS with a debian 12 installed, the python version is::
|
||||
|
||||
Python 3.11.2 (main, Mar 13 2023, 12:18:29) [GCC 12.2.0] on linux
|
||||
Type "help", "copyright", "credits" or "license" for more information.
|
||||
>>> import flask
|
||||
>>>
|
||||
|
||||
|
||||
@ -0,0 +1,12 @@
|
||||
ansible==8.3.0
|
||||
ansible-core==2.15.3
|
||||
cffi==1.15.1
|
||||
cryptography==41.0.3
|
||||
dnspython==2.4.2
|
||||
Jinja2==3.1.2
|
||||
MarkupSafe==2.1.3
|
||||
packaging==23.1
|
||||
pycparser==2.21
|
||||
pymongo==4.5.0
|
||||
PyYAML==6.0.1
|
||||
resolvelib==1.0.1
|
||||
@ -0,0 +1,23 @@
|
||||
Deployment from an archive
|
||||
=============================
|
||||
|
||||
ansible *Unarchive* deployment procedure
|
||||
|
||||
Create the git archive
|
||||
--------------------------
|
||||
|
||||
First, let's create the git archive from the actes princier's repository
|
||||
|
||||
git archive command::
|
||||
|
||||
git archive --format=tgz --prefix='app/' -o actesprinciers.tgz v0.2_maquette
|
||||
|
||||
Place the archive in your `files` folder
|
||||
-------------------------------------------
|
||||
|
||||
Application archive to be deployed shall be present in the `files` folder::
|
||||
|
||||
files/actesprinciers.tgz
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,25 @@
|
||||
- name: Deployment - Archive the app for deployment
|
||||
become: false
|
||||
ansible.builtin.shell: "git archive --format=tgz --prefix='app/' -o ../deployment/roles/archive/files/{{deployment_repo_name}}.tgz {{ release_tag }}"
|
||||
args:
|
||||
chdir: ../{{deployment_repo_name}}/
|
||||
delegate_to: 127.0.0.1
|
||||
|
||||
- name: Deployment - if exists - removes /opt/ directory
|
||||
shell: rm -rf /opt/
|
||||
|
||||
- name: Deployment - Creates /opt/ (application) directory
|
||||
file:
|
||||
path: /opt
|
||||
state: directory
|
||||
|
||||
- name: Deployment - extract application archive
|
||||
ansible.builtin.unarchive:
|
||||
src: "{{deployment_repo_name}}.tgz"
|
||||
dest: /opt/
|
||||
|
||||
- name: Deployment - copies the credentials file from the local app working copy repository
|
||||
ansible.builtin.copy:
|
||||
src: "../{{deployment_repo_name}}/params.yaml"
|
||||
dest: "/opt/app"
|
||||
mode: '0644'
|
||||
@ -0,0 +1,3 @@
|
||||
---
|
||||
release_tag: "{{ application_release_tag }}"
|
||||
deployment_repo_name: webapp
|
||||
@ -0,0 +1,7 @@
|
||||
certbot_site_names: {
|
||||
host1: "{{ domain_name }}",
|
||||
}
|
||||
# host2: "",
|
||||
certbot_package: "python3-certbot-nginx"
|
||||
certbot_plugin: "nginx"
|
||||
certbot_mail_address: "{{ mail_address }}"
|
||||
@ -0,0 +1,37 @@
|
||||
---
|
||||
- name: Update & upgrade system
|
||||
apt:
|
||||
update_cache: yes
|
||||
upgrade: dist
|
||||
|
||||
- name: Install required packages
|
||||
apt:
|
||||
name:
|
||||
- cron
|
||||
- python3-pip
|
||||
- python3-virtualenv
|
||||
- python3-setuptools
|
||||
- htop
|
||||
- man
|
||||
- net-tools
|
||||
- bash-completion
|
||||
- locales
|
||||
- python-is-python3
|
||||
- wget
|
||||
- zip
|
||||
- bzip2
|
||||
- tree
|
||||
- vim
|
||||
- vim-common
|
||||
- screen
|
||||
- curl
|
||||
- unzip
|
||||
state: present
|
||||
|
||||
- name: Remove useless stuff
|
||||
apt:
|
||||
name:
|
||||
- bind9
|
||||
- telnet
|
||||
- ftp
|
||||
state: absent
|
||||
@ -0,0 +1 @@
|
||||
consent: false
|
||||
@ -0,0 +1,23 @@
|
||||
Deployment from an archive
|
||||
=============================
|
||||
|
||||
ansible *Unarchive* deployment procedure
|
||||
|
||||
Create the git archive
|
||||
--------------------------
|
||||
|
||||
First, let's create the git archive from the actes princier's repository
|
||||
|
||||
git archive command::
|
||||
|
||||
git archive --format=tgz --prefix='datascience/' -o datascience.tgz <tag_name>
|
||||
|
||||
Place the archive in your `files` folder
|
||||
-------------------------------------------
|
||||
|
||||
Application archive to be deployed shall be present in the `files` folder::
|
||||
|
||||
files/datascience.tgz
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,100 @@
|
||||
- name: Deployment - Archive datascience for pipeline run on the server
|
||||
become: false
|
||||
ansible.builtin.shell: "git archive --format=tgz --prefix='datascience/' -o ../deployment/roles/datascience/files/{{datascience_repo_name}}.tgz {{ release_tag }}"
|
||||
args:
|
||||
chdir: ../{{datascience_repo_name}}/
|
||||
delegate_to: 127.0.0.1
|
||||
|
||||
- name: Deployment - removes old datascience directory
|
||||
shell: rm -rf /home/{{ user }}/datascience
|
||||
|
||||
- name: Deployment - Creates datascience directory
|
||||
become: false
|
||||
file:
|
||||
path: /home/{{ user }}/datascience
|
||||
state: directory
|
||||
|
||||
- name: Deployment - extract datascience archive
|
||||
become: false
|
||||
ansible.builtin.unarchive:
|
||||
src: "{{datascience_repo_name}}.tgz"
|
||||
dest: /home/{{ user }}/
|
||||
|
||||
- name: Deployment - copies the credentials file from the local datascience working copy repository
|
||||
become: false
|
||||
ansible.builtin.copy:
|
||||
src: "../{{datascience_repo_name}}/actes-princiers/conf/local/parameters.yml"
|
||||
dest: "/home/{{ user }}/datascience/actes-princiers/conf/local/"
|
||||
mode: '0644'
|
||||
|
||||
#- name: Drop all collections in the mongo database
|
||||
# become: false
|
||||
# ansible.builtin.script:
|
||||
# executable: python3
|
||||
# cmd: "drop_database.py {{ mongodb_ip }} {{ mongodb_admin }} {{mongodb_password}}"
|
||||
# delegate_to: 127.0.0.1
|
||||
# ignore_errors: true
|
||||
|
||||
- name: Create working directory for mongo admin scripts
|
||||
become: false
|
||||
#become_user: "{{ user }}"
|
||||
ansible.builtin.file:
|
||||
path: /home/{{ user }}/tmp/
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Installing workplace script librairies
|
||||
become: false
|
||||
ansible.builtin.pip:
|
||||
name: pymongo
|
||||
virtualenv: /home/{{ user }}/tmp/.venv
|
||||
|
||||
- name: Upload drop_database python script
|
||||
become: false
|
||||
ansible.builtin.copy:
|
||||
src: files/drop_database.py
|
||||
dest: "/home/{{ user }}/tmp/"
|
||||
mode: '0755'
|
||||
|
||||
- name: Run drop_database script
|
||||
become: false
|
||||
ansible.builtin.shell: "cd /home/{{ user }}/tmp && . .venv/bin/activate && ./drop_database.py {{ mongodb_ip }} {{ mongodb_admin }} {{mongodb_password}}"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
ignore_errors: true
|
||||
|
||||
- name: Install python librairies into the specified virtual environment
|
||||
become: false
|
||||
ansible.builtin.pip:
|
||||
requirements: /home/{{ user }}/datascience/actes-princiers/src/requirements.txt
|
||||
virtualenv: /home/{{ user }}/datascience/.venv
|
||||
|
||||
#- name: Uninstall kedro-telemetry
|
||||
# become: false
|
||||
# ansible.builtin.pip:
|
||||
# name: kedro-telemetry
|
||||
# virtualenv: /home/{{ user }}/datascience/.venv
|
||||
# state: absent
|
||||
|
||||
- name: Kedro - copy telemetry file
|
||||
become: false
|
||||
ansible.builtin.copy:
|
||||
src: files/telemetry
|
||||
dest: "/home/{{ user }}/datascience/actes-princiers/.telemetry"
|
||||
mode: '0644'
|
||||
|
||||
- name: Install python librairies into the specified virtual environment
|
||||
become: false
|
||||
ansible.builtin.pip:
|
||||
requirements: /home/{{ user }}/datascience/actes-princiers/src/requirements.txt
|
||||
virtualenv: /home/{{ user }}/datascience/.venv
|
||||
|
||||
- name: Launches the kedro JSON creation pipeline and populates the database
|
||||
become: false
|
||||
ansible.builtin.shell: |
|
||||
cd /home/{{ user }}/datascience/ && . .venv/bin/activate && cd actes-princiers && kedro run --tags="etl_transform" && kedro run --tags="populate_database"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
# chdir: /home/{{ user }}/datascience/actes-princiers/
|
||||
# executable: /home/{{ user }}/datascience/.venv/bin/kedro
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
---
|
||||
release_tag: "{{ datascience_release_tag }}"
|
||||
datascience_repo_name: datascience
|
||||
user: "{{ linux_user }}"
|
||||
mongodb_ip: 127.0.0.1
|
||||
mongodb_admin: "{{ dbadmin }}"
|
||||
mongodb_password: "{{ dbpassword }}"
|
||||
@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""Mongo create admin user utility
|
||||
"""
|
||||
import sys
|
||||
import urllib.parse
|
||||
|
||||
import pymongo
|
||||
|
||||
|
||||
mongo_ip = sys.argv[1]
|
||||
mongo_admin = sys.argv[2]
|
||||
mongo_password = sys.argv[3]
|
||||
|
||||
#mongo_admin = urllib.parse.quote_plus(mongo_admin)
|
||||
#mongo_password = urllib.parse.quote_plus(mongo_password)
|
||||
|
||||
|
||||
client = pymongo.MongoClient(f"mongodb://{mongo_ip}:27017/")
|
||||
|
||||
client.admin.command(
|
||||
'createUser', mongo_admin,
|
||||
pwd=mongo_password,
|
||||
roles=[ { 'role': "userAdminAnyDatabase", 'db': "admin" }, "readWriteAnyDatabase" ]
|
||||
)
|
||||
|
||||
@ -0,0 +1,4 @@
|
||||
user: "{{ linux_user }}"
|
||||
mongodb_ip: 127.0.0.1
|
||||
mongodb_admin: "{{ dbadmin }}"
|
||||
mongodb_password: "{{ dbpassword }}"
|
||||
@ -0,0 +1,5 @@
|
||||
- name: restart nginx
|
||||
service:
|
||||
name: nginx
|
||||
state: restarted
|
||||
|
||||
@ -0,0 +1,30 @@
|
||||
---
|
||||
- name: Install Nginx
|
||||
apt:
|
||||
name:
|
||||
- nginx
|
||||
state: present
|
||||
|
||||
#- name: "create www directory"
|
||||
# file:
|
||||
# path: /var/www/{{ domain }}
|
||||
# state: directory
|
||||
# mode: '0775'
|
||||
# owner: "{{ ansible_user }}"
|
||||
# group: "{{ ansible_user }}"
|
||||
|
||||
- name: delete default nginx site
|
||||
file:
|
||||
path: /etc/nginx/sites-enabled/default
|
||||
state: absent
|
||||
notify: restart nginx
|
||||
|
||||
- name: copy nginx site.conf
|
||||
template:
|
||||
src: templates/site.conf.j2
|
||||
dest: /etc/nginx/sites-enabled/{{ domain }}
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
notify: restart nginx
|
||||
|
||||
@ -0,0 +1,2 @@
|
||||
---
|
||||
domain: "{{ domain_name }}"
|
||||
@ -0,0 +1,6 @@
|
||||
requirements
|
||||
pip
|
||||
virtualenv
|
||||
setuptools
|
||||
|
||||
|
||||
@ -0,0 +1,38 @@
|
||||
#- name: Execute the flask init command
|
||||
# ansible.builtin.shell: |
|
||||
# source bootstrap.sh
|
||||
# flask db init
|
||||
# args:
|
||||
# chdir: /opt/app/
|
||||
# creates: actes_princiers.sqlite
|
||||
# executable: /usr/bin/bash
|
||||
|
||||
#- name: Start the flask run application
|
||||
# ansible.builtin.shell: |
|
||||
# source bootstrap.sh
|
||||
# flask run &
|
||||
# args:
|
||||
# chdir: /opt/app/
|
||||
# executable: /usr/bin/bash
|
||||
|
||||
- name: Template a file to /etc/file.conf
|
||||
ansible.builtin.template:
|
||||
src: templates/princelyacts.service.jinja
|
||||
dest: /etc/systemd/system/princelyacts.service
|
||||
#owner: "{{ system_user }}"
|
||||
#group: "{{ system_user }}"
|
||||
mode: '0777'
|
||||
|
||||
- name: start systemd app service
|
||||
systemd: name=princelyacts.service state=restarted enabled=yes
|
||||
|
||||
- name: check if flask app runs
|
||||
ansible.builtin.shell: netstat -tulnp | grep :5000
|
||||
register: flask_status
|
||||
|
||||
- name: check if flask app is up
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- flask_status != 0
|
||||
fail_msg: "flask web application service is down (status:{!"
|
||||
success_msg: "flask web application is up and running..."
|
||||
@ -0,0 +1,2 @@
|
||||
system_user: "{{ linux_user }}"
|
||||
|
||||
Loading…
Reference in New Issue