Commit 5b6dfb1b authored by Nelso Jost's avatar Nelso Jost

FIX: deployment operations

parent 555b59d2
PY := python3
BOARD_ID := 2
PORT := 8000
SERVER := nginx-gunicorn
HOST := localhost
VENV := .venv
VENVPY := ${VENV}/bin/python
PYBIN := python3
VENVDIR := $(shell pwd)/.venv
VENVPY := ${VENVDIR}/bin/python
PORT := 5000
all: help
help:
@ echo "USAGE:"
@ echo "------"
@ echo "1. Prepare the ambient with (only first time):"
@ echo " "
@ echo " make install # assure necessary system tools (via apt-get)"
@ echo " make setup # create a virtualenv and init the database"
@ echo " "
@ echo "2. Run the server with one of the options:"
@ echo " "
@ echo " make run # for development (default port: 5000)"
@ echo " make deploy # for production (default port: 8000)"
@ echo " make deploy PORT=9999 # specify another port (wsgi)"
@ echo " "
@ echo "-------------------"
@ echo "Aditional commands:"
@ echo " "
@ echo " make venv # (re)create the virtualenv (under /venv)"
@ echo " make venv PY=python # specify another interpreter (here 2.x)"
@ echo " "
@ echo " make initdb # (re)create the database"
@ echo " "
@ echo " make shell # launch ipython3 -i (use to explore the DB)"
@ echo " "
@ echo " make fakelog # sends fake data to the server"
@ echo " make fakelog BOARD_ID=3 # specify another board (default: 2)"
@ echo " "
@ echo " make tree # show a nice directory tree of the project"
@ echo " "
@ echo " make help # show this message"
apt-install:
sudo apt-get install nginx python3 python3-pip supervisor
@ echo "USAGE: make <target> where <target> can be"
@ echo ""
@ echo " debreq apt-get install Debian-based system dependencies"
@ echo " venv create local Python virtualenv with PYBIN variable"
@ echo ""
@ echo " mysql-setup create MySQL user ctaemm and emmdb (not tables)"
@ echo " mysql-info return MySQL info: users, dbs and emmdb usage"
@ echo " mysql-chroot change MySQL root password"
@ echo " mysql-enter starts MySQL interactive session as root"
@ echo " mysql-schema shows the database schema"
@ echo ""
@ echo " dbinit recreate database from scratch (drop data) and populate it"
@ echo " shell start ipython3 interactively for high level DB admin"
@ echo ""
@ echo " run start Flask's development server at"
@ echo " http://localhost:5000 (DEBUG=True)"
@ echo " options: PORT=___ (default: 5000)"
@ echo ""
@ echo " deploy register supervisor dameon for gunicorn WSGI app"
@ echo " and production ready server nginx"
@ echo " undeploy undo the previous operation"
debreq:
sudo apt-get install nginx python3 python3-pip supervisor mysql-server
sudo pip3 install virtualenv
sudo pip3 install --upgrade pip
setup:
venv:
virtualenv -v --python='${PYBIN}' ${VENVDIR}
${VENVDIR}/bin/pip3 install --upgrade pip
${VENVDIR}/bin/pip3 install -r requirements.pip
@ echo "-------------------------------------------------------"
virtualenv -v --python='${PY}' ${VENV}
@ echo "Virtualenv with interpreter '${PY}' was created at ${VENV}/"
@ echo "-------------------------------------------------------"
${VENV}/bin/pip3 install --upgrade pip
@ echo "-------------------------------------------------------"
${VENV}/bin/pip3 install -r requirements.pip
@ echo "-------------------------------------------------------"
@ echo "Virtualenv is ready at ${VENV}/!"
@ echo " "
@ echo "TOTAL SIZE: "
@ du -sh ${VENV}
@ echo "Virtualenv successfully created at"
@ du -sh ${VENVDIR}
setup: debreq venv
clean-venv:
rm -rf ${VENV}
rm -rf ${VENVDIR}
initdb:
@ echo "-------------------------------------------------------"
${VENVPY} manage.py initdb
dbinit:
${VENVPY} manage.py dbinit
list-routes:
${VENVPY} manage.py list_routes
run:
@ echo "-------------------------------------------------------"
${VENVPY} manage.py runserver --host ${HOST} --port 5000
${VENVPY} manage.py runserver --port ${PORT}
shell:
${VENV}/bin/ipython manage.py shell
${VENVDIR}/bin/ipython manage.py shell
fakelog:
${VENVPY} tests/fakemeteorolog.py ${BOARD_ID}
tree:
tree . -I "${VENV}|__pycache__|*.pyc"
tree . -I "${VENVDIR}|__pycache__|*.pyc"
mysql-init:
${VENVPY} production/manage_mysql.py init
mysql-getinfo:
${VENVPY} production/manage_mysql.py getinfo
mysql-client:
mysql -u root -p
mysql-showschema:
${VENVPY} production/manage_mysql.py showschema
mysql-chroot:
${VENVPY} production/mysqlsetup.py chroot
mysql-resetroot:
sudo dpkg-reconfigure mysql-server-5.5
mysql-setup:
${VENVPY} production/mysqlsetup.py setup
mysql-describe:
chmod +x production/mysql_describe.sh && ./production/mysql_describe.sh
mysql-info:
${VENVPY} production/mysqlsetup.py info
mysql-enter:
mysql -uroot -p
gunicorn:
cd production/nginx-gunicorn && ${VENVDIR}/bin/gunicorn wsgi:app
deploy:
sudo ${VENVPY} production/nginx-gunicorn/deploy.py --port=${PORT}
sudo ${VENVPY} production/nginx-gunicorn/deploy.py
undeploy:
sudo ${VENVPY} production/nginx-gunicorn/deploy.py -u
clean-logs-nginx:
rm -rf production/nginx-gunicorn/logs
mkdir -p production/nginx-gunicorn/logs
reset-settings:
python3 scripts/reset_settings.py
clean-logs:
rm -rf logs/*
clean-all: clean-venv
clean: clean-venv reset-settings
py3clean app
rm -f app/db.sqlite
rm -rf doc/_build
logtail-nginx-access:
tail production/nginx-gunicorn/logs/nginx-access.log
logtail-nginx-error:
tail production/nginx-gunicorn/logs/nginx-error.log
# cta-emm-web
Estações Meteorológicas Modulares Web App - UFRGS CTA
(Modular Meteorologic Stations Web App)
# emm-webapp
This project consists of a web application with purpose of storing and managing open meteorological data, to be hosted on a server in the UFRGS brazilian university. Primarly the app will offer full support for the [CTA Meteorolog Arduino](http://cta.if.ufrgs.br/projects/estacao-meteorologica-modular) station prototype, along with a RESTful API that allows anyone to upload meteorological data using their own meteorological station.
Este projeto consiste em uma aplicação web responsável pelo armazenamento e gerenciamento dos dados meteorológicos abertos do projeto EMM, atualmente publicada em:
* [dados.cta.if.ufrgs.br/emm](http://dados.cta.if.ufrgs.br/emm/)
The software is being developed using the language Python 3 and the [Flask](http://flask.pocoo.org/) web framework.
O protótipo inicial conta com uma [API RESTful](https://en.wikipedia.org/wiki/Representational_state_transfer) simples, desenvolvida em [Flask](http://flask.pocoo.org/), para coletar dados enviados pelo software [loggger](https://git.cta.if.ufrgs.br/meteorolog/logger) e armazená-los em um servidor do [CTA](http://cta.if.ufrgs.br/), na UFRGS.
## Installation and Usage (via Makefile)
## Instalação e uso
The accompaning Makefile provides quick commands to make all the preparations needed to develop, test and deploy the web app.
### 1. Prepare the ambient
O projeto inclui um arquivo `Makefile` contendo diversos comandos rápidos para realização das principais operações de instalação e manutenção da web app e servidor de dados.
Para executar comandos do `Makefile` é preciso estar dentro da raiz do repositório:
```
$ git clone https://git.cta.if.ufrgs.br/meteorolog/cta-emm-web.git
$ cd cta-emm-web
$ make install # uses apt-get to install required software system-wide
$ make setup # creates a virtualenv (for local requirements) and initiate the database
~/emm-webapp $ ls
app doc Makefile production requirements.pip
data logs manage.py README.md tests
~/emm-webapp $ make <comando>
```
Execute `make` ou `make help` para obter uma lista dos comandos disponíveis.
* OBS: Debian packages to be installed with `make install` via `apt-get`:
* **nginx**: reverse proxy web server (for serving static files);
* **supervisor**: allows easy monitoring and restarting of the gunicorn process;
* **python3**: interpreter wich will be runing the app;
* **python3-pip**: Python's own package manager;
* **virtualenv** (installed via pip): allows the creation of a local, custom, Python installation, which contains all required Python libraries for the project (see the file `requirements.pip`);
### 2. Run the server
### 1. Preparando ambiente de execução
* **Development**:
* **1.1** Obtenha uma cópia do repositório atual:
```
$ make run # launch Flask's werkzeug development server (DEBUG=True)
$ git clone https://git.cta.if.ufrgs.br/meteorolog/emm-webapp.git
$ cd emm-webapp
```
Check under http://localhost:5000. By default runs with `DEBUG=True`, which means that the server will reload automatically on source code changes.
Hit CTRL + C to cancel the execution.
* **Production**:
* **1.2** Instale os programas necessários (requer permissão de root):
```
$ make deploy # register site on nginx and launch a gunicorn process via supervisor
$ make deb-req
```
Check under http://localhost or the registered domain. Default port: 8000.
* Alternativaly, you can specify another port:
Os seguintes programas serão instalados no sistema via apt-get (logo, apenas para distribuições Debian):
* **nginx**: Servidor web, utilizado para reverse proxy;
* **supervisor**: Gerenciador de processos em background;
* **python3**: Interpretador da linguagem na qual a web app foi escrita;
* **python3-pip**: Gerenciador de pacotes do Python;
* **virtualenv**: Permite a criação de um ambiente virtual contendo uma instalação isolada de Python e das dependencias do projeto (ver arquivo `requirements.pip`);
* **1.3** Crie o ambiente virtual de Python onde a web app será executada:
```
$ make deploy PORT=9090
$ make venv
```
* Undo the production deployment and quit related processeses with:
* **1.4** Exporte uma frase-chave para ser usada como token de segurança da web app:
```
$ make undeploy
$ export EMMPASS=<DigiteUmaFraseSemEspaços>
```
OBS: dê um espaço em branco depois do `$` antes de digitar o comando `export` para que o texto não seja gravado no histórico do terminal!
### 3. Additional commands
### 2. Preparando o banco de dados
* Explore the database (see [SQLAlchemy](http://www.sqlalchemy.org/)) with:
```
$ make shell
venv/bin/ipython manage.py shell
É possível trabalhar em modo desenvolvimento ou produção configurando a variável ambiente `EMMCONFIG`.
In [1]: db
Out[1]: <SQLAlchemy engine='sqlite:////.../cta-emm-web/app/db.sqlite'>
#### 2.1 Desenvolvimento: SQLite, arquivo local
In [2]: boards.query.all()
Out[2]: [<Board id=1 nickname="Pezzi">, <Board id=2 nickname="CAp">]
```
$ export EMMCONFIG=dev
$ make initdb
```
## Installation and Usage (without Makefile)
OBS: Not required if you managed to work with the Makefile (previous section).
#### 2.2 Produção: servidor MySQL
### 1. Prepare the ambient
O `Makefile` proporciona vários comandos para execução de scripts MySQL (ver `make help` e `production/manage_mysql.py` para maiores detalhes).
* Clone the repository:
Necessário apenas uma vez, o comando seguinte cria o usuário `ctaemm` e o banco `emmdb` no servidor:
```
$ git clone https://git.cta.if.ufrgs.br/meteorolog/cta-emm-web.git
$ cd cta-emm-web
$ make mysql-setup
```
OBS: Nesse o ponto o banco ainda está vazio (sem tabelas).
* Install required software system-wide:
```
$ sudo apt-get install nginx python3 python3-pip supervisor
$ export EMMCONFIG=prod
$ make initdb
```
OBS: As tabelas serão recriadas e dados existentes serão perdidos!
* Create the virtual environment:
```
$ virtualenv -v --python='python3' venv
```
### 3. Executando a web app
* Activate the virtual environment:
```
$ source venv/bin/activate
```
É possível executar a aplicação em modo desenvolvimento ou produção. Em ambos os casos, é um servidor WSGI rodando em uma porta específica que intermedia as requisições web (GET, POST, etc) com a aplicação Python.
* Install Python requirements:
```
(venv) $ pip3 install --upgrade pip
(venv) $ pip3 install -r requirements.pip
```
A porta pode (e deve) ser configurada na variável ambiente `EMMPORT`. Padrão: 5000.
* Initialize the database:
```
(venv) $ python manage.py initdb
```
#### 3.1 Desenvolvimento
### 2. Run the server
* Execução em primeiro plano;
* Servidor WSGI: Werkzeug (padrão integrado ao Flask);
* Flag `DEBUG=True` (brecha de segurança!);
* Banco de dados local SQLite;
* **Development**:
Execute fazendo:
```
(venv) $ python manage.py runserver
$ make run
```
Check under http://localhost:5000. By default runs with `DEBUG=True`.
* **Production**:
Publicado por padrão em: `http://localhost:5000`.
Pressione CTRL+C para interromper a execução.
#### 3.2 Produção
* Execução em plano de fundo;
* Servidor WSGI: Gunicorn (instalado via pip em `make venv`);
* Flag `DEBUG=False`;
* Servidor de banco de dados MySQL;
Requer variável ambiente `EMMSERVER` contendo o nome do servidor de rede da máquina.
A publicação é feita com:
```
(venv) $ mkdir -p production/nginx-gunicorn/logs
(venv) $ python production/nginx-gunicorn/deploy.py
$ make deploy
```
Check under http://localhost or the registered domain.
* Undo the deployment and quit related processeses with:
```
(venv) $ python production/nginx-gunicorn/undeploy.py
```
Assumindo `EMMSERVER=localhost`, o site estará publicado localmente em:
* http://localhost/emm
### 3. Additional commands
Em resumo, o comando acima faz:
* Cria um servidor WSGI Gunicorn na porta `EMMPORT`;
* Registra o processo Gunicorn na ferramenta Supervisor para iniciação automática junto ao sistema operacional;
* Registra o site no servidor web Nginx na forma de um reverse proxy da porta 80 para a aplicação WSGI na porta `EMMPORT`.
* Explore the database (see [SQLAlchemy](http://www.sqlalchemy.org/)) with:
A publicação pode ser desfeita com:
```
(venv) $ python manage.py shell
$ make undeploy
```
* Quit the virtual environment with:
### 4. Manutenção
Independente do banco de dados utilizado, ele pode ser manipulado em alto nível graças ao mapeador objeto-relacional [SQLAlchemy](http://www.sqlalchemy.org/) utilizado pela aplicação.
O comando:
```
$ make shell
```
(venv) $ deactivate
inica uma sessão IPython onde as tabelas e registros podem ser acessados e alterados como objetos Python. A exemplo:
```
>>> tables # para ver as tabelas disponíveis
>>> boards # a exemplo, esta é a tabela boards
app.models.board.Board
>>> boards.__fields__ # lista os campos da tabela
{'description', 'id', 'latitude', 'longitude', 'nickname', 'user_id'}
>>> boards.ls # lista todas linhas/registros da tabela
[<Board id=1 user='admin' nickname='Pezzi' sensor_count=4>,
<Board id=2 user='admin' nickname='CAP' sensor_count=4>]
>>> b = boards.get(2) # pega um registro via ID
>>> b.nickname = 'UFRGS' # altera um campo
>>> commit() # confirma a operação no DB
>>> boards.ls
[<Board id=1 user='admin' nickname='Pezzi' sensor_count=4>,
<Board id=2 user='admin' nickname='UFRGS' sensor_count=4>]
```
......@@ -17,6 +17,7 @@ This approach has several advantages:
* Allows creation of multiple instances -- better for unit testing.
'''
from .config import EMMCONFIG
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
......@@ -26,8 +27,6 @@ from flask.ext.mail import Mail
import json
import os
def path_here(x):
return os.path.join(os.path.abspath(os.path.dirname(__file__)), x)
db = SQLAlchemy()
bootstrap = Bootstrap()
......@@ -37,27 +36,20 @@ login_manager = LoginManager()
login_manager.session_protection = 'strong'
login_manager.login_view = 'auth.login'
def create_app(config_name):
def create_app():
''' Creates the Flask application object and register configurations,
blueprints and related global objects for it.
'''
app = Flask(__name__)
from .config import config
app.config.from_object(config[config_name])
config[config_name].init_app(app)
from .config import config_select
app.config.from_object(config_select[EMMCONFIG])
config_select[EMMCONFIG].init_app(app)
with open(path_here('../data/DB_INIT.json'), encoding='utf-8') as f:
app.config['DB_INIT'] = json.loads(f.read())
from .controllers import blueprints
from .main import main as main_blueprint
app.register_blueprint(main_blueprint)
from .auth import auth as auth_blueprint
app.register_blueprint(auth_blueprint, url_prefix='/auth')
from .api import api as api_blueprint
app.register_blueprint(api_blueprint, url_prefix='/api')
for blueprint in blueprints:
app.register_blueprint(blueprint)
db.init_app(app)
bootstrap.init_app(app)
......
from flask import Blueprint
api = Blueprint('api', __name__)
from . import views
from . import api
from ..models import *
from .. import db
from flask import jsonify, request
import json
from datetime import datetime
@api.route('/post/rawsensordata/<int:board_id>', methods=['POST'])
def api_post_rawsensordata(board_id):
''' Expects a request with valid JSON attribute containing sensor data.
Returns a JSON response with success or error description.
Expected JSON format:
{'datetime': {'value': str, 'format': str},
'sensors': {'<nickname_1>': number,
'<nickname_2>': number,
...,
'<nickname_N>': number}}
'''
try:
board = Board.get(board_id)
except:
return jsonify({'error': 'Invalid board ID'})
if not request.json:
return jsonify({'error':
'Request does not have a valid JSON attribute.'})
r = request.json
if type(r) is str:
r = json.loads(request.json)
if not 'user_hash' in r or r['user_hash'] != board.user.password_hash:
return jsonify({'error': 'Missing or invalid JSON "user_hash" '
'attribute for board of ID {}.'.format(board_id)})
for d in r['data']:
if not 'datetime' in d:
return jsonify({'error': "JSON missing or invalid datetime attribute."
" Expeted format: {'value': str, 'format': str, 'source': str}"})
try:
dt = datetime.strptime(d['datetime']['value'],
d['datetime']['format'])
except:
return jsonify({'error': "Invalid datetime format."})
if not 'sensors' in d or not len(d['sensors']):
return jsonify({'error': 'JSON missing or invalid sensors attribute'})
sensors = {}
for nickname in d['sensors']:
try:
sensors[board.get_sensor_or_404(nickname)] = d['sensors'][nickname]
except:
return jsonify({'error': "This board does not have a sensor named"
"'{}'.".format(nickname)})
for sensor, value in sensors.items():
try:
sensor.add_rawdata(datetime=dt, value=value,
avoid_duplication=True)
except Exception as e:
return jsonify({'error': '{}'.format(e)})
# "Datetime conflit for sensor '{}':"
# "raw data with datetime '{}' already "
# "exists!".format(nickname, dt)})
db.session.commit()
return jsonify({'success': '{} new points were saved on the board.'
.format(len(r['data']))})
from flask import Blueprint
auth = Blueprint('auth', __name__)
from . import views
......@@ -3,16 +3,31 @@
# License: AGPL
# Purpose: Hold application-wide settings.
#-------------------------------------------------------------------------------
from configparser import ConfigParser
import yaml
import os
basedir = os.path.abspath(os.path.dirname(__file__))
def path_here(filename):
return os.path.realpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), filename))
with open(path_here('data/DB_INIT.yaml'), encoding='utf-8') as f:
DB_INIT = yaml.safe_load(f.read())
cp = ConfigParser()
cp.read(path_here('../settings.ini'))
EMMCONFIG = cp['app']['config']
class Config:
host = '0.0.0.0'
WTF_CSRF_ENABLED = True
# wtforms secure submit transactions
SECRET_KEY = os.environ.get('EMMPASS')
SECRET_KEY = cp['app']['pass']
UPLOAD_FOLDER = 'upload'
......@@ -20,14 +35,17 @@ class Config:
SQLALCHEMY_COMMIT_ON_TEARDOWN = True
# flask-mail settings
MAIL_SERVER = 'smtp.gmail.com'
MAIL_SERVER = cp['mail']['smtp']
MAIL_PORT = 587
MAIL_USE_SSL = False
MAIL_USE_TLS = True
MAIL_USERNAME = 'cta.emm.web'
MAIL_PASSWORD = os.environ['EMMPASS']
MAIL_USERNAME = cp['mail']['address'].split('@')[0].strip()
MAIL_PASSWORD = cp['mail'].get('pass') or cp['app']['pass']
MAIL_SUBJECT_PREFIX = '[CTA-EMM-WEB] '
MAIL_SENDER = 'CTA-EMM-WEB <cta.emm.web@gmail.com>'
MAIL_SENDER = 'CTA-EMM-WEB <{}>'.format(cp['mail']['address'])
MAIL_ADDRESS = cp['mail']['address']
DB_INIT = DB_INIT
@staticmethod
def init_app(app):
......@@ -50,22 +68,28 @@ class DevelopmentConfig(Config):
class ProductionConfig(Config):
SERVER_NAME = cp['server']['name']
DEBUG = False
# format: mysql+DRIVER://USER:PASSWORD@localhost/DATABASE
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://ctaemm:{}@localhost/emmdb'\
.format(os.environ.get('EMMPASS'))
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://{USER}:{PASS}@localhost/{DB}'\
.format(USER=cp['db']['user'],
PASS=cp['db'].get('pass') or cp['app']['pass'],
DB=cp['db']['name'])
class TestingConfig(ProductionConfig):
# activat es auto reload on file changes and werkzeug's debugger
DEBUG = True
SERVER_NAME = None
config = {'dev': DevelopmentConfig,
'test': TestingConfig,
'prod': ProductionConfig,
'default': DevelopmentConfig}
config_select = {
'dev': DevelopmentConfig,
'test': TestingConfig,
'prod': ProductionConfig,
'default': DevelopmentConfig}
# [app] -- include js client code to peform page reload periodically
# BEWARE: may overload server connection
......
from .main import main_blueprint
from .board import board_blueprint
from .auth import auth_blueprint
from .api import api_blueprint
blueprints = [main_blueprint,
board_blueprint,
auth_blueprint,
api_blueprint]
from flask import Blueprint
api_blueprint = Blueprint('api', __name__, url_prefix='/api')
from . import routes
from ... import db
from ...models import *
from . import api_blueprint as api
from flask import jsonify, request
import json
from datetime import datetime
def JSONError(message):
return jsonify({'Error': message})
@api.route('/post/rawsensordata', methods=['POST'])
def post_rawsensordata():
''' Expects a request with valid JSON attribute containing sensor data.
Returns a JSON response with success or error description.
Expected JSON format:
{
'board_hash': str,
'data': [
{
'datetime': {'value': str, 'format': str, 'source': str},
'sensors': {'<name1>': number, '<name2>': number, ...}
}
]
}
'''
if not request.json:
return JSONError("Request does not have a valid JSON attribute.")
js = request.json
if type(js) is str:
js = json.loads(js)
if not 'board_hash' in js or len(str(js['board_hash']).strip()) == 0:
return JSONError("JSON missing or empty attribute 'board_hash'.")
board = Board.query.filter_by(_userhash=js['board_hash']).first()
if not board:
return JSONError("Invalid 'board_hash': board not found.")
for i, d in enumerate(js['data']):
try:
dt = datetime.strptime(d['datetime']['value'],
d['datetime']['format'])
except Exception as e:
return JSONError("JSON missing or invalid 'datetime' attribute. "
"Exception: {}".format(e))
if not 'sensors' in d or not len(d['sensors']):
return JSONError("JSON missing or invalid 'sensors' attribute at "
"data line {}".format(i))