Commit c2535f77 authored by Nelso Jost's avatar Nelso Jost

FIX: secured api routes and improved deployment operations

parent ba8017e1
......@@ -66,7 +66,7 @@ initdb:
run:
@ echo "-------------------------------------------------------"
${VENV}/bin/python manage.py runserver --host ${HOST} --port 5000
export EMMCONFIG=dev && ${VENV}/bin/python manage.py runserver --host ${HOST} --port 5000
shell:
${VENV}/bin/ipython manage.py shell
......@@ -80,12 +80,14 @@ tree:
mysql-setup:
mysql -u root -p < production/MYSQL_SETUP
mysql-info:
mysql -u root -p < production/MYSQL_FETCH_INFO
deploy:
mkdir -p production/nginx-gunicorn/logs
sudo ${VENV}/bin/python production/nginx-gunicorn/deploy.py ${PORT}
sudo ${VENV}/bin/python production/nginx-gunicorn/deploy.py --port=${PORT}
undeploy:
sudo ${VENV}/bin/python production/nginx-gunicorn/undeploy.py
sudo ${VENV}/bin/python production/nginx-gunicorn/deploy.py -u
clean-logs-nginx:
rm -rf production/nginx-gunicorn/logs
......
......@@ -32,21 +32,25 @@ login_manager = LoginManager()
login_manager.session_protection = 'strong'
login_manager.login_view = 'auth.login'
def create_app():
def create_app(config_name):
''' Creates the Flask application object and register configurations,
blueprints and related global objects for it.
'''
app = Flask(__name__)
from . import config
app.config.from_object(config)
from .config import config
app.config.from_object(config[config_name])
config[config_name].init_app(app)
from .views import main as main_blueprint
from .main import main as main_blueprint
app.register_blueprint(main_blueprint)
from .auth import auth as auth_blueprint
app.register_blueprint(auth_blueprint, url_prefix='/auth')
from .api import api as api_blueprint
app.register_blueprint(api_blueprint, url_prefix='/api')
db.init_app(app)
bootstrap.init_app(app)
login_manager.init_app(app)
......
from flask import Blueprint
api = Blueprint('api', __name__)
from . import views
from . import api
from ..models import *
from .. import db
from flask import jsonify, request
import json
from datetime import datetime
@api.route('/post/rawsensordata/<int:board_id>', methods=['POST'])
def api_post_rawsensordata(board_id):
''' Expects a request with valid JSON attribute containing sensor data.
Returns a JSON response with success or error description.
Expected JSON format:
{'datetime': {'value': str, 'format': str},
'sensors': {'<nickname_1>': number,
'<nickname_2>': number,
...,
'<nickname_N>': number}}
'''
try:
board = Board.get(board_id)
except:
return jsonify({'error': 'Invalid board ID'})
if not request.json:
return jsonify({'error':
'Request does not have a valid JSON attribute.'})
d = request.json
if type(d) is str:
d = json.loads(request.json)
if not 'user_hash' in d or d['user_hash'] != board.user.password_hash:
return jsonify({'error': 'Missing or invalid JSON "user_hash" '
'attribute.'})
if not 'datetime' in d:
return jsonify({'error': "JSON missing or invalid datetime attribute."
" Expeted format: {'value': str, 'format': str, 'source': str}"})
try:
dt = datetime.strptime(d['datetime']['value'],
d['datetime']['format'])
except:
return jsonify({'error': "Invalid datetime format."})
if not 'sensors' in d or not len(d['sensors']):
return jsonify({'error': 'JSON missing or invalid sensors attribute'})
sensors = {}
for nickname in d['sensors']:
try:
sensors[board.get_sensor_or_404(nickname)] = d['sensors'][nickname]
except:
return jsonify({'error': "This board does not have a sensor named"
"'{}'.".format(nickname)})
for sensor, value in sensors.items():
try:
sensor.add_rawdata(datetime=dt, value=value,
avoid_duplication=True)
except:
return jsonify({'error': "Datetime conflit for sensor '{}':"
"raw data with datetime '{}' already "
"exists!".format(nickname, dt)})
db.session.commit()
return jsonify({'success': 'Data was sucessfully saved on the board.'})
......@@ -5,36 +5,52 @@
#-------------------------------------------------------------------------------
import os
basedir = os.path.abspath(os.path.dirname(__file__))
class Config:
host = '0.0.0.0'
# [flask] -- allow auto restart on file changes
DEBUG = True
host = '0.0.0.0'
# wtforms secure submit transactions
SECRET_KEY = os.environ.get('EMMPASS')
SECRET_KEY = os.environ.get('FLASK_SECRET_KEY') or \
'hard to guess string'
UPLOAD_FOLDER = 'upload'
# [flask-sqlalchemy] -- for sqlite version
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'db.sqlite')
# automatically commits at the end of a request
SQLALCHEMY_COMMIT_ON_TEARDOWN = True
# [flask-sqlalchemy] -- for mysql version
# format: mysql://username:password@hostname/database
# SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://ctaemm:ctaemm@localhost/emmdb'
# flask-mail settings
MAIL_SERVER = 'smtp.gmail.com'
MAIL_PORT = 587
MAIL_USE_SSL = False
MAIL_USE_TLS = True
MAIL_USERNAME = 'cta.emm.web'
MAIL_PASSWORD = os.environ['EMMPASS']
# [flask-sqlalchemy] -- assures a commit to prevent data loss
SQLALCHEMY_COMMIT_ON_TEARDOWN = True
MAIL_SUBJECT_PREFIX = '[CTA-EMM-WEB] '
MAIL_SENDER = 'CTA-EMM-WEB <cta.emm.web@gmail.com>'
UPLOAD_FOLDER = 'upload'
@staticmethod
def init_app(app):
pass
MAIL_SERVER = 'smtp.gmail.com'
MAIL_PORT = 587
MAIL_USE_SSL = False
MAIL_USE_TLS = True
MAIL_USERNAME = 'cta.emm.web'
MAIL_PASSWORD = os.environ['EMM_ADMIN_PWD']
class DevelopmentConfig(Config):
# activat es auto reload on file changes and werkzeug's debugger
DEBUG = True
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'db.sqlite')
class ProductionConfig(Config):
DEBUG = False
# format: mysql+ENGINE://USER:PASSWORD@localhost/DATABASE
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://ctaemm:ctaemm@localhost/emmdb'
config = {'dev': DevelopmentConfig,
'prod': ProductionConfig,
'default': DevelopmentConfig}
FLASK_MAIL_SUBJECT_PREFIX = '[CTA-EMM-WEB] '
FLASK_MAIL_SENDER = 'CTA-EMM-WEB <cta.emm.web@gmail.com>'
class DATETIME:
INTERNAL_FORMAT = "%Y%m%d%H%M%S"
......
......@@ -9,8 +9,8 @@ def send_async_email(app, msg):
mail.send(msg)
def send_email(to, subject, template, **kwargs):
msg = Message(config.FLASK_MAIL_SUBJECT_PREFIX + subject,
sender = config.FLASK_MAIL_SENDER,
msg = Message(config.MAIL_SUBJECT_PREFIX + subject,
sender = config.MAIL_SENDER,
recipients = [to])
msg.body = render_template(template + '.txt', **kwargs)
msg.html = render_template(template + '.html', **kwargs)
......
from flask import Blueprint
main = Blueprint('main', __name__)
from . import views
......@@ -7,9 +7,9 @@ from wtforms.widgets import FileInput
from flask import session, request, redirect, url_for
from flask.ext.login import current_user
from .models import *
from ..models import *
from . import config
from .. import config
__all__ = ['BoardSelectionForm', 'ManageBoardForm', 'SensorForm', 'MyFileField',
'ImportCSVForm', 'ExportCSVForm']
......
......@@ -23,12 +23,12 @@ will actually import stuff from app/__init__.py and that ile needs
to import the blueprint from here in order to build the app.
'''
from . import db, config
from .models import *
from .. import db, config
from ..models import *
from .forms import *
from .email import send_email
from ..email import send_email
from flask import Blueprint, url_for, render_template, jsonify, \
from flask import Blueprint, url_for, render_template, \
request, abort, redirect, Response, flash, session,\
send_from_directory, after_this_request
......@@ -40,7 +40,7 @@ import gzip
from pprint import pprint
import json
main = Blueprint('main', __name__)
from . import main
def init_sensor_forms(sensors, nickname_index_focus=None, include_stats=False):
''' Creates a list of NewSensorForm objects for each sensor json data on
......@@ -424,39 +424,3 @@ def view_export_csv(board_id):
# auto_page_reload=config.AUTO_PAGE_RELOAD,
# form=form)
@main.route('/api/post/rawsensordata/<int:board_id>', methods=['POST'])
def api_post_rawsensordata(board_id):
''' Add a bunch of sensor data from specified board.
Expects JSON data on the format:
{'datetime': str,
'sensors': {'<nickname_1>': 'value_1',
'<nickname_2>': 'value_2',
...,
'<nickname_N>': 'value_N'}
}
'''
try:
board = Board.get(board_id)
except:
abort(400)
if not request.json:
abort(400)
d = request.json
if type(d) is str:
d = json.loads(request.json)
if not 'datetime' in d or not 'sensors' in d or not len(d['sensors']):
abort(400)
for nickname, value in d['sensors'].items():
sensor = board.get_sensor(nickname).add_rawdata(
datetime=d['datetime']['value'], value=value)
db.session.commit()
main.need_refresh = True
return jsonify(d)
......@@ -6,6 +6,8 @@ from .sensor import Sensor
from flask import abort
from datetime import datetime
class Board(db.Model, DBUtils):
__tablename__ = 'boards'
......@@ -96,42 +98,46 @@ class Board(db.Model, DBUtils):
else:
raise ValueError("Unexpected format on 'json_data' container")
def import_csv_sensor_data(self, filename, cols, sep=','):
''' Add raw sensor data from a CSV file where each line must have
a datetime column, along with at least one sensor data column.
def import_csv_sensor_data(self, filename, sep=',', debug=False,
avoid_duplication=False):
''' Add raw sensor data from a CSV file.
filename -- path to the CSV file to be open for reading
cols -- sequence of str that designates CSV columns
A 'datetime' column must be present at any position,
along with valid sensor nicknames for sensor data.
Use '#' to ignore a column.
sep -- CSV delimiter character
'''
sensors, ignores, datetime_j = {}, set(), None
The first line of the file must indicate sensor nicknames for each
column of the CSV and also the datetime column.
for j, key in enumerate(cols):
if key == '#':
ignores.add(j)
elif key == 'datetime':
datetime_j = j
else:
try:
sensors[j] = self.get_sensor(key)
except:
raise ValueError("Invalid sensor name '{}' for column {}"
.format(key, j))
if datetime_j is None:
raise ValueError("Missing required column 'datetime'")
Datetime column name is designated as: datetime<fmt>
where <fmt> is a valid Python datetime format.
if not sensors:
raise ValueError("Not one valid sensor was given by 'columns'")
Example of line with columns specifications:
datetime%Y-%m-%d-%H-%M-%S;DHT22_TEMP;LDR
here the first column is the datetime with format %Y-%m-%d-%H-%M-%S
and the other two columns are sensor nicknames.
The CSV separator in this case is sep=';'.
'''
d = {'datetime': {}, 'sensors': {}}
with open(filename) as f:
reader = csv.reader(f, delimiter=sep)
for i, row in enumerate(reader):
for j, sensor in sensors.items():
sensor.add_rawdata(datetime=row[datetime_j],
value=row[j])
if debug:
print('{}\r'.format(i), end='')
if i == 0:
for j, c in enumerate(row):
if 'datetime' in c:
d['datetime'] = {'fmt': c[c.find('datetime')+8:],
'j': j}
elif c == '#':
continue
else:
d['sensors'][j] = self.get_sensor(c)
continue
dt = datetime.strptime(row[d['datetime']['j']],
d['datetime']['fmt'])
for j, sensor in d['sensors'].items():
sensor.add_rawdata(datetime=dt, value=row[j],
avoid_duplication=avoid_duplication)
@classmethod
def delete(cls, key):
......
from .. import config, db
from .. import db
from .dbutils import DBUtils
from datetime import datetime
class RawSensorData(db.Model, DBUtils):
__tablename__ = 'rawsensordata'
id = db.Column(db.Integer, primary_key=True)
sensor_id = db.Column(db.Integer, db.ForeignKey('sensors.id'))
datetime = db.Column(db.String(20))
datetime = db.Column(db.DateTime())
value = db.Column(db.String(50))
__tablename__ = 'rawsensordata'
def __repr__(self):
return "<RawSensorData id={} sensor_id={} datetime='{}' value={}>"\
.format(self.id, self.sensor_id, self.datetime, self.value)
@property
def datetime_obj(self):
return datetime.strptime(self.datetime,
config.DATETIME.INTERNAL_FORMAT)
......@@ -6,6 +6,8 @@ import csv
class Sensor(db.Model, DBUtils):
__tablename__ = 'sensors'
id = db.Column(db.Integer, primary_key=True)
board_id = db.Column(db.Integer, db.ForeignKey('boards.id'))
nickname = db.Column(db.String(20))
......@@ -18,18 +20,18 @@ class Sensor(db.Model, DBUtils):
rawdata = db.relationship('RawSensorData', backref='sensor',
lazy='dynamic')
__tablename__ = 'sensors'
def __repr__(self):
return "<Sensor id={} board_id={} nickname='{}' data_count={}>".format(
self.id, self.board_id, self.nickname, self.rawdata.count())
def add_rawdata(self, **kwargs):
def add_rawdata(self, datetime, value, avoid_duplication=False):
''' Add raw sensor data. RawSensorData fields are given by kwargs.
'''
if 'sensor_id' in 'kwargs':
del kwargs['sensor_id']
db.session.add(RawSensorData(sensor_id = self.id, **kwargs))
if avoid_duplication:
if self.rawdata.filter_by(datetime=kwargs['datetime']).first():
raise ValueError('Datetime already exists!')
db.session.add(RawSensorData(sensor_id=self.id, datetime=datetime, value=value))
@classmethod
def delete(cls, key):
......@@ -82,7 +84,7 @@ class Sensor(db.Model, DBUtils):
'''
def row_generator(sensor):
for rawdata in sensor.rawdata.all():
dt = rawdata.datetime_obj
dt = rawdata.datetime
yield '[new Date({Y}, {M}, {D}, {h}, {m}, {s}), {y_value}]'\
.format(Y=dt.year, M=dt.month, D=dt.day, h=dt.hour,
m=dt.minute, s=dt.second, y_value=rawdata.value)
......
#-------------------------------------------------------------------------------
# Author: Nelso G. Jost <nelsojost@gmail.com> Copyright(C) 2015
# License: AGPL
# Purpose: Holds all database related functionality (models and operations).
#-------------------------------------------------------------------------------
'''
This project uses the SQL Alchemy ORM (Object-Relational-Mapper) to perform
all its database operations, integrated via Flask-SQLAlchemy extension.
DATABASE SCHEMA
---------------
+-----------------------+ +---------------------------+
| Board | | Sensor |
+-----------------------+ +---------------------------+
| id : int, PK |<(1)--+ | id : int, PK |<-(1)-+
| nickname : str | +--(n)>| board_id : int, FK | |
| latitude : str | | nickname : str | |
| longitude : str | | quantity : str | |
| description : str | | measured_object : str | |
+-----------------------+ | data_format : str | |
| unity : str | |
| description : str | |
+---------------------------+ |
|
+---------------------+ |
| RawSensorData | |
+---------------------+ |
| id : int, PK | |
| sensor_id : int, FK |<-(n)-------+
| datetime : str |
| value : str |
+---------------------+
OVERVIEW
---------
Tables are declared on this file through the concept of models -- a fancy name
for Python classes that inherit db.Model and specify table fields/columns with
class attributes of type db.Column. Relationships between tables are also
declared on the models with db.relationship class attributes. Finally, since
models are Python classes, operations can also be implemented.
Model Conventions for this project:
* All models inherits BaseModelUtils to provide a set of basic,
short-named, operations for easing of use on a Python shell.
* All tables have an integer field 'id' as primary key, that will be
auto-generated if not provided.
Convention is to NOT provide it. Thus, a call to db.session.flush()
or db.session.commit() must be made to generate this value.
* All models declare the __tablename__ attribute, which holds a nicer,
pural, name for the table.
* All models declare the __formfields__ attribute, which holds the names
of fields editable by the end user on HTML forms.
* Attribute declaration follow the pattern:
[fields]
[relationships]
__tablename__
__formfields__
[BaseModelUtils attributes]
USAGE
-----
The Board model is the entry point to explore meteorological data.
First,
'''
# __all__ = ['Board', 'Sensor', 'RawSensorData', 'list_models']
#
# from . import db
# from . import config
# from datetime import datetime
# import csv
# list_models = [Board, Sensor, RawSensorData]
# class PhysicalQuantity(db.Model):
# __tablename__ = 'physicalquantity'
#
# id = db.Column(db.Integer, primary_key=True)
# name = db.Column(db.String(20))
# class MeasuredTarget(db.Model):
# __tablename__ = 'measuredtarget'
#
# id = db.Column(db.Integer, primary_key=True)
# name = db.Column(db.String(20))
datetime%Y%m%d%H%M%S DHT22_TEMP DHT22_AH # BMP085_PRESSURE LDR
20130226224346 26.4 44.0 668 101101 1.08
20130226224446 26.2 45.0 666 101109 1.17
20130226224546 26.0 44.0 663 101093 1.17
......
......@@ -56,6 +56,12 @@ Where [command] can be:
# call rollback() before end of the session to avoid writing
# changes permanently on the database
'''
import os
if not 'EMMPASS' in os.environ:
os.environ['EMMPASS'] = input('EMMPASS=')
if not 'EMMCONFIG'in os.environ:
os.environ['EMMCONFIG'] = input('EMMCONFIG=')
from flask.ext.script import Manager, Shell
import app as ctaemmweb
......@@ -68,34 +74,70 @@ for n in ('commit', 'rollback', 'flush'):
globals()[n] = getattr(db.session, n)
import csv
import os
import requests
from datetime import datetime, timedelta
import json
app = create_app()
app = create_app(os.environ.get('EMMCONFIG'))
manager = Manager(app)
class FakeLogger:
datetime = datetime(2015, 7, 24, 16, 14, 0)
datetime_fmt = '%Y-%m-%d-%H-%M-%S'
timedelta = timedelta(minutes=5)
BID = 2
SERVER = 'localhost'
PORT = 5000
URL = 'http://{server}:{port}/api/post/rawsensordata/{bid}'
@classmethod
def post(cls, sensor_data=None):
cls.datetime += cls.timedelta
d = {'datetime': {'value': cls.datetime.strftime(cls.datetime_fmt),
'format': cls.datetime_fmt},
'sensors': sensor_data,
'user_hash': Board.get(cls.BID).user.password_hash}
print('Send: ', d)
r = requests.post(cls.URL.format(server=cls.SERVER, port=cls.PORT,
bid=cls.BID), json=json.dumps(d))
print(r)
try:
print('Recieved: ', r.json())
except:
pass
return r
fakelog = FakeLogger
@manager.command
def initdb():
''' (re)creates all the database from models and populate it.
'''
print('\nUsing database:\n ' + app.config['SQLALCHEMY_DATABASE_URI'])
db.drop_all()
db.create_all()
user_admin = users.add(username='admin', email='cta.emm.web@gmail.com',
confirmed=True,
password=os.environ['EMM_ADMIN_PWD'])
print("\nCreating user 'admin' and default boards..", end='')
user_admin = User.add(username='admin', email='cta.emm.web@gmail.com',
confirmed=True, password=os.environ['EMMPASS'])
for board_nickname in ('Pezzi', 'CAP'):
Board.add(nickname=board_nickname, user=user_admin,
sensors=config.DEFAULT_SENSORS)
db.session.commit()
print(' done!')
CSV_FILENAME = os.path.join('data', '26-Fev-2013_-_04-Mar-2013.log')
print('\nImporting data from:\n {}'.format(CSV_FILENAME))
boards.get(Board.nickname == 'Pezzi').import_csv_sensor_data(
filename=os.path.join('data', '26-Fev-2013_-_04-Mar-2013.log'),
cols=['datetime', 'DHT22_TEMP', 'DHT22_AH', '#',
'BMP085_PRESSURE', 'LDR'],
sep='\t')
filename=CSV_FILENAME, sep='\t', debug=True)
print('Writing changes on the disc..', end='')
db.session.commit()
print(' done!')
manager.add_command("shell", Shell(make_context=
......
SELECT table_schema emmdb,
sum( data_length + index_length ) / 1024 / 1024 "Data Base Size in MB",
sum( data_free )/ 1024 / 1024 "Free Space in MB"
FROM information_schema.TABLES
GROUP BY table_schema ;
import argparse
import jinja2
import os
import sys
import subprocess
import time
try:
PORT = str(int(sys.argv[1]))
except:
PORT = '8000'
parser = argparse.ArgumentParser(
description="Deploy the web app onto the production server")
parser.add_argument('-u', dest='undeploy', action='store_const', const=True,
help="Undeploy the web app of the production server")
parser.add_argument('--port', dest='port', type=int, default=8000,
help="Specifies the port to which the web app will be deployed")
CONFIG = {
'base_dir': os.path.abspath('.'), # project root directory (see docs)
'wsgi_port': PORT
}
args = vars(parser.parse_args())
print('Working directory:', CONFIG['base_dir'])
def get_filename_here(filename):
def make_path_here(filename):
return os.path.join(os.path.abspath(os.path.dirname(__file__)), filename)
NGINX_CONF_FILENAME = get_filename_here('nginx.conf')
NGINX_SITES_AVAILABLE_FILENAME = '/etc/nginx/sites-available/ctaemm'
NGINX_SITES_ENABLED_FILENAME = '/etc/nginx/sites-enabled/ctaemm'
GUNICORN_PATH = os.path.join(CONFIG['base_dir'], '.venv/bin/gunicorn')
GUNICORN_CONFIG_FILENAME = get_filename_here('gunicorn.conf.py')
GUNICORN_TEMP_CONFIG_FILENAME = get_filename_here('.gunicorn.conf.py')
GUNICORN_PID_FILENAME = get_filename_here('.pid_gunicorn')
SUPERVISOR_TEMPLATE_FILENAME = get_filename_here('supervisor.conf')
SUPERVISOR_CONF_FILENAME = '/etc/supervisor/conf.d/gunicorn_ctaemm.conf'
EMMPASS = None
if not args['undeploy']:
try:
EMMPASS = os.environ['EMMPASS']
print('EMMPASS=', EMMPASS, sep='')
if not EMMPASS:
raise Exception
except:
EMMPASS = input('EMMPASS=')
print('EMMPASS=', EMMPASS, sep='')
# this assumes that the deploy script was launched from the root directory
# of the project
APP_PATH = os.path.abspath('.')
WSGI_PATH = make_path_here('')
WSGI_PORT = args['port']
NGINX_TEMPLATE_CONF_FILENAME = make_path_here('nginx.conf')
NGINX_SITE_NAME = 'ctaemm'
NGINX_SITES_AVAILABLE_FILENAME = '/etc/nginx/sites-available/' \
+ NGINX_SITE_NAME
NGINX_SITES_ENABLED_FILENAME = '/etc/nginx/sites-enabled/' \
+ NGINX_SITE_NAME
GUNICORN_BIN_PATH = APP_PATH + '/.venv/bin/gunicorn'