Commit dc150585 authored by Nelso Jost's avatar Nelso Jost

FIX: mail sending; NEW: major deploy refactor

parent f8acf2ee
logs/
.venv/
.tag*
*.swp
*.autosave
*.sqlite
*.pyc
.pid_gunicorn
.gunicorn.conf.py
.gunicorn.conf
__pycache__
......@@ -3,111 +3,129 @@ VENVDIR := $(shell pwd)/.venv
VENVPY := ${VENVDIR}/bin/python
PORT := 5000
.PHONY: log
all: help
help:
@ echo "USAGE: make <target> where <target> can be"
@ echo ""
@ echo " debreq apt-get install Debian-based system dependencies"
@ echo " venv create local Python virtualenv with PYBIN variable"
@ echo ""
@ echo " mysql-init create MySQL user and DB (not tables)"
@ echo " mysql-summary shows all current users and DBs"
@ echo " mysql-describe show current DB schema"
@ echo " mysql-reset allows root password reset"
@ echo ""
@ echo " dbinit recreate DB schema (drop data if exists) and populate it"
@ echo " shell start ipython3 interactively for high level DB admin"
@ echo ""
@ echo " run start Flask's development server at"
@ echo " http://localhost:5000 (DEBUG=True)"
@ echo " options: PORT=___ (default: 5000)"
@ echo ""
@ echo " deploy register supervisor dameon for gunicorn WSGI app"
@ echo " and production ready server nginx"
@ echo " undeploy undo the previous operation"
debreq:
sudo apt-get install nginx python3 python3-pip supervisor mysql-server
@ echo "$$HELP_MAKEFILE"
debian-requirements:
sudo apt-get install nginx python3 python3-pip supervisor mysql-server multitail tree
sudo pip3 install virtualenv
sudo pip3 install --upgrade pip
venv:
virtualenv -v --python='${PYBIN}' ${VENVDIR}
${VENVDIR}/bin/pip3 install --upgrade pip
${VENVDIR}/bin/pip3 install -r requirements.pip
${VENVDIR}/bin/pip3 install --upgrade pip
${VENVDIR}/bin/pip3 install -r requirements.txt
@ echo "-------------------------------------------------------"
@ echo "Virtualenv successfully created at"
@ du -sh ${VENVDIR}
setup: debreq venv
clean-venv:
rm -rf ${VENVDIR}
setup: debian-requirements venv
dbinit:
${VENVPY} manage.py dbinit
list-routes:
${VENVPY} manage.py list_routes
run:
${VENVPY} manage.py runserver --port ${PORT}
shell:
${VENVDIR}/bin/ipython manage.py shell
fakelog:
${VENVPY} tests/fakemeteorolog.py ${BOARD_ID}
deploy:
sudo ${VENVPY} prod/nginx-gunicorn/deploy.py
tree:
tree . -I "${VENVDIR}|__pycache__|*.pyc"
log:
multitail -s 2 /var/log/supervisor/supervisord.log \
log/supervisor/stdout.log \
log/supervisor/stderr.log \
/var/log/nginx/access.log \
log/gunicorn/access.log \
log/gunicorn/error.log
summary:
sudo ${VENVPY} prod/nginx-gunicorn/deploy.py --summary
mysql-init:
${VENVPY} production/manage_mysql.py init
undeploy:
sudo ${VENVPY} prod/nginx-gunicorn/deploy.py --undeploy
mysql-user:
${VENVPY} production/manage_mysql.py user
redeploy: undeploy deploy
gunicorn:
${VENVPY} prod/nginx-gunicorn/deploy.py --make-gunicorn-conf
${VENVDIR}/bin/gunicorn -c `pwd`/'.gunicorn.conf' manage:app
run:
${VENVPY} manage.py runserver --port ${PORT}
mysql-setup: mysql-usercreate mysql-init dbinit
mysql-summary:
${VENVPY} production/manage_mysql.py summary
${VENVPY} prod/manage_mysql.py summary
mysql-describe:
chmod +x production/mysql_describe.sh && ./production/mysql_describe.sh
mysql-init:
${VENVPY} prod/manage_mysql.py init
mysql-usercreate:
${VENVPY} prod/manage_mysql.py user-create
mysql-userdrop:
${VENVPY} prod/manage_mysql.py user-drop
mysql-showschema:
${VENVPY} production/manage_mysql.py showschema
${VENVPY} prod/manage_mysql.py showschema
mysql-describe:
chmod +x prod/mysql_describe.sh && ./production/mysql_describe.sh
mysql-enter:
mysql-shell:
mysql -uroot -p
mysql-reset:
sudo dpkg-reconfigure mysql-server-5.5
gunicorn:
cd production/nginx-gunicorn && ${VENVDIR}/bin/gunicorn wsgi:app
deploy:
sudo ${VENVPY} production/nginx-gunicorn/deploy.py
list-routes:
${VENVPY} manage.py list_routes
fakelog:
${VENVPY} tests/fakemeteorolog.py ${BOARD_ID}
undeploy:
sudo ${VENVPY} production/nginx-gunicorn/deploy.py -u
tree:
tree . -I "${VENVDIR}|__pycache__|*.pyc"
reset-settings:
python3 scripts/reset_settings.py
cp -rfv app/default_settings.ini settings.ini
clean-venv:
rm -rfv ${VENVDIR}
clean-logs:
rm -rf logs/*
rm -rf log/gunicorn/* log/supervisor/*
touch log/gunicorn/access.log log/gunicorn/error.log log/supervisor/stdout.log log/supervisor/stderr.log
clean: clean-venv reset-settings clean-logs
sudo py3clean app
sudo rm -rfv doc/_build db.sqlite log/gunicorn/pid __pycache__ tests/__pycache__
define HELP_MAKEFILE
USAGE: make <target> where <target> can be
clean: clean-venv reset-settings
sudo find . -type f -name "*.py[co]" -delete
sudo find . -type d -name "__pycache__" -delete
sudo rm -f app/db.sqlite
sudo rm -rf doc/_build
setup install/check system dependencies and create virtualenv
dbinit recreate DB schema (drop data if exists) and populate it
shell start ipython3 -i session with active app for exploring
run start Flask's dev-server at localhost:5000 (DEBUG=True )
logtail-nginx-access:
tail production/nginx-gunicorn/logs/nginx-access.log
deploy setup/start production server (Nginx + Gunicorn + Supervisor)
undeploy undo production server deploynment
redeploy perform make undeploy deploy
logtail-nginx-error:
tail production/nginx-gunicorn/logs/nginx-error.log
mysql-setup perform make mysql-usercreate mysql-init dbinit
mysql-summary shows all current users and DBs
mysql-init create MySQL user and DB (not tables)
mysql-usercreate create MySQL user according to settings.ini
mysql-usercreat remove MySQL user
mysql-showschema show current DB schema
mysql-describe attempt to fetch all tables specifications
endef
export HELP_MAKEFILE
......@@ -17,8 +17,6 @@ This approach has several advantages:
* Allows creation of multiple instances -- better for unit testing.
'''
from .config import EMMCONFIG
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.bootstrap import Bootstrap
......@@ -29,22 +27,61 @@ import json
import os
db = SQLAlchemy()
bootstrap = Bootstrap()
mail = Mail()
flaskbootstrap = Bootstrap()
flaskmail = Mail()
login_manager = LoginManager()
login_manager.session_protection = 'strong'
login_manager.login_view = 'auth.login'
class ReverseProxied(object):
# http://flask.pocoo.org/snippets/35/
'''Wrap the application in this middleware and configure the
front-end server to add these headers, to let you quietly bind
this to a URL other than / and to an HTTP scheme that is
different than what is used locally.
Nginx example
-----------------------------------------------------------------
location /myprefix {
proxy_pass http://192.168.0.1:5001;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /myprefix;
}
-----------------------------------------------------------------
:param app: the WSGI application
'''
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
if script_name:
environ['SCRIPT_NAME'] = script_name
path_info = environ['PATH_INFO']
if path_info.startswith(script_name):
environ['PATH_INFO'] = path_info[len(script_name):]
scheme = environ.get('HTTP_X_SCHEME', '')
if scheme:
environ['wsgi.url_scheme'] = scheme
return self.app(environ, start_response)
def create_app():
''' Creates the Flask application object and register configurations,
blueprints and related global objects for it.
'''
app = Flask(__name__)
app.wsgi_app = ReverseProxied(app.wsgi_app)
from .config import config_select
app.config.from_object(config_select[EMMCONFIG])
config_select[EMMCONFIG].init_app(app)
from .config import config
app.config.from_object(config)
config.init_app(app)
from .controllers import blueprints
......@@ -52,8 +89,8 @@ def create_app():
app.register_blueprint(blueprint)
db.init_app(app)
bootstrap.init_app(app)
flaskbootstrap.init_app(app)
login_manager.init_app(app)
mail.init_app(app)
flaskmail.init_app(app)
return app
......@@ -4,30 +4,46 @@
# Purpose: Hold application-wide settings.
#-------------------------------------------------------------------------------
from configparser import ConfigParser
from datetime import datetime
import yaml
import os
basedir = os.path.abspath(os.path.dirname(__file__))
from threading import Thread
def path_here(filename):
return os.path.realpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), filename))
SETTINGS_FILENAME = path_here('../settings.ini')
CP = ConfigParser()
CP.read(SETTINGS_FILENAME)
APPCONFIG = None
def get_config(keypath, required=False):
section, key = keypath.split('.')
value = CP[section][key]
if required:
while not value:
value = input('\n{}='.format(keypath))
if value:
CP[section][key] = value
break
with open(SETTINGS_FILENAME, 'w') as f:
CP.write(f)
return value
with open(path_here('data/DB_INIT.yaml'), encoding='utf-8') as f:
DB_INIT = yaml.safe_load(f.read())
cp = ConfigParser()
cp.read(path_here('../settings.ini'))
EMMCONFIG = cp['app']['config']
APPCONFIG = get_config('app.config')
class Config:
host = '0.0.0.0'
WTF_CSRF_ENABLED = True
# wtforms secure submit transactions
SECRET_KEY = cp['app']['pass']
SECRET_KEY = get_config('app.secret_key', required=True)
WTF_CSRF_ENABLED = True
UPLOAD_FOLDER = 'upload'
......@@ -35,15 +51,15 @@ class Config:
SQLALCHEMY_COMMIT_ON_TEARDOWN = True
# flask-mail settings
MAIL_SERVER = cp['mail']['smtp']
MAIL_PORT = 587
MAIL_USE_SSL = False
MAIL_USE_TLS = True
MAIL_USERNAME = cp['mail']['address'].split('@')[0].strip()
MAIL_PASSWORD = cp['mail'].get('pass') or cp['app']['pass']
MAIL_ADDRESS = get_config('mail.address', required=True)
MAIL_USERNAME = MAIL_ADDRESS.split('@')[0].strip()
MAIL_PASSWORD = get_config('mail.password')
MAIL_SERVER = get_config('mail.smtp')
MAIL_PORT = 465
MAIL_USE_SSL = True
MAIL_USE_TLS = False
MAIL_SUBJECT_PREFIX = '[CTA-EMM-WEB] '
MAIL_SENDER = 'CTA-EMM-WEB <{}>'.format(cp['mail']['address'])
MAIL_ADDRESS = cp['mail']['address']
MAIL_SENDER = 'CTA-EMM-WEB <{}>'.format(MAIL_ADDRESS)
DB_INIT = DB_INIT
......@@ -64,36 +80,30 @@ class DevelopmentConfig(Config):
# activat es auto reload on file changes and werkzeug's debugger
DEBUG = True
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'db.sqlite')
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + path_here('../db.sqlite')
class ProductionConfig(Config):
SERVER_NAME = cp['server']['name']
SERVER_NAME = get_config('nginx.server_name', APPCONFIG == 'prod')
DEBUG = False
# format: mysql+DRIVER://USER:PASSWORD@localhost/DATABASE
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://{USER}:{PASS}@localhost/{DB}'\
.format(USER=cp['db']['user'],
PASS=cp['db'].get('pass') or cp['app']['pass'],
DB=cp['db']['name'])
.format(USER=get_config('mysql.user', APPCONFIG == 'prod'),
PASS=get_config('mysql.password', APPCONFIG == 'prod'),
DB=get_config('mysql.db_name', APPCONFIG == 'prod'))
class TestingConfig(ProductionConfig):
# activat es auto reload on file changes and werkzeug's debugger
DEBUG = True
SERVER_NAME = None
config_select = {
config = {
'dev': DevelopmentConfig,
'test': TestingConfig,
'prod': ProductionConfig,
'default': DevelopmentConfig}
# [app] -- include js client code to peform page reload periodically
# BEWARE: may overload server connection
class AUTO_PAGE_RELOAD:
active = False
interval = 2000
'default': DevelopmentConfig}[APPCONFIG]
print("{} [INFO] loaded app config '{}'".format(datetime.now(), APPCONFIG))
......@@ -9,6 +9,7 @@ from flask import render_template, redirect, request, url_for, flash, session
from flask.ext.login import login_user, logout_user, login_required, \
current_user
from datetime import datetime
@auth.route('/login', methods=['GET', 'POST'])
def login():
......
[app]
config = dev
secret_key =
wsgi_port = 8000
[nginx]
server_name = localhost
site_file = default
site_root = emm
[mail]
address = cta.emm.web@gmail.com
smtp = smtp.gmail.com
password =
[mysql]
db_name =
user =
password =
from . import config, mail
from . import config, flaskmail
from flask.ext.mail import Message
from flask import render_template, current_app
from datetime import datetime
from threading import Thread
def send_async_email(app, msg):
with app.app_context():
mail.send(msg)
def send_email(to, subject, template, **kwargs):
app = current_app._get_current_object()
msg = Message(app.config['MAIL_SUBJECT_PREFIX'] + subject,
......@@ -17,6 +17,16 @@ def send_email(to, subject, template, **kwargs):
msg.body = render_template(template + '.txt', **kwargs)
msg.html = render_template(template + '.html', **kwargs)
thr = Thread(target=send_async_email, args=[app, msg])
def send_async_email():
try:
with app.app_context():
flaskmail.send(msg)
except:
return False
print("{} [INFO] finished sending email from '{}' to '{}' "
"with subject '{}'".format(datetime.now(),
msg.sender, to, subject))
thr = Thread(target=send_async_email)
thr.start()
return thr
......@@ -28,6 +28,8 @@ from app.dbmanage import DBInitCommand
from flask.ext.script import Manager, Shell
from datetime import datetime
app = create_app()
manager = Manager(app)
manager.add_command("dbinit", DBInitCommand(app=app))
......@@ -43,7 +45,5 @@ manager.add_command("shell", Shell(make_context=
def list_routes():
print(app.url_map)
print('\nUsing database:\n ' + app.config['SQLALCHEMY_DATABASE_URI'])
if __name__ == '__main__':
manager.run()
......@@ -10,48 +10,9 @@ def path_here(filename):
cp = ConfigParser()
cp.read(path_here('../settings.ini'))
EMMDBNAME = cp['db']['name']
EMMDBUSER = cp['db']['user']
EMMDBPASS = cp['db'].get('pass') or cp['app']['pass']
SCRIPT_USER =\
"""\
#DROP USER '{EMMDBUSER}'@'localhost';
CREATE USER '{EMMDBUSER}'@'localhost' IDENTIFIED BY '{EMMDBPASS}';
"""
SCRIPT_INIT =\
"""\
CREATE DATABASE IF NOT EXISTS {EMMDBNAME}
CHARACTER SET utf8 COLLATE utf8_bin;
GRANT ALL on {EMMDBNAME}.* TO '{EMMDBUSER}'@'localhost'
IDENTIFIED BY '{EMMDBPASS}' WITH GRANT OPTION;
FLUSH PRIVILEGES;
"""
SCRIPT_SUMMARY =\
"""\
SELECT User, Host FROM mysql.user;
SHOW DATABASES;
"""
SHSCRIPT_SHOW_SCHEMA =\
"""
#!/bin/bash
# credits: http://www.pmoghadam.com/homepage/HTML/mysql-describe-all.html
TABLES=$(mysql -u{EMMDBUSER} -p{EMMDBPASS} {EMMDBNAME} -e 'show tables;' | tail -n +2)
for TABLE in $TABLES; do
echo; echo $TABLE
echo;
mysql -u{EMMDBUSER} -p{EMMDBPASS} {EMMDBNAME} -e "describe $TABLE"
done
echo; echo "ALL TABLES:"
mysql -u{EMMDBUSER} -p{EMMDBPASS} {EMMDBNAME} -e "show tables;"
""".format(**globals())
DBNAME= cp['mysql']['db_name']
USER = cp['mysql']['user']
PASS = cp['mysql']['password']
def sysexec(cmd):
......@@ -59,7 +20,7 @@ def sysexec(cmd):
.communicate()[0].decode("utf-8"))
def exec_mysql_script(script):
def exec_mysql(script):
''' Create a temp file with 'script' content and pass it to mysql
for execution. The script template is formatted with globals().
'''
......@@ -73,16 +34,55 @@ def exec_mysql_script(script):
if len(sys.argv) > 1:
if sys.argv[1] == 'init':
exec_mysql_script(SCRIPT_INIT)
exec_mysql('''
CREATE DATABASE IF NOT EXISTS {DBNAME}
CHARACTER SET utf8 COLLATE utf8_bin;
GRANT ALL on {DBNAME}.* TO '{USER}'@'localhost'
IDENTIFIED BY '{PASS}' WITH GRANT OPTION;
FLUSH PRIVILEGES;
'''.format(**globals()))
elif sys.argv[1] == 'user-create':
exec_mysql('''
elif sys.argv[1] == 'user':
exec_mysql_script(SCRIPT_USER)
#DROP USER '{USER}'@'localhost';
CREATE USER '{USER}'@'localhost' IDENTIFIED BY '{PASS}';
'''.format(**globals()))
elif sys.argv[1] == 'user-drop':
exec_mysql('''
DROP USER '{USER}'@'localhost';
'''.format(**globals()))
elif sys.argv[1] == 'summary':
exec_mysql_script(SCRIPT_SUMMARY)
exec_mysql('''
SELECT User, Host FROM mysql.user;
SHOW DATABASES;
''')
elif sys.argv[1] == 'showschema':
sysexec(SHSCRIPT_SHOW_SCHEMA)
sysexec('''\
#!/bin/bash
# credits: http://www.pmoghadam.com/homepage/HTML/mysql-describe-all.html
TABLES=$(mysql -u{USER} -p{PASS} {DBNAME} -e 'show tables;' | tail -n +2)
for TABLE in $TABLES; do
echo; echo $TABLE
echo;
mysql -u{USER} -p{PASS} {DBNAME} -e "describe $TABLE"
done
echo; echo "ALL TABLES:"
mysql -u{USER} -p{PASS} {DBNAME} -e "show tables;"
'''.format(**globals()))
else:
print('Invalid command!')
......
......@@ -4,17 +4,17 @@
DBNAME=$(python3 -c "from configparser import ConfigParser; \
cp = ConfigParser(); \
cp.read('settings.ini'); \
print(cp['db']['name'])");
print(cp['mysql']['db_name'])");
DBUSER=$(python3 -c "from configparser import ConfigParser; \
cp = ConfigParser(); \
cp.read('settings.ini'); \
print(cp['db']['user'])");
print(cp['mysql']['user'])");
DBPASS=$(python3 -c "from configparser import ConfigParser; \
cp = ConfigParser(); \
cp.read('settings.ini'); \
print(cp['db'].get('pass') or cp['app']['pass'])");
print(cp['mysql']['password'])");
TABLES=$(mysql -u$DBUSER -p$DBPASS $DBNAME -e 'show tables;' | tail -n +2)
......
'''
http://stackoverflow.com/questions/13934801/supervisord-logs-dont-show-my-ouput
'''
import jinja2
import readchar
from configparser import ConfigParser
from datetime import datetime
import os
import sys
import subprocess
import time
from multiprocessing import Process
def path_here(filename):
return os.path.realpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), filename))
cp = ConfigParser()
cp.read(path_here('../../settings.ini'))
APP_PATH = path_here('../..')
WSGI_PATH = path_here('')
APP_SECRET_KEY = cp['app']['secret_key']
APP_WSGI_PORT = cp['app']['wsgi_port']
NGINX_SERVER_NAME = cp['nginx']['server_name']
NGINX_SITE_FILE = cp['nginx']['site_file']
NGINX_SITE_ROOT = cp['nginx']['site_root']
NGINX_LOCATION_FILE = NGINX_SITE_FILE + '-' + NGINX_SITE_ROOT
NGINX_AVAILABLE_PATH = '/etc/nginx/sites-available/' + NGINX_SITE_FILE
NGINX_ENABLED_PATH = '/etc/nginx/sites-enabled/' + NGINX_SITE_FILE
NGINX_LOCATION_PATH = '/etc/nginx/conf.d/' + NGINX_LOCATION_FILE
GUNICORN_CONFIG_PATH = os.path.join(APP_PATH, '.gunicorn.conf')
SUPERVISOR_PROCESS = NGINX_LOCATION_FILE
SUPERVISOR_CONFIG_PATH = '/etc/supervisor/conf.d/{}.conf'.format(
SUPERVISOR_PROCESS)
def yesno(prompt, default='yes'):
print(prompt + (' [Y/n] ' if default == 'yes' else ' [y/N] '),
end='', flush=True)
ans = readchar.readchar().lower()
print(ans)
if ans == 'y':
return True
elif ans == 'n':
return False
else:
return True if default == 'yes' else False
def write_template_file(template, dest_filename):
with open(path_here(template)) as f_template:
with open(dest_filename, 'w') as f_dest:
contents = jinja2.Template(
f_template.read()).render(**globals())
f_dest.write(contents)
print("\n Writed file '{}'".format(dest_filename))
print('='*60 + '\n' + contents + '\n' + '='*60)
def shell_exec(command, decorate=False):
print('\n $ ' + command)
if decorate:
print('='*60)
proc = subprocess.Popen(command, shell=True)
proc.wait()
output_code = proc.poll()
if output_code != 0:
print('[Error] Last command fail with code', output_code)
raise SystemExit(0)
if decorate:
print('\n' + '='*60)
def deploy():
write_template_file('nginx_server.conf', NGINX_AVAILABLE_PATH)
write_template_file('nginx_location.conf', NGINX_LOCATION_PATH)
shell_exec('rm -f {}'.format(NGINX_ENABLED_PATH))
shell_exec('ln -s {} {}'.format(NGINX_AVAILABLE_PATH, NGINX_ENABLED_PATH))
shell_exec('nginx -s reload')
write_template_file('gunicorn.conf', GUNICORN_CONFIG_PATH)
write_template_file('supervisor.conf', SUPERVISOR_CONFIG_PATH)
Process(target=lambda: shell_exec('\
multitail -s 2 /var/log/supervisor/supervisord.log \
log/supervisor/stdout.log \
log/supervisor/stderr.log \
/var/log/nginx/access.log \