Commit 46d9f9a2 authored by Nelso Jost's avatar Nelso Jost

NEW: improved database layout

parent 5b6dfb1b
File mode changed from 100755 to 100644
......@@ -98,9 +98,10 @@ clean-logs:
rm -rf logs/*
clean: clean-venv reset-settings
py3clean app
rm -f app/db.sqlite
rm -rf doc/_build
sudo find . -type f -name "*.py[co]" -delete
sudo find . -type d -name "__pycache__" -delete
sudo rm -f app/db.sqlite
sudo rm -rf doc/_build
logtail-nginx-access:
tail production/nginx-gunicorn/logs/nginx-access.log
......
......@@ -3,9 +3,12 @@ from ...models import *
from . import api_blueprint as api
from flask import jsonify, request
import json
from flask import jsonify, request, make_response, Response, \
stream_with_context
import csv
import io
import json
from datetime import datetime
def JSONError(message):
......@@ -41,8 +44,9 @@ def post_rawsensordata():
if not board:
return JSONError("Invalid 'board_hash': board not found.")
for i, d in enumerate(js['data']):
for i, d in enumerate(js['data']):
print('validating', i, d)
try:
dt = datetime.strptime(d['datetime']['value'],
d['datetime']['format'])
......@@ -54,19 +58,85 @@ def post_rawsensordata():
return JSONError("JSON missing or invalid 'sensors' attribute at "
"data line {}".format(i))
for sensor_name, sensor_value in d['sensors'].items():
sensor_values = {}
for name, val in d['sensors'].items():
try:
sensor = board.get_sensor_or_404(sensor_name)
bs = board.get_sensor(name)
sensor_values[bs.id] = val
except:
return JSONError("Board '{}' does not have a sensor named '{}'."
.format(board.nickname, sensor_name))
try:
sensor.add_rawdata(datetime=dt, value=sensor_value,
avoid_duplication=True)
except Exception as e:
return JSONError('{}'.format(e))
.format(board.nickname, name))
try:
board.add_rawdata(dt=d['datetime']['value'],
dt_format=d['datetime']['format'],
sensor_values=sensor_values)
except Exception as e:
return JSONError('{}'.format(e))
db.session.commit()
try:
db.session.commit()
except Exception as e:
return JSONError('DBException: {}'.format(e))
return jsonify({'success': '{} new points were saved on the board.'
.format(len(js['data']))})
@api.route('/get/csv/rawsensordata/<int:board_id>',
methods=['GET', 'POST'])
def get_rawsensordata_all(board_id):
try:
board = Board.query.filter_by(id=board_id).first_or_404()
except Exception as e:
return 'DBException: {}'.format(e)
dt_format, csv_sep = '%Y-%m-%d %H:%M:%S', ';'
if 'sep' in request.args:
csv_sep = request.args['sep']
if csv_sep in ('t', 'tab', '\\t'):
csv_sep = '\t'
if 'dt_format' in request.args:
dt_format = request.args['dt_format']
if request.json:
js = request.json
if type(js) is str:
js = json.loads(js)
if 'dt_format' in js:
dt_format = js['dt_format']
if 'csv_sep' in js:
csv_sep = js['csv_sep']
try:
datetime.now().strftime(dt_format)
except:
return 'INVALID DATETIME FORMAT {}'.format(dt_format)
board_sensors = board.sensors.order_by('order')
csvfile = io.StringIO()
csvwriter = csv.writer(csvfile, delimiter=csv_sep)
csvwriter.writerow(['datetime{}'.format(dt_format)] +
[bs.sensor.name for bs in board_sensors])
def read_and_flush():
csvfile.seek(0)
data = csvfile.read()
csvfile.seek(0)
csvfile.truncate()
return data
def generate(is_first_row=True):
for i, rd in enumerate(board.rawdata):
csvwriter.writerow([rd.datetime.strftime(dt_format)] +
[rd.sensor_values[bs.id] for bs in board_sensors])
if i % 100:
yield read_and_flush()
return Response(stream_with_context(generate()), mimetype='text/csv',
headers={'Content-Disposition':
'attachment; filename=emm-board-{}.csv'.format(board.id)})
......@@ -49,7 +49,7 @@ class BoardSelectForm(Form):
@classmethod
def handle_post(cls, board=None):
print(request.form)
# print(request.form)
if 'board_id' in request.form and request.form['board_id'] != '-1':
cls.save_session_current_board(int(request.form['board_id']))
......@@ -109,39 +109,36 @@ class BoardManageForm(Form):
self.board_exposition.data = board.exposition.name
self.board_longitude.data = board.longitude
self.board_latitude.data = board.latitude
if request.method != 'POST':
self.sensors = [sensor.dump(
include=['unity_label', 'measurement_name'])
for sensor in board.sensors.all()]
else:
if request.method != 'POST':
self.sensors = config.DB_INIT['DEFAULT_SENSORS']
self.sensors = [Sensor.get(name=sensor_name).dump(
include=['measurement_name', 'unity_label'])
for sensor_name in config.DB_INIT['DEFAULT_SENSORS']]
def validate_board_nickname(self, field):
b = boards.get(nickname=field.data)
if b and b._userhash != self.board_userhash.data:
if boards.query.filter_by(nickname=field.data).first():
raise ValidationError("Já existe uma estação cadastrada com esse"
" apelido!")
def validate_and_save(self):
if self.validate():
b = boards.get(nickname=self.board_nickname.data)
r = request.form
print('new board', b)
if not b:
kwargs = {k.replace('board_', ''): v for k, v
in request.form.items()
if k != 'board_userhash' and k.startswith('board_')}
kwargs['sensors'] = self.sensors
try:
print('Trying to validate new board')
if self.validate():
print('Validated!')
r = request.form
kwargs = {k.replace('board_', ''): v
for k, v in request.form.items()
if k.startswith('board_')}
kwargs['sensors'] = [s['name'] for s in
self.gather_request_sensors()]
kwargs['user_id'] = current_user.id
print('gather sensors: ', kwargs['sensors'])
b = Board.add(**kwargs)
else:
b.nickname = r['board_nickname']
b.description = r['board_description']
b.exposition = Exposition.get(name=r['board_exposition'])
b.longitude = r['board_longitude']
b.latitude = r['board_latitude']
db.session.commit()
return b
print('new board:', b)
return b
except Exception as e:
print('validation exception:', e)
db.session.rollback()
def gather_request_sensors(self):
sensors = {}
......@@ -153,7 +150,7 @@ class BoardManageForm(Form):
sensors[i] = {field: v}
else:
sensors[i][field] = v
return [sensors[i] for i in range(len(sensors))]
return [sensors[i] for i in sorted(sensors.keys())]
class SensorListItemForm(Form):
......
......@@ -64,14 +64,14 @@ def sensor(id, sensor_nickname):
POST --> board.main(selected_id) | main.index;
'''
b = Board.query.get_or_404(id)
sensor = b.get_sensor(sensor_nickname)
s = b.get_sensor(sensor_nickname).sensor
if request.method == 'POST':
return BoardSelectForm.handle_post(b)
##TODO: improve loading system (data interpolation maybe?)
return render_template('board/sensor.html', board=b, the_sensor=sensor,
return render_template('board/sensor.html', board=b, the_sensor=s,
board_select_form = BoardSelectForm(selected_id=b.id))
......@@ -107,15 +107,11 @@ def insert():
''' GET --> templates/insertboard.html
POST | btn_cancel --> main.index
| btn_save --> main.view_board (new board)
| btn_add_sensor --> refresh (sensor list management)
| btn_delete_sensor --> refresh (sensor list management)
'''
##TODO: maybe its better to do the list managment on client side?
board_manage_form = BoardManageForm()
# pprint(request.form)
if request.method == 'POST':
# pprint(request.form)
pprint(request.form)
# got any button click? if so, the btn will appear on the request.form
if 'btn_cancel' in request.form:
return redirect(url_for('main.index'))
......
......@@ -10,12 +10,12 @@ def index():
''' GET --> index.html
POST --> board.view_board()
'''
boards = Board.dump(include_fields=['username', 'exposition_name',
'last_rawdata_json'],
ignore_fields=['_userhash'])
boards = Board.dump_all(include=['username', 'exposition_name',
'last_rawdata_json'],
exclude=['_userhash'])
if request.method == 'POST':
return BoardSelectForm.handle_post()
return render_template('index.html', board_select_form=BoardSelectForm(),
boards=boards)
boards=boards)
......@@ -27,46 +27,52 @@ MEASUREMENTS:
- name: ATMOSPHERIC_TEMPERATURE
label_en: Atmospheric Temperature
label_br: Temperatura atmosférica
short_br: TEMP
unities: ["ºC", "ºF", "K", "u.a."]
- name: AIR_RELATIVE_HUMIDITY
label_en: Relative Humidity of Air
label_br: Umidade relativa do ar
short_br: U.AR
unities: ["%", "u.a."]
- name: ATMOSPHERIC_PRESSURE
label_en: Atmospheric Pressure
label_br: Pressão atmosférica
short_br: PRESS
unities: ["Pa", "hPa", "mmHg", "atm", "u.a."]
- name: LUMINOSITY
label_en: Luminosity
label_br: Luminosidade ambiente
short_br: LUM
unities: ["u.a.", "lux"]
DEFAULT_SENSORS:
SENSORS:
- nickname: DHT22_TEMP
- name: DHT22_TEMP
measurement_name: ATMOSPHERIC_TEMPERATURE
unity_label: "ºC"
description: |
https://www.sparkfun.com/datasheets/Sensors/Temperature/DHT22.pdf
- nickname: DHT22_AH
- name: DHT22_AH
measurement_name: AIR_RELATIVE_HUMIDITY
unity_label: "%"
description: |
https://www.sparkfun.com/datasheets/Sensors/Temperature/DHT22.pdf
- nickname: BMP085_PRESSURE
- name: BMP085_PRESSURE
measurement_name: ATMOSPHERIC_PRESSURE
unity_label: "Pa"
description: |
http://www.adafruit.com/datasheets/BMP085_DataSheet_Rev.1.0_01July2008.pdf
- nickname: LDR
- name: LDR
measurement_name: LUMINOSITY
unity_label: "u.a."
description: |
http://www.biltek.tubitak.gov.tr/gelisim/elektronik/dosyalar/40/LDR_NSL19_M51.pdf
DEFAULT_SENSORS: [DHT22_TEMP, DHT22_AH, BMP085_PRESSURE, LDR]
......@@ -36,6 +36,8 @@ class DBInitCommand(Command):
Unity.import_from_json(*db_init['UNITIES'])
Measurement.import_from_json(*db_init['MEASUREMENTS'])
Sensor.import_from_json(*db_init['SENSORS'])
Board.add(nickname='Pezzi', user=user_admin,
longitude='-51.11948668956755',
latitude='-30.072849663041758',
......
......@@ -14,15 +14,16 @@ from .basemixin import BaseMixin, TableList
from .role import Role
from .exposition import Exposition
from .user import User
from .board import Board
from .unity import Unity
from .measurement import Measurement
from .unitymeasurement import UnityMeasurement
from .sensor import Sensor
from .boardsensor import BoardSensor
from .rawsensordata import RawSensorData
from .board import Board
del role, exposition, user, board, unity, measurement, unitymeasurement, \
sensor, rawsensordata
sensor, rawsensordata, boardsensor
# === EXPORT VARIABLES SETUP (no need to update) =============================
......
......@@ -12,25 +12,31 @@ class BaseMixin:
def json(self):
''' Returns a valid JSON dict with record values.
'''
return {col.name: getattr(self, col.name) for col in self.cols}
return {col.name: getattr(self, col.name + '_json')
if hasattr(self, col.name + '_json')
else getattr(self, col.name)
for col in self.cols}
@ClassProperty
@classmethod
def new_json(self):
return {col.name: None for col in self.cols}
def dump(self, include=None, exclude=None):
d = self.json
if exclude:
for field in exclude:
d.pop(field)
if include:
for field in include:
d[field] = getattr(self, field)
return d
@classmethod
def dump(cls, include_fields=None, ignore_fields=None):
def remove_fields(x):
j = x.json
if ignore_fields:
for field in ignore_fields:
j.pop(field)
if include_fields:
for field in include_fields:
j[field] = getattr(x, field)
return j
return [remove_fields(x) for x in cls.query.all()]
def dump_all(cls, include=None, exclude=None):
return [x.dump(include=include, exclude=exclude)
for x in cls.query]
@classmethod
def get(cls, *args, **kwargs):
......
This diff is collapsed.
from . import db, BaseMixin
class BoardSensor(db.Model, BaseMixin):
__tablename__ = 'boardsensors'
id = db.Column(db.Integer, primary_key=True)
board_id = db.Column(db.Integer, db.ForeignKey('boards.id'))
sensor_id = db.Column(db.Integer, db.ForeignKey('sensors.id'))
order = db.Column(db.Integer)
def __repr__(self):
return "<BoardSensor id={} board_id={} sensor_id={} sensor.name='{}' "\
"order={}>".format(self.id, self.board_id, self.sensor_id,
self.sensor.name, self.order)
......@@ -10,6 +10,7 @@ class Measurement(db.Model, BaseMixin):
name = db.Column(db.String(50), unique=True, nullable=False)
label_br = db.Column(db.String(50), unique=True, nullable=False)
label_en = db.Column(db.String(50), unique=True, nullable=False)
short_br = db.Column(db.String(10), unique=True, nullable=False)
unity_measurements = db.relationship('UnityMeasurement',
backref='measurement',
......
......@@ -3,11 +3,16 @@ from . import db, BaseMixin
class RawSensorData(db.Model, BaseMixin):
__tablename__ = 'rawsensordata'
id = db.Column(db.Integer, primary_key=True)
sensor_id = db.Column(db.Integer, db.ForeignKey('sensors.id'))
datetime = db.Column(db.DateTime())
value = db.Column(db.String(50))
id = db.Column(db.Integer, primary_key=True)
board_id = db.Column(db.Integer, db.ForeignKey('boards.id'))
datetime = db.Column(db.DateTime())
sensor_values = db.Column(db.PickleType)
def __repr__(self):
return "<RawSensorData id={} sensor_id={} datetime='{}' value={}>"\
.format(self.id, self.sensor_id, self.datetime, self.value)
return "<RawSensorData id={} board_id={} datetime='{}' sensor_values={}>"\
.format(self.id, self.board_id, self.datetime.isoformat(),
self.sensor_values)
@property
def datetime_json(self):
return self.datetime.isoformat()
from . import db, BaseMixin
from .rawsensordata import RawSensorData
from .unity import Unity
from .measurement import Measurement
from .unitymeasurement import UnityMeasurement
class Sensor(db.Model, BaseMixin):
__tablename__ = 'sensors'
id = db.Column(db.Integer, primary_key=True)
board_id = db.Column(db.Integer, db.ForeignKey('boards.id'))
nickname = db.Column(db.String(20))
description = db.Column(db.String(300))
unity_measurement_id = db.Column(db.Integer,
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(20))
description = db.Column(db.String(300))
unitymeasurement_id = db.Column(db.Integer,
db.ForeignKey('unitymeasurements.id'),
nullable=False)
rawdata = db.relationship('RawSensorData', backref='sensor',
lazy='dynamic')
boards = db.relationship('BoardSensor', backref='sensor',
lazy='dynamic')
def __repr__(self):
return "<Sensor id={} board_id={} nickname='{}' "\
"unity='{}' rawdata_count={}>".format(
self.id, self.board_id, self.nickname,
self.unity_measurement.unity.label,
self.rawdata.count())
return "<Sensor id={} name='{}' measurement='{}' unity='{}'>"\
.format(self.id, self.name, self.measurement.name,
self.unity.label)
@property
def measurement_name(self):
return self.unity_measurement.measurement.name;
return self.unitymeasurement.measurement.name
@property
def unity_label(self):
return self.unity_measurement.unity.label
@property
def last_rawdata(self):
return self.rawdata.order_by(RawSensorData.datetime.desc()).first()
def dump(self, include=None):
d = self.json
if include:
for field in include:
d[field] = getattr(self, field)
return d
def add_rawdata(self, datetime, value, avoid_duplication=False):
''' Add raw sensor data. RawSensorData fields are given by kwargs.
'''
if avoid_duplication:
if self.rawdata.filter_by(datetime=datetime).first():
raise ValueError('Datetime already exists for {}!'
.format(self.nickname))
db.session.add(RawSensorData(sensor_id=self.id,
datetime=datetime,
value=value))
return self.unitymeasurement.unity.label
@classmethod
def delete(cls, key):
''' Recursively remove all data related to the sensor matching the
given key, incluing itself.
def add(cls, **kwargs):
'''
sensor = cls.get(key)
sensor.rawdata.delete()
db.session.delete(sensor)
@classmethod
def new_from_json(cls, json_dict, board_id=None):
kw = {'id': cls.get_new_id()}
for field in cls.FIELDS:
kw[field] = json_dict.get(field)
if board_id is not None:
kw['board_id'] = board_id
return cls(**kw)
@classmethod
def add_from_json(cls, json_dict, board_id=None):
sensor = cls.new_from_json(json_dict, board_id)
Additional kwargs (besides):
unity_label (str)
measurement_name
'''
if 'unity_label' in kwargs and 'measurement_name' in kwargs:
u = Unity.get(label=kwargs.pop('unity_label'))
m = Measurement.get(name=kwargs.pop('measurement_name'))
kwargs['unitymeasurement_id'] = UnityMeasurement.get(
unity_id=u.id, measurement_id=m.id).id
else:
raise KeyError("Missing 'unity_label' and/or 'measurement_name' "
"attributes.")
sensor = Sensor(**kwargs)
db.session.add(sensor)
return sensor
def update_from_json(self, json_data):
for field in self.FIELDS:
setattr(self, field, json_data.get(field))
db.session.add(self)
def export_csv_file(self, filename, sep=','):
with open(filename, 'w') as f:
writer = csv.writer(f, delimiter=sep)
for d in self.rawdata.all():
writer.writerow([d.datetime_obj.strftime('%Y/%m/%d %H:%M:%S'),
d.value])
def get_dygraph_data(self):
''' Returns a string containing Javascript array code with values from
all the RawSensorData belonging to the given sensor.
'''
def row_generator(sensor):
for rawdata in sensor.rawdata.all():
dt = rawdata.datetime
yield '[new Date({Y}, {M}, {D}, {h}, {m}, {s}, 0), {y_value}]'\
.format(Y=(dt.year if dt.month != 1 else dt.year -1),
M=(dt.month-1 if dt.month != 1 else 12),
D=dt.day, h=dt.hour,
m=dt.minute, s=dt.second, y_value=rawdata.value)
# for some reason Dygraph.js render the month 1 month on the future!
return '[{}]'.format(','.join(r for r in row_generator(self)))
# def get_dygraph_data(self):
# ''' Returns a string containing Javascript array code with values from
# all the RawSensorData belonging to the given sensor.
# '''
# def row_generator(sensor):
# for rawdata in sensor.rawdata.all():
# dt = rawdata.datetime
# yield '"{Y}/{M}/{D} {h}:{m}:{s},{y_value}\\n"'\
# .format(Y=dt.year, M=dt.month, D=dt.day, h=dt.hour,
# m=dt.minute, s=dt.second, y_value=rawdata.value)
# return '"Data,{}\\n" + \n{}'.format(
# self.unity_measurement.measurement.name,
# '+ \n'.join(r for r in row_generator(self)))
......@@ -9,7 +9,7 @@ class UnityMeasurement(db.Model, BaseMixin):
unity_id = db.Column(db.Integer, db.ForeignKey('unities.id'),
nullable=False)
sensors = db.relationship('Sensor', backref='unity_measurement',
sensors = db.relationship('Sensor', backref='unitymeasurement',
lazy='dynamic')
def __repr__(self):
......
......@@ -10,7 +10,15 @@
color: white;
bottom: 43px;
left: -20px;
width: 200px;
/* width: 200px; */
}
.label-col {
text-align: right;
}
.popup-dt-col {
text-align: center;
}
th, td {
......
......@@ -40,13 +40,13 @@ var remove_sensor_row = function(sensor_button) {
var add_sensor_row = function(sensor) {
var $table_sensors = $("#table_sensors tr");
console.log($table_sensors);
// console.log($table_sensors);
var i = $table_sensors.length - 1;
var $row = $("<tr>");
var $new_nickname = $nickname.clone()
.attr("name", "sensor_nickname_" + String(i));
if (sensor) $new_nickname.val(sensor.nickname);
.attr("name", "sensor_name_" + String(i));
if (sensor) $new_nickname.val(sensor.name);
var $new_measurement_name = $measurement_name.clone()
.attr("name", "sensor_measurement_name_" + String(i))
......
......@@ -52,7 +52,7 @@ var iconStyle = new ol.style.Style({
anchorYUnits: 'fraction',
opacity: 0.75,
scale: 0.025,
src: '/static/images/red-pin.svg' }))
src: image_pin_filename }))
});
//add the feature vector to the layer vector, and apply a style to whole layer
......
console.log(boards);
var sp = '&nbsp;';
String.prototype.supplant = function (o) {
return this.replace(/{([^{}]*)}/g,
......@@ -73,7 +74,7 @@ var iconStyle = new ol.style.Style({
anchorYUnits: 'fraction',
opacity: 0.75,
scale: 0.025,
src: '/static/images/red-pin.svg' }))
src: image_pin_filename }))
});
//add the feature vector to the layer vector, and apply a style to whole layer
......@@ -86,26 +87,29 @@ var rasterLayer = new ol.layer.Tile({
source: new ol.source.OSM()
});
/**
* Create the map.
*/
var map = new ol.Map({
target: 'map',
// HTML <div> where the map is gonna appear
target: 'map',
layers: [rasterLayer, vectorLayer],
// types of drawing to be rendered
layers: [rasterLayer, vectorLayer],
overlays: [popup_overlay, label_overlay],
// supported <div> on top of all map layers that can be showed/hided
overlays: [popup_overlay, label_overlay],
controls: ol.control.defaults({
attributionOptions: /** @type {olx.control.AttributionOptions} */ ({
// support mouse interactions (hover and clicks)
controls: ol.control.defaults({
attributionOptions: /** @type {olx.control.AttributionOptions} */ ({
collapsible: false
})
})
}).extend([mousePositionControl]),
view: new ol.View({
center: ol.proj.fromLonLat([-51.1528, -30.0525]),
zoom: 12
})
// present map window contents