Commit 27cccac5 authored by Nelso Jost's avatar Nelso Jost

FIX: refactor finished and documentation improved

parent 8074a13f
PYVER := 3 PYBIN := python3
VENV := .venv VENV := .venv
VENVPY := ${VENV}/bin/python
INO_DIR := .ino INO_DIR := .ino
USE := ino USE := ino
...@@ -74,8 +75,8 @@ setup: clean-venv create-venv ...@@ -74,8 +75,8 @@ setup: clean-venv create-venv
create-venv: create-venv:
@ echo "-------------------------------------------------------" @ echo "-------------------------------------------------------"
virtualenv -v --python='python${PYVER}' ${VENV} virtualenv -v --python='${PYBIN}' ${VENV}
@ echo "Virtualenv with 'python${PYVER}' interpreter was created at ${VENV}" @ echo "Virtualenv with '${PYBIN}' interpreter was created at ${VENV}"
@ echo "-------------------------------------------------------" @ echo "-------------------------------------------------------"
${VENV}/bin/pip install --upgrade pip ${VENV}/bin/pip install --upgrade pip
@ echo "-------------------------------------------------------" @ echo "-------------------------------------------------------"
...@@ -90,26 +91,26 @@ clean-venv: ...@@ -90,26 +91,26 @@ clean-venv:
rm -rf ${VENV} rm -rf ${VENV}
serial: serial:
${VENV}/bin/python -i logger/init_serial.py ${VENVPY} -i logger/init_serial.py
firmware: ${USE}-install firmware: ${USE}-install
chmod +x scripts/ino-build.sh chmod +x scripts/ino-build.sh
./scripts/ino-build.sh ${USE} ${INO_DIR} ${ARDUINO} ./scripts/ino-build.sh ${USE} ${INO_DIR} ${ARDUINO}
sync-rtc: sync-rtc:
${VENV}/bin/python${PYVER} logger/run.py --syncrtc ${VENVPY} logger/run.py --syncrtc
run: logger-run:
${VENV}/bin/python logger/run.py ${VENVPY} logger/run.py --verbose
deploy: undeploy deploy: undeploy
mkdir -p logger/logs mkdir -p logger/logs
sudo ${VENV}/bin/python${PYVER} logger/deploy.py sudo ${VENVPY} logger/deploy.py
undeploy: undeploy:
sudo ${VENV}/bin/python${PYVER} logger/deploy.py -u sudo ${VENVPY} logger/deploy.py -u
tail-log: logger-tail:
$(eval TMP := $(shell ls -t -I "pid*|stdout*" logger/logs | head -n 1)) $(eval TMP := $(shell ls -t -I "pid*|stdout*" logger/logs | head -n 1))
@ echo "Last log file updated: logger/logs/$(TMP)" @ echo "Last log file updated: logger/logs/$(TMP)"
@ echo "File size: `du -h logger/logs/$(TMP) | cut -f1`" @ echo "File size: `du -h logger/logs/$(TMP) | cut -f1`"
...@@ -131,7 +132,7 @@ plot-data: ...@@ -131,7 +132,7 @@ plot-data:
@ cd tools && gnuplot -persist -e "config='config.plt'; col=${col}" loop.plt @ cd tools && gnuplot -persist -e "config='config.plt'; col=${col}" loop.plt
clean-data: clean-data:
rm -rfv data/*.csv data/outgoing/*.json rm -rfv data/*
clean-logs: clean-logs:
rm -rfv logger/logs/* rm -rfv logger/logs/*
......
This diff is collapsed.
# Makefile for Sphinx documentation # Makefile for Sphinx documentation
#
# You can set these variables from the command line. # You can set these variables from the command line.
SPHINXOPTS = SPHINXOPTS =
SPHINXBUILD = sphinx-build SPHINXBUILD = sphinx-build
...@@ -8,9 +7,9 @@ PAPER = ...@@ -8,9 +7,9 @@ PAPER =
BUILDDIR = _build BUILDDIR = _build
# User-friendly check for sphinx-build # User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) # ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) # $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif # endif
# Internal variables. # Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_a4 = -D latex_paper_size=a4
...@@ -19,7 +18,7 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . ...@@ -19,7 +18,7 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others # the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext setup
help: help:
@echo "Please use \`make <target>' where <target> is one of" @echo "Please use \`make <target>' where <target> is one of"
......
from .main import Meteorologger
import configparser import configparser
import sys import sys
import os
from pprint import pprint
DEFAULT_SETTINGS =\
DEFAULT_INI=\
"""\ """\
[server] [server]
; ID of the board to which data will be uploaded to ; base URL of the server
; URL = http://dados.cta.if.ufrgs.br/emm
; board identification number (auto generated when a new board is created)
BOARD_ID = BOARD_ID =
; authentication token for the board's user ; authentication token for the board's user
;
USER_HASH = USER_HASH =
; wep app base URL
;
URL = http://dados.cta.if.ufrgs.br/emm
[logger] [reading]
; comma-separated list of sensor nicknames that will be read ; CSV list of sensor names/nicknames (order reflect columns on datalog files)
; the order reflect columns of the local DATALOG csv file
;
SENSORS = DHT22_TEMP, DHT22_AH, BMP085_PRESSURE, LDR SENSORS = DHT22_TEMP, DHT22_AH, BMP085_PRESSURE, LDR
; time between readings attempts (cycles of the logger execution) ; time between logger cycles -- format: hours:minutes:seconds
; format: hours:minutes:seconds
;
INTERVAL = 0:5:0 INTERVAL = 0:5:0
; True for try reading RTC_DS1307 or False for use the system time ; true for read board clock; if fail, or false, system's time will be used
; if the RTC reading fails, the system time will be used instead RTC_DS1307 = true
;
USE_RTC_DS1307 = False
; expected time stamp of the RTC (see Python's datetime module documentation)
;
RTC_DATETIME_FORMAT = %Y-%m-%d %H:%M:%S
[datalog] [datalog]
; CSV delimiter (sugestions: ',' or ';' or '\t') ; CSV delimiter for local log files (sugestions: ',' or ';' or '\t')
;
CSV_SEP = '\t' CSV_SEP = '\t'
; format of the datetime column (see Python docs on datetime module) ; format of the datetime column (see Python docs on datetime module)
;
DATETIME_FORMAT = %Y-%m-%d-%H-%M-%S DATETIME_FORMAT = %Y-%m-%d-%H-%M-%S
[arduino] [arduino]
; USB port that have the board plugged in ; CSV list of ports to attempt or blank for automatic search on
; keep blank for automatic search on /dev/ttyACM* and /dev/ttyUSB* ; /dev/ttyACM* and /dev/ttyUSB* (where * varies from 0 to 10)
;
SERIAL_PORT = SERIAL_PORT =
""" """
def make_path_here(filename): def make_path_here(filename):
''' Append filename to the current __file__ path. ''' ''' Append filename to the current __file__ path. '''
return os.path.join(os.path.abspath(os.path.dirname(__file__)), filename) return os.path.join(os.path.abspath(os.path.dirname(__file__)), filename)
class Config(dict):
class RTCDateTime:
SENSOR_NAME = 'RTC_DS1307'
READ_TIMESTAMP = '%Y-%m-%d %H:%M:%S'
__qualname__ = "RTCDateTime fmt='{}'".format(READ_TIMESTAMP)
def __init__(self, s):
self.dt = datetime.strptime(s, self.RTC_DT_FMT)
def __str__(self):
return self.dt.strftime('%Y%m%d%H%M%S')
class Config:
SETTINGS_FILENAME = make_path_here('../../settings.ini') SETTINGS_FILENAME = make_path_here('../../settings.ini')
DATALOG_PATH = make_path_here('../../data/')
EXECUTION_LOG_PATH = make_path_here('../logs/')
OUTGOING_FILENAME = make_path_here('../../data/outgoing.json')
URL_POST_RAWSENSORDATA = '{base}api/post/rawsensordata/{bid}' URL_API_POST_RAWSENSORDATA = '{base}api/post/rawsensordata/{bid}'
SERIAL_CSV_SEP = ','
def __init__(self): def __init__(self):
super().__init__() super().__init__()
self.load_settings() self.default = configparser.ConfigParser()
self.default.read_string(DEFAULT_INI)
try:
self.load_settings()
except Exception as e:
print('\nConfigurationError:', e)
def __getitem__(self, key):
return self._parser._sections[key]
def ask_restore_settings_file(self): def ask_restore_settings_file(self):
a = input("\nRestore default file now and overwrite all a = input("\nRestore default file now and overwrite all"
"current values? [y/N] ") "current values? [y/N] ")
if 'y' in a.lowercase() if 'y' in a.lowercase():
try: try:
with open(self.SETTINGS_FILENAME, 'w') as f: with open(self.SETTINGS_FILENAME, 'w') as f:
f.write(DEFAULT_SETTINGS) f.write(DEFAULT_SETTINGS)
...@@ -84,53 +99,94 @@ class Config(dict): ...@@ -84,53 +99,94 @@ class Config(dict):
sys.exit(1) sys.exit(1)
def get(self, section, key, validate, ask_restore=True): def load_settings(self):
try:
r = self.config[section]
except:
print("Config file\n {}\n\nis missing the section [{}]!"
.format(self.SETTINGS_FILENAME, section))
if ask_restore:
self.ask_restore_settings_file()
try:
r = self.config[section][key]
print("On the config file\n {}\n\nsection [{}] is missing the "
"key '{}'!")
if ask_restore:
self.ask_restore_settings_file()
try:
r = validate(self.config[section][key])
except:
print("On the config file\n {}\n\ninvalid value for '{}' for "
"key the [{}][{}]!".format(""))
if ask_restore:
self.ask_restore_settings_file()
def load_settings(self, filename):
''' '''
Load the configuration file onto the self.CFG attribute. Load the INI configuration file onto the self._parser attribute.
Some keys will be tested and filenames will be normalized.
''' '''
self.config = configparser.ConfigParser() self._parser = configparser.ConfigParser()
try: try:
self.config.read(self.SETTINGS_FILENAME) self._parser.read(self.SETTINGS_FILENAME)
except: except:
print("Unable to open configuration file at\n {}".format( print("Unable to open configuration file at\n {}".format(
self.SETTINGS_FILENAME)) self.SETTINGS_FILENAME))
self.ask_restore_settings_file()
self.URL_POST_RAWSENSORDATA = self.URL_POST_RAWSENSORDATA.format( pprint(self._parser._sections)
base=self.CFG('server', 'URL',
lambda x: x if x.endswith('/') else x + '/'), self.validate_server_url()
bid=self.CFG('server', 'BOARD_ID', int) self.validate_reading_sensors()
self.validate_reading_interval()
self.validate_datalog_csv_sep()
self.validate_arduino_serial_port()
pprint(self._parser._sections)
self.LOGGER_INTERVAL_SECONDS =\
self.CFG['LOGGER']['INTERVAL']['seconds']\
+ 60 * self.CFG['LOGGER']['INTERVAL']['minutes']\
+ 3600 * self.CFG['LOGGER']['INTERVAL']['hours']
self.DATALOG_CSV_SEP = bytes(self.CFG['DATALOG']['CSV_SEP'], def validate_server_url(self):
'utf8').decode('unicode_escape') try:
bid = int(self['server']['board_id'])
except:
raise Exception(
"Invalid numeric value for the server/BOARD_ID key.\n"
"Expected to be of type integer.\n"
"Was given:\n {}".format(self['server']['board_id']))
self['server']['api_post_url'] = self.URL_API_POST_RAWSENSORDATA\
.format(base=self['server']['url'] +
('' if self['server']['url'].endswith('/') else '/'),
bid=bid)
def validate_reading_sensors(self):
self['reading']['sensors'] = [x.strip() for x in
self['reading']['sensors'].split(',') if x.strip() != '']
if len(self['reading']['sensors']) == 0:
raise Exception(
"At least one sensor must be present on the reading/"
"SENSORS key.")
if 'true' in self['reading']['rtc_ds1307'].lower():
self['reading']['sensors'].append(RTCDateTime.SENSOR_NAME)
self['reading']['command'] = 'readSensors,' + ','.join(
self['reading']['sensors'])
def validate_reading_interval(self):
try:
numbers = self['reading']['interval'].split(':')
if len(numbers) != 3: raise Exception
self['reading']['interval'] = {k: int(v) for k, v in
zip(['H', 'M', 'S'], numbers)}
self['reading']['interval_seconds'] =\
3600 * self['reading']['interval']['H'] + \
60 * self['reading']['interval']['M'] + \
self['reading']['interval']['S']
except:
raise Exception(
"Invalid time value for the reading/INTERVAL key.\n"
"Expected format:\n"
" hours:minutes:seconds (integer numbers)\n"
"Was given:\n {}".format(self['reading']['interval']))
def validate_datalog_csv_sep(self):
value = self['datalog']['csv_sep']
self['datalog']['csv_sep'] = bytes(value, 'utf8').decode(
'unicode_escape')
if not value in (',', ';', r'\t'):
raise Exception(
"Invalid character value for the datalog/CSV_SEP key.\n"
"Supported values:\n ',' ';' '\\t'\n"
"Was given:\n {}".format(value))
def validate_arduino_serial_port(self):
self['arduino']['serial_port'] = [x.strip() for x in
self['arduino']['serial_port'].split(',') if x.strip() != '']
if len(self['arduino']['serial_port']) == 0:
self['arduino']['serial_port'] = []
for i in range(5):
self['arduino']['serial_port'].append('/dev/ttyACM{}'.format(i))
self['arduino']['serial_port'].append('/dev/ttyUSB{}'.format(i))
This diff is collapsed.
type: map
mapping:
SERVER:
required: true
type: map
mapping:
API_POST_URL:
required: true
type: str
LOGGER:
required: true
type: map
mapping:
SENSORS:
required: true
type: seq
sequence:
- type: str
unique: true
INTERVAL:
# default: {days: 0, hours: 0, minutes: 0, seconds: 10}
required: true
type: map
mapping:
days:
required: true
type: int
range:
min: 0
hours:
required: true
type: int
range:
min: 0
minutes:
required: true
type: int
range:
min: 0
seconds:
required: true
type: int
range:
min: 0
USE_RTC_DATETIME:
required: true
allowempty: true
type: map
RTC_DATETIME_FORMAT:
# default: '%Y-%m-%d %H:%M%:%S'
required: true
type: str
ARDUINO:
required: true
type: map
mapping:
SERIAL_PORTS:
required: true
type: seq
sequence:
- type: str
BAUD_RATE:
required: true
type: int
enum: [300, 1200, 2400, 4800, 9600, 14400, 19200, 28800, 38400, 57600, 115200]
DATALOG:
required: true
type: map
mapping:
CSV_SEP:
required: true
type: str
DATETIME_FORMAT:
required: true
type: str
SENSORS_AVAILABLE:
required: true
type: map
allowempty: true
[program:{{ process_name }}]
command={{ base_dir }}/.venv/bin/python3 {{ base_dir }}/run.py -s
directory={{ base_dir }}
user=root
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile={{ base_dir }}/logs/stdout.log
...@@ -15,7 +15,7 @@ PID_FILENAME = 'logger/logs/pid_{}'.format(PROCESS_NAME) ...@@ -15,7 +15,7 @@ PID_FILENAME = 'logger/logs/pid_{}'.format(PROCESS_NAME)
TEMPLATE_SUPERVISOR_CONF =\ TEMPLATE_SUPERVISOR_CONF =\
"""\ """\
[program:{PROCESS_NAME}] [program:{PROCESS_NAME}]
command={BASE_DIR}/.venv/bin/python3 {BASE_DIR}/logger/run.py -s command={BASE_DIR}/.venv/bin/python {BASE_DIR}/logger/run.py
directory={BASE_DIR} directory={BASE_DIR}
user=root user=root
autostart=true autostart=true
...@@ -53,7 +53,7 @@ def deploy_supervisor(): ...@@ -53,7 +53,7 @@ def deploy_supervisor():
pid = int(pid) pid = int(pid)
with open(PID_FILENAME, 'w') as f: with open(PID_FILENAME, 'w') as f:
f.write(str(pid) + '\n') f.write(str(pid) + '\n')
except: except:
print("\nUnable to save PID={} at '{}'".format(pid, PID_FILENAME)) print("\nUnable to save PID={} at '{}'".format(pid, PID_FILENAME))
......
...@@ -4,7 +4,6 @@ import time ...@@ -4,7 +4,6 @@ import time
from app import Meteorologger from app import Meteorologger
ser = Meteorologger().get_serial() ser = Meteorologger().get_serial()
time.sleep(2) # waiting for board reset
def send(command_str, response_wait=0): def send(command_str, response_wait=0):
''' '''
......
docopt==0.6.2
MarkupSafe==0.23
pykwalify==1.2.0
pyreadline==2.0
pyserial==2.7 pyserial==2.7
python-dateutil==2.4.2
PyYAML==3.11
requests==2.6.0 requests==2.6.0
six==1.9.0
from app import Meteorologger from app.main import Meteorologger
import sys import sys
if __name__ == '__main__': if __name__ == '__main__':
if '--syncrtc' in sys.argv: if '--syncrtc' in sys.argv:
Meteorologger().sync_rtc() Meteorologger().sync_rtc()
else: else:
Meteorologger(verbose = False if '-s' in sys.argv else True)#.run() Meteorologger(verbose='--verbose' in sys.argv).run()
[server] [server]
; ID of the board to which data will be uploaded to ; base URL of the server
; URL = http://localhost:5000
BOARD_ID = 2
; authentication token for the board's user ; board identification number (auto generated when a new board is created)
; BOARD_ID = 2
USER_HASH = pbkdf2:sha1:1000$jVxGFjnU$c5f4a675969d620185660b10968c8027f05e01e7
; wep app base URL ; authentication token for the board's user
; USER_HASH = pbkdf2:sha1:1000$ltKczydK$ec43131e8d1694e4b1e8c1d1fccbfe3b68c5dcf5
URL = http://localhost:5000/emm
[logger] [reading]
; comma-separated list of sensor nicknames that will be read ; CSV list of sensor names/nicknames (order reflect columns on datalog files)
; the order reflect columns of the local DATALOG csv file
;
SENSORS = DHT22_TEMP, DHT22_AH, BMP085_PRESSURE, LDR SENSORS = DHT22_TEMP, DHT22_AH, BMP085_PRESSURE, LDR
; time between readings attempts (cycles of the logger execution) ; time between logger cycles -- format: hours:minutes:seconds
; format: hours:minutes:seconds INTERVAL = 0:0:10
;
INTERVAL = 0:5:0
; True for try reading RTC_DS1307 or False for use the system time
; if the RTC reading fails, the system time will be used instead
;
USE_RTC_DS1307 = False
; expected time stamp of the RTC (see Python's datetime module documentation) ; true for read board clock; if fail, or false, system's time will be used
; RTC_DS1307 = true
RTC_DATETIME_FORMAT = %Y-%m-%d %H:%M:%S
[datalog] [datalog]
; CSV delimiter (sugestions: ',' or ';' or '\t') ; CSV delimiter for local log files (valid options: , or ; or \t )
; CSV_SEP = \t
CSV_SEP = '\t'
; format of the datetime column (see Python docs on datetime module) ; format of the datetime column (see Python docs on datetime module)
;
DATETIME_FORMAT = %Y-%m-%d-%H-%M-%S DATETIME_FORMAT = %Y-%m-%d-%H-%M-%S
[arduino] [arduino]
; USB port that have the board plugged in ; CSV list of ports to attempt or blank for automatic search on
; keep blank for automatic search on /dev/ttyACM* and /dev/ttyUSB* ; /dev/ttyACM* and /dev/ttyUSB* (where * varies from 0 to 10)
;
SERIAL_PORT = SERIAL_PORT =
SERVER:
# full URL that accepts POST method for sending data to the server
# WARNING: replace the end 'BID' with a valid board ID number
#
API_POST_URL: 'http://dados.cta.if.ufrgs.br/emm/api/post/rawsensordata/BID'
LOGGER:
# list of sensors to be read (ordering reflect columns of DATALOG file)
# to ignore one, comment it out by putting # in the beginning of the line
#
SENSORS:
- DHT22_TEMP
- DHT22_AH
- BMP085_PRESSURE
- LDR
# time between readings attempts (cycles of the logger execution)
#
INTERVAL: {days: 0, hours: 0, minutes: 5, seconds: 0}
# set false to use the system time or the RTC sensor name to use instead
# if the reading is invalid somehow, the system's time will be used instead
#
USE_RTC_DATETIME: none
# expected format of the time stamp returned by the RTC reading
#
RTC_DATETIME_FORMAT: '%Y-%m-%d %H:%M:%S'
ARDUINO:
# list of USB ports (the first one will be used for ino upload)
# Uno and Mega usually appears at /dev/ttyACMx; else try /dev/ttyUSBx
#
SERIAL_PORTS:
- /dev/ttyACM0
- /dev/ttyUSB0
- /dev/ttyACM1
- /dev/ttyUSB1
# Arduino serial communication protocol (same as in meteorolog.ino)
# OBS: 115200 seems unstable with Uno + PySerial
#
BAUD_RATE: 9600
DATALOG:
# CSV delimiter to be used on the file DATA_LOG_FILENAME
# Sugestions: ',' or ';' or '\t' for tab (do not use the period '.')
#
CSV_SEP: '\t'
# format of the datetime column (see Python docs on datetime module)
#
DATETIME_FORMAT: '%Y%m%d%H%M%S'
# ============================================================================
# list of all the sensors available for this board, and their specs
# WARNING: should be synced with the server
# !! Edit only if you know what you are doing !!
#
SENSORS_AVAILABLE:
RTC_DS1307:
data_format: datetime
DHT22_TEMP:
data_format: float
DHT22_AH:
data_format: float
BMP085_PRESSURE:
data_format: int
LDR:
data_format: float