Skip to content

GitLab

Commit f0751a3a authored by Nelso Jost's avatar Nelso Jost
Browse files

NEW: better usage; single-command for setup, build, upload and deploy-logger

parent 2da2c4ba
Hide whitespace changes
Inline Side-by-side
Makefile
View file @ f0751a3a
PYVER := 3
VENV := .venv
INODIR := .ino
all: help
help:
@ echo "USAGE"
@ echo "USAGE: make <command>"
@ echo ""
@ echo " make apt-install -- Uses Debian's apt to install required system tools"
@ echo "COMMANDS:"
@ echo ""
@ echo " make setup -- Install Python requirements (need only once)"
@ echo " full -- setup & bu & deploy-logger"
@ echo ""
@ echo " make build -- Compile the sketch"
@ echo " make upload -- Send the compiled sketch to the board"
@ echo " make serial -- Enter IPython session with serial opened"
@ echo " apt-install -- Uses Debian's apt to install required system tools"
@ echo " venv -- Creates a Python local virtual environment at ${VENV}"
@ echo " setup -- apt-install & venv"
@ echo ""
@ echo " make bu -- build & upload"
@ echo " make bus -- build & upload & serial"
@ echo " build -- Compile the sketch usin Ino Tool"
@ echo " upload -- Send the compiled sketch to the board using Ino Tool"
@ echo " serial -- Enter IPython session with serial opened"
@ echo " bu -- build & upload"
@ echo " bus -- build & upload & serial"
@ echo ""
@ echo " make sync-rtc -- Synchronizes the board's clock with the system's"
@ echo " sync-rtc -- Synchronizes the board's clock with the system's"
@ echo ""
@ echo " run-logger -- Starts the logger. Keep log at logger/logs"
@ echo " deploy-logger -- Activate logger daemon. Keep log at logger/logs/"
@ echo " undeploy-logger -- Deactivate logger daemon"
@ echo ""
@ echo " tail-log -- Exibits and follow last modified execution log file"
@ echo " tail-data -- Exibits and follow last modified datalog file"
@ echo ""
@ echo " plot-data col=<C> -- Uses Gnuplot to plot last modified datalog file"
@ echo " Replace <C> with the column number to plot as y axis"
@ echo " clean-venv -- Remove Python's virtual environment directory"
@ echo " clean-ino -- Remove Ino folder"
@ echo " clean-data -- !!! CAUTION !!! Remove all datalog files"
@ echo " clean-logs -- !!! CAUTION !!! Remove all execution log files"
@ echo " clean-all -- Performs all the above cleans"
@ echo ""
@ echo " make run-logger -- Starts the logger. Log: apper on the screen"
@ echo " make deploy-logger -- Activate logger daemon. Log at logger/logs/execution.log"
@ echo " make undeploy-logger -- Deactivate logger daemon."
@ echo " make tail-logger -- Exibits and follow daemon execution log."
apt-install:
sudo apt-get install python python3 python3-pip supervisor
sudo apt-get install python python3 python3-pip supervisor python-pip
sudo pip${PYVER} install virtualenv
install-ino:
sudo apt-get install python-pip
sudo pip2 install ino
setup: apt-install venv
setup: clean-venv
venv: clean-venv
@ echo "-------------------------------------------------------"
virtualenv -v --python='python${PYVER}' ${VENV}
@ echo "Virtualenv with 'python${PYVER}' interpreter was created at ${VENV}"
......@@ -49,21 +63,20 @@ setup: clean-venv
clean-venv:
rm -rf ${VENV}
clean:
rm -rf .build src/*
build:
mkdir -p .ino
mkdir -p .ino/src/ .ino/lib/
cp -rf meteorolog/. .ino/src/
cp logger/ino.ini .ino/
cd .ino/ && ino build
$(eval MODEL := $(shell ${VENV}/bin/python -c "from logger.app import Meteorologger; print(Meteorologger().CFG['ARDUINO']['BOARD_MODEL'])"))
mkdir -p ${INODIR}
mkdir -p ${INODIR}/src/ ${INODIR}/lib/
cp -rf meteorolog/. ${INODIR}/src/
cd ${INODIR}/ && ino build -m $(MODEL)
upload:
cd .ino/ && ino upload
$(eval MODEL := $(shell ${VENV}/bin/python -c "from logger.app import Meteorologger; print(Meteorologger().CFG['ARDUINO']['BOARD_MODEL'])"))
$(eval PORT := $(shell ${VENV}/bin/python -c "from logger.app import Meteorologger; print(Meteorologger().CFG['ARDUINO']['SERIAL_PORTS'][0])"))
cd ${INODIR}/ && ino upload -m $(MODEL) -p $(PORT)
serial:
${VENV}/bin/ipython3 -i logger/init_serial.py
${VENV}/bin/ipython -i logger/init_serial.py
bu: build upload
......@@ -77,14 +90,42 @@ run-logger:
deploy-logger: undeploy-logger
mkdir -p logger/logs
sudo touch data/datalog.csv data/outgoing.json
sudo chmod a+w data/*
sudo ${VENV}/bin/python${PYVER} logger/deploy.py
undeploy-logger:
sudo ${VENV}/bin/python${PYVER} logger/deploy.py -u
tail-logger:
tail -F logger/logs/execution.log
$(eval TMP := $(shell ls -t -I stdout* logger/logs/ | head -n 1))
@ echo "Last log file updated: logger/logs/$(TMP)"
@ echo "File size: `du -h logger/logs/$(TMP) | cut -f1`"
@ echo ""
@ tail -F logger/logs/$(TMP)
plot-data:
@ echo "Quit by closing the window with Q and hitting Ctrl+C here to end the process"
@ cd tools && gnuplot -persist -e "config='config.plt'; col=${col}" loop.plt
tail-data:
$(eval TMP := $(shell ls -t -I outgoing* data/ | head -n 1))
@ echo "Last datalog file updated: data/$(TMP)"
@ echo "Number of lines/points: `cat data/$(TMP) | wc -l`"
@ echo "File size: `du -h data/$(TMP) | cut -f1`"
@ echo ""
@ head -1 data/$(TMP)
@ echo ""
@ tail -F data/$(TMP)
clean-data:
rm -rfv data/*.csv data/outgoing/*.json
clean-logs:
rm -rfv logger/logs/*
clean-ino:
rm -rf ${INODIR}
clean-all: clean-data clean-logs clean-ino clean-venv
cd logger && sudo py3clean app
full: apt-install setup install-ino bu deploy-logger
full: setup bu deploy-logger
logger/app/main.py
View file @ f0751a3a
......@@ -50,7 +50,8 @@ class Meteorologger:
SETTINGS_SCHEMA_FILENAME = 'logger/app/settings_schema.yaml'
SETTINGS_FILENAME = 'settings.yaml'
EXECUTION_LOG_FILENAME = 'logger/logs/execution.log'
DATALOG_DIR = 'data/'
EXECUTION_LOG_PATH = 'logger/logs/'
OUTGOING_BASENAME = 'outgoing.json'
FILE_TIMESTAMP_FORMAT = '%Y-%m-%d-%H-%M-%S'
......@@ -62,7 +63,7 @@ class Meteorologger:
SERIAL_READ_TIMEOUT = 1.5 # seconds
FIND_PORT_TIMEOUT = 10 # seconds
BOARD_RESET_TIMEOUT = 2 # seconds
BOARD_RESPONSE_DELAY = 0 # seconds
BOARD_RESPONSE_DELAY = 3 # seconds
verbose = True
......@@ -99,14 +100,15 @@ class Meteorologger:
print("\nPlease fix it up or regenerate it.")
sys.exit(1)
self.SERIAL_PORTS = self.CFG['ARDUINO']['SERIAL_PORT'].split(',')
self.READING_INTERVAL_SECONDS =\
self.CFG['LOGGER']['INTERVAL']['seconds']\
+ 60 * self.CFG['LOGGER']['INTERVAL']['minutes']\
+ 3600 * self.CFG['LOGGER']['INTERVAL']['hours']\
+ 86400 * self.CFG['LOGGER']['INTERVAL']['days']
self.DATALOG_CSV_SEP = bytes(self.CFG['DATALOG']['CSV_SEP'],
'utf8').decode('unicode_escape')
def create_json(self, raw_line):
'''
Given the raw serial line response (CSV string), builds and returns
......@@ -128,7 +130,7 @@ class Meteorologger:
type_name = self.CFG['SENSORS_AVAILABLE'][nickname]['data_format']
if type_name == 'datetime':
if using_rtc and not v.strip().startswith('<'):
if using_rtc and not 'not_found' in v.strip():
d['datetime']['value'] = datetime.strptime(
v, rtc_datetime_fmt).strftime(d['datetime']['format'])
continue
......@@ -136,8 +138,9 @@ class Meteorologger:
try:
v = self.DATA_FORMATS[type_name](v.strip())
except:
logging.error("[{}]: '{}' is not a valid {}"
logging.warning("[{}]: '{}' is not a valid {}"
.format(nickname, v, type_name))
d['sensors'][nickname] = 'NaN'
continue
d['sensors'][nickname] = v
......@@ -157,19 +160,17 @@ class Meteorologger:
DATALOG_CSV as specficied on self.SETTINGS_FILENAME.
'''
# convert raw str into normal escaped str (e.g., r'\\t' --> '\t')
csv_sep = bytes(self.CFG['DATALOG']['CSV_SEP'],
'utf8').decode('unicode_escape')
csv_line = json_data['datetime']['value'] + csv_sep
csv_line = json_data['datetime']['value'] + self.DATALOG_CSV_SEP
for nickname in self.CFG['LOGGER']['SENSORS']:
if nickname in json_data['sensors']:
csv_line += str(json_data['sensors'][nickname])
csv_line += csv_sep
csv_line += self.DATALOG_CSV_SEP
csv_line = csv_line[:-1]
try:
datalog_filename = self.CFG['DATALOG']['FILENAME']
datalog_filename = self.SESSION_DATALOG_FILENAME
with open(datalog_filename, 'a') as f:
f.write(csv_line + '\n')
logging.info("Updated datalog file: '{}'".format(datalog_filename))
......@@ -181,9 +182,7 @@ class Meteorologger:
def send_to_server(self, json_data):
r = None
outgoing_filename = os.path.join(
os.path.dirname(self.CFG['DATALOG']['FILENAME']),
'outgoing.json')
outgoing_filename = self.SESSION_OUTGOING_FILENAME
URL = self.CFG['SERVER']['API_POST_URL']
try:
if os.path.exists(outgoing_filename):
......@@ -224,7 +223,8 @@ class Meteorologger:
def serial_read(self, port_index=None):
'''
Sends the 'csv_nickname_list' string to the serial port of index
'port_index' (for self.SERIAL_PORTS) and returns the response line.
'port_index' (self.CFG['ARDUINO']['SERIAL_PORTS']) and returns
the response line.
csv_nickname_list example: str
Example: 'DHT22_TEMP,DHT22_AH,BMP085_PRESSURE,LDR'
......@@ -234,7 +234,7 @@ class Meteorologger:
['read'] + self.CFG['LOGGER']['SENSORS'])
try:
if isinstance(port_index, int):
serial_port = self.SERIAL_PORTS[port_index]
serial_port = self.CFG['ARDUINO']['SERIAL_PORTS'][port_index]
else:
serial_port = self.SERIAL_PORTS[0]
......@@ -255,7 +255,7 @@ class Meteorologger:
logging.info("sent: '{}' ({} bytes)".format(
read_command, result))
time.sleep(self.CFG['ARDUINO']['RESPONSE_DELAY'])
time.sleep(self.BOARD_RESPONSE_DELAY)
result_line = ser.readline()
......@@ -285,7 +285,7 @@ class Meteorologger:
def sync_rtc(self, port_index=None):
if isinstance(port_index, int):
serial_port = self.SERIAL_PORTS[port_index]
serial_port = self.CFG['ARDUINO']['SERIAL_PORT'][port_index]
else:
serial_port = self.SERIAL_PORTS[0]
......@@ -307,7 +307,7 @@ class Meteorologger:
print("sent: '{}' ({} bytes)".format(command_ser, result))
time.sleep(self.CFG['ARDUINO']['RESPONSE_DELAY'])
time.sleep(self.BOARD_RESPONSE_DELAY)
result_line = ser.readline()
......@@ -315,12 +315,43 @@ class Meteorologger:
ser.close()
def get_serial(self):
port_index = 0
while True:
serial_port = self.SERIAL_PORTS[port_index]
try:
print("\n" + "="*60)
print("Attempting serial connection at {}".format(serial_port))
ser = serial.Serial(port=serial_port,
baudrate=self.CFG['ARDUINO']['BAUD_RATE'],
timeout=self.SERIAL_READ_TIMEOUT,
xonxoff=True)
print("Done! Check object 'ser'")
return ser
except:
print("Unable do establish the connection.")
if port_index < len(self.SERIAL_PORTS) - 1:
port_index += 1
else:
port_index = 0
print("Trying another port in about {} seconds.."
.format(self.FIND_PORT_TIMEOUT))
time.sleep(self.FIND_PORT_TIMEOUT)
def setup_logging(self):
if self.verbose:
# logging.basicConfig(
# level=logging.DEBUG)
logging.basicConfig(
level=logging.INFO,
filename=self.SESSION_EXECUTION_LOG_FILENAME,
format='%(asctime)s : %(levelname)s : %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
if self.verbose:
root = logging.getLogger('')
root.setLevel(logging.INFO)
console = logging.StreamHandler()
......@@ -328,12 +359,39 @@ class Meteorologger:
fmt='%(asctime)s : %(levelname)s : %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'))
root.addHandler(console)
else:
logging.basicConfig(
level=logging.INFO,
filename=self.EXECUTION_LOG_FILENAME,
format='%(asctime)s : %(levelname)s : %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
def setup_files(self):
session_datetime = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
try:
os.mkdir(self.DATALOG_DIR)
except FileExistsError:
pass
except Exception as e:
print("Unable to create directory data/ to store datalog")
print(e)
return False
self.SESSION_DATALOG_FILENAME = self.DATALOG_DIR \
+ 'datalog-' + session_datetime + '.csv'
self.SESSION_OUTGOING_FILENAME = self.DATALOG_DIR \
+ 'outgoing/outgoing-' + session_datetime + '.json'
self.SESSION_EXECUTION_LOG_FILENAME = self.EXECUTION_LOG_PATH \
+ 'execution-' + session_datetime + '.log'
try:
with open(self.SESSION_DATALOG_FILENAME, 'w') as f:
f.write(self.DATALOG_CSV_SEP.join(
['dt' + self.CFG['DATALOG']['DATETIME_FORMAT']] +
[x for x in self.CFG['LOGGER']['SENSORS']
if x != 'RTC_DS1307'])
+ '\n')
except Exception as e:
print("Unable to write datalog file at data/")
print(e)
return False
return True
def run(self):
'''
......@@ -347,13 +405,15 @@ class Meteorologger:
3. write_datalog() # write current data on local file for backup
4. send_to_server() # try to send; if fails, save data for later
'''
if not self.setup_files():
return
self.setup_logging()
logging.info('EXECUTION START')
port_index = 0
try:
while True:
logging.info('='*40)
logging.debug('Attempting to read from serial')
logging.debug('Attempting to serial connection')
csv_result = self.serial_read(port_index)
......@@ -370,12 +430,12 @@ class Meteorologger:
logging.debug('Attempting to send data to the server')
self.send_to_server(json_data)
else:
if port_index < len(self.SERIAL_PORTS) - 1:
if port_index < len(self.CFG['ARDUINO']['SERIAL_PORTS'])-1:
port_index += 1
else:
port_index = 0
logging.debug("Trying another port in about {} seconds.."
logging.info("Trying another port in about {} seconds.."
.format(self.FIND_PORT_TIMEOUT))
time.sleep(self.FIND_PORT_TIMEOUT)
continue
......
logger/app/settings_schema.yaml
View file @ f0751a3a
......@@ -62,29 +62,25 @@ mapping:
required: true
type: map
mapping:
SERIAL_PORT:
SERIAL_PORTS:
required: true
type: str
type: seq
sequence:
- type: str
BAUD_RATE:
required: true
type: int
enum: [300, 1200, 2400, 4800, 9600, 14400, 19200, 28800, 38400, 57600, 115200]
RESPONSE_DELAY:
BOARD_MODEL:
required: true
type: int
range:
min: 0
type: str
DATALOG:
required: true
type: map
mapping:
FILENAME:
required: true
type: str
CSV_SEP:
required: true
type: str
......
logger/init_serial.py
View file @ f0751a3a
import serial
import time
ser = serial.Serial(port='/dev/ttyACM0',
baudrate=9600)
from app import Meteorologger
ser = Meteorologger().get_serial()
def send(command_str):
'''
Send the string 'command_str' to the serial port and return the response.
'''
ser.write(bytes(command_str, encoding='utf-8'))
time.sleep(1.5)
try:
......
meteorolog/mysensors.cpp
View file @ f0751a3a
......@@ -42,9 +42,13 @@ bool is_bmp085_connected=false;
String read_BMP085_PRESSURE()
{
if (is_bmp085_connected)
{
return String(bmp.readPressure());
}
else
{
return String("<bmp085_not_found>");
}
}
// ----------------------------------------------------------------
......@@ -57,9 +61,13 @@ RTC_DS1307 rtc;
String read_RTC_DS1307()
{
if (rtc.isrunning())
{
return get_datetime_str(rtc.now());
}
else
{
return String("<RTC_DS1307_not_found>");
}
}
String get_datetime_str(DateTime dt)
......@@ -84,7 +92,12 @@ String get_datetime_str(DateTime dt)
// Expects something like "2015,6,28,13,13,10"
String set_time_from_csv(String s)
{
DateTime dt;
if (!rtc.isrunning())
{
return String("<RTC_DS1307_not_found>");
}
DateTime dt;
int i_month = s.indexOf(CSV_SEP);
int i_day = s.indexOf(CSV_SEP, i_month + 1);
......
meteorolog/utils.cpp
View file @ f0751a3a
......@@ -54,14 +54,21 @@ String execute_command(String s)
String args = s.substring(s.indexOf(CSV_SEP) + 1);
if (command == "read" && args.length() > 0)
{
{
return parse_line_sensorlist(s.substring(s.indexOf(CSV_SEP) + 1));
}
else if (command == "setrtc" && args.length() > 0)
{
return set_time_from_csv(args);
}
}
else if (command == "listNicknames")
{
return get_nicknames_str_list();
}
else if (command == "listShortNicknames")
{
return get_short_nicknames_str_list();
}
else
{
return String("<invalid_command:'") + s + String("'>");
......@@ -97,3 +104,27 @@ String read_sensor_by_nickname(String nickname)
}
return String("<invalid_nickname:") + nickname + String(">");
}
String get_nicknames_str_list()
{
String s = "";
for (int i=0; i < __SENSOR_COUNT; i++)
{
s += sensor_nicknames[i];
if (i < __SENSOR_COUNT - 1) s += ',';
}
return s;
}
String get_short_nicknames_str_list()
{
String s = "";
for (int i=0; i < __SENSOR_COUNT; i++)
{
s += sensor_short_nicks[i];
if (i < __SENSOR_COUNT - 1) s += ',';
}
return s;
}
meteorolog/utils.h
View file @ f0751a3a
......@@ -45,4 +45,7 @@ String parse_line_sensorlist(String line);
typedef String (* ReadSensorFP)(void);
String read_sensor_by_nickname(String nickname);
String get_nicknames_str_list();
String get_short_nicknames_str_list();
#endif
settings.yaml
View file @ f0751a3a
......@@ -20,6 +20,7 @@ LOGGER:
INTERVAL: {days: 0, hours: 0, minutes: 0, seconds: 10}
# set false to use the system time or the RTC sensor name to use instead
# if the reading is invalid somehow, the system's time will be used instead
#
USE_RTC_DATETIME: RTC_DS1307
......@@ -29,25 +30,25 @@ LOGGER:
ARDUINO:
# comma-separated list of USB ports that may have the Arduino plugged in
# Arduino Uno usually mounts on /dev/ttyACMx; the others go on /dev/USBx
# list of USB ports (the first one will be used for ino upload)
# Uno and Mega usually appears at /dev/ttyACMx; else try /dev/ttyUSBx
#
SERIAL_PORT: /dev/ttyACM0, /dev/ttyACM1
SERIAL_PORTS:
- /dev/ttyACM0
- /dev/ttyACM1
# Arduino serial communication protocol (same as in meteorolog.ino)
# OBS: 115200 seems unstable with Uno + PySerial
#
BAUD_RATE: 9600
# time (in seconds) to wait before send a new serial read request
#
RESPONSE_DELAY: 3
DATALOG:
# directory to store datalog.csv data as backup
# board model used for Ino Tool compilation process
# see all supported with the command: $ ino list-models
# commom possibilities: atmega328 (for duemilanove), mega, mega2560
#
FILENAME: data/datalog.csv
BOARD_MODEL: uno
DATALOG:
# CSV delimiter to be used on the file DATA_LOG_FILENAME
# Sugestions: ',' or ';' or '\t' for tab (do not use the period '.')
#
......@@ -59,7 +60,8 @@ DATALOG:
# ============================================================================
# list of all the sensors available for this board, and their specs
# WARNING: should be synced with the server (edit only if you know)
# WARNING: should be synced with the server
# !! Edit only if you know what you are doing !!
#
SENSORS_AVAILABLE:
......
tools/config.plt 0 → 100644
View file @ f0751a3a
# TIP from: http://negativeprobability.blogspot.com.br/2013/01/real-time-plot-updates-using-gnuplot.html
set xdata time
set timefmt "%Y%m%d%H%M%S"
# get last updated file on data/
set macro
filepath = "../data/"."`cd ../data && ls -t -I outgoing* | head -n 1`"
# print filepath
# filter = 'sed -e "s/-\?\(nan\|inf\)/?/ig"'
set datafile missing "NaN"
plot filepath every ::1 using 1:col with lines