Commit f0751a3a authored by Nelso Jost's avatar Nelso Jost
Browse files

NEW: better usage; single-command for setup, build, upload and deploy-logger

parent 2da2c4ba
PYVER := 3 PYVER := 3
VENV := .venv VENV := .venv
INODIR := .ino
all: help all: help
help: help:
@ echo "USAGE" @ echo "USAGE: make <command>"
@ echo "" @ echo ""
@ echo " make apt-install -- Uses Debian's apt to install required system tools" @ echo "COMMANDS:"
@ echo "" @ echo ""
@ echo " make setup -- Install Python requirements (need only once)" @ echo " full -- setup & bu & deploy-logger"
@ echo "" @ echo ""
@ echo " make build -- Compile the sketch" @ echo " apt-install -- Uses Debian's apt to install required system tools"
@ echo " make upload -- Send the compiled sketch to the board" @ echo " venv -- Creates a Python local virtual environment at ${VENV}"
@ echo " make serial -- Enter IPython session with serial opened" @ echo " setup -- apt-install & venv"
@ echo "" @ echo ""
@ echo " make bu -- build & upload" @ echo " build -- Compile the sketch usin Ino Tool"
@ echo " make bus -- build & upload & serial" @ echo " upload -- Send the compiled sketch to the board using Ino Tool"
@ echo " serial -- Enter IPython session with serial opened"
@ echo " bu -- build & upload"
@ echo " bus -- build & upload & serial"
@ echo "" @ echo ""
@ echo " make sync-rtc -- Synchronizes the board's clock with the system's" @ echo " sync-rtc -- Synchronizes the board's clock with the system's"
@ echo ""
@ echo " run-logger -- Starts the logger. Keep log at logger/logs"
@ echo " deploy-logger -- Activate logger daemon. Keep log at logger/logs/"
@ echo " undeploy-logger -- Deactivate logger daemon"
@ echo ""
@ echo " tail-log -- Exibits and follow last modified execution log file"
@ echo " tail-data -- Exibits and follow last modified datalog file"
@ echo ""
@ echo " plot-data col=<C> -- Uses Gnuplot to plot last modified datalog file"
@ echo " Replace <C> with the column number to plot as y axis"
@ echo " clean-venv -- Remove Python's virtual environment directory"
@ echo " clean-ino -- Remove Ino folder"
@ echo " clean-data -- !!! CAUTION !!! Remove all datalog files"
@ echo " clean-logs -- !!! CAUTION !!! Remove all execution log files"
@ echo " clean-all -- Performs all the above cleans"
@ echo "" @ echo ""
@ echo " make run-logger -- Starts the logger. Log: apper on the screen"
@ echo " make deploy-logger -- Activate logger daemon. Log at logger/logs/execution.log"
@ echo " make undeploy-logger -- Deactivate logger daemon."
@ echo " make tail-logger -- Exibits and follow daemon execution log."
apt-install: apt-install:
sudo apt-get install python python3 python3-pip supervisor sudo apt-get install python python3 python3-pip supervisor python-pip
sudo pip${PYVER} install virtualenv sudo pip${PYVER} install virtualenv
install-ino:
sudo apt-get install python-pip
sudo pip2 install ino sudo pip2 install ino
setup: apt-install venv
setup: clean-venv venv: clean-venv
@ echo "-------------------------------------------------------" @ echo "-------------------------------------------------------"
virtualenv -v --python='python${PYVER}' ${VENV} virtualenv -v --python='python${PYVER}' ${VENV}
@ echo "Virtualenv with 'python${PYVER}' interpreter was created at ${VENV}" @ echo "Virtualenv with 'python${PYVER}' interpreter was created at ${VENV}"
...@@ -49,21 +63,20 @@ setup: clean-venv ...@@ -49,21 +63,20 @@ setup: clean-venv
clean-venv: clean-venv:
rm -rf ${VENV} rm -rf ${VENV}
clean:
rm -rf .build src/*
build: build:
mkdir -p .ino $(eval MODEL := $(shell ${VENV}/bin/python -c "from logger.app import Meteorologger; print(Meteorologger().CFG['ARDUINO']['BOARD_MODEL'])"))
mkdir -p .ino/src/ .ino/lib/ mkdir -p ${INODIR}
cp -rf meteorolog/. .ino/src/ mkdir -p ${INODIR}/src/ ${INODIR}/lib/
cp logger/ino.ini .ino/ cp -rf meteorolog/. ${INODIR}/src/
cd .ino/ && ino build cd ${INODIR}/ && ino build -m $(MODEL)
upload: upload:
cd .ino/ && ino upload $(eval MODEL := $(shell ${VENV}/bin/python -c "from logger.app import Meteorologger; print(Meteorologger().CFG['ARDUINO']['BOARD_MODEL'])"))
$(eval PORT := $(shell ${VENV}/bin/python -c "from logger.app import Meteorologger; print(Meteorologger().CFG['ARDUINO']['SERIAL_PORTS'][0])"))
cd ${INODIR}/ && ino upload -m $(MODEL) -p $(PORT)
serial: serial:
${VENV}/bin/ipython3 -i logger/init_serial.py ${VENV}/bin/ipython -i logger/init_serial.py
bu: build upload bu: build upload
...@@ -77,14 +90,42 @@ run-logger: ...@@ -77,14 +90,42 @@ run-logger:
deploy-logger: undeploy-logger deploy-logger: undeploy-logger
mkdir -p logger/logs mkdir -p logger/logs
sudo touch data/datalog.csv data/outgoing.json
sudo chmod a+w data/*
sudo ${VENV}/bin/python${PYVER} logger/deploy.py sudo ${VENV}/bin/python${PYVER} logger/deploy.py
undeploy-logger: undeploy-logger:
sudo ${VENV}/bin/python${PYVER} logger/deploy.py -u sudo ${VENV}/bin/python${PYVER} logger/deploy.py -u
tail-logger: tail-logger:
tail -F logger/logs/execution.log $(eval TMP := $(shell ls -t -I stdout* logger/logs/ | head -n 1))
@ echo "Last log file updated: logger/logs/$(TMP)"
@ echo "File size: `du -h logger/logs/$(TMP) | cut -f1`"
@ echo ""
@ tail -F logger/logs/$(TMP)
plot-data:
@ echo "Quit by closing the window with Q and hitting Ctrl+C here to end the process"
@ cd tools && gnuplot -persist -e "config='config.plt'; col=${col}" loop.plt
tail-data:
$(eval TMP := $(shell ls -t -I outgoing* data/ | head -n 1))
@ echo "Last datalog file updated: data/$(TMP)"
@ echo "Number of lines/points: `cat data/$(TMP) | wc -l`"
@ echo "File size: `du -h data/$(TMP) | cut -f1`"
@ echo ""
@ head -1 data/$(TMP)
@ echo ""
@ tail -F data/$(TMP)
clean-data:
rm -rfv data/*.csv data/outgoing/*.json
clean-logs:
rm -rfv logger/logs/*
clean-ino:
rm -rf ${INODIR}
clean-all: clean-data clean-logs clean-ino clean-venv
cd logger && sudo py3clean app
full: apt-install setup install-ino bu deploy-logger full: setup bu deploy-logger
...@@ -50,7 +50,8 @@ class Meteorologger: ...@@ -50,7 +50,8 @@ class Meteorologger:
SETTINGS_SCHEMA_FILENAME = 'logger/app/settings_schema.yaml' SETTINGS_SCHEMA_FILENAME = 'logger/app/settings_schema.yaml'
SETTINGS_FILENAME = 'settings.yaml' SETTINGS_FILENAME = 'settings.yaml'
EXECUTION_LOG_FILENAME = 'logger/logs/execution.log' DATALOG_DIR = 'data/'
EXECUTION_LOG_PATH = 'logger/logs/'
OUTGOING_BASENAME = 'outgoing.json' OUTGOING_BASENAME = 'outgoing.json'
FILE_TIMESTAMP_FORMAT = '%Y-%m-%d-%H-%M-%S' FILE_TIMESTAMP_FORMAT = '%Y-%m-%d-%H-%M-%S'
...@@ -62,7 +63,7 @@ class Meteorologger: ...@@ -62,7 +63,7 @@ class Meteorologger:
SERIAL_READ_TIMEOUT = 1.5 # seconds SERIAL_READ_TIMEOUT = 1.5 # seconds
FIND_PORT_TIMEOUT = 10 # seconds FIND_PORT_TIMEOUT = 10 # seconds
BOARD_RESET_TIMEOUT = 2 # seconds BOARD_RESET_TIMEOUT = 2 # seconds
BOARD_RESPONSE_DELAY = 0 # seconds BOARD_RESPONSE_DELAY = 3 # seconds
verbose = True verbose = True
...@@ -99,14 +100,15 @@ class Meteorologger: ...@@ -99,14 +100,15 @@ class Meteorologger:
print("\nPlease fix it up or regenerate it.") print("\nPlease fix it up or regenerate it.")
sys.exit(1) sys.exit(1)
self.SERIAL_PORTS = self.CFG['ARDUINO']['SERIAL_PORT'].split(',')
self.READING_INTERVAL_SECONDS =\ self.READING_INTERVAL_SECONDS =\
self.CFG['LOGGER']['INTERVAL']['seconds']\ self.CFG['LOGGER']['INTERVAL']['seconds']\
+ 60 * self.CFG['LOGGER']['INTERVAL']['minutes']\ + 60 * self.CFG['LOGGER']['INTERVAL']['minutes']\
+ 3600 * self.CFG['LOGGER']['INTERVAL']['hours']\ + 3600 * self.CFG['LOGGER']['INTERVAL']['hours']\
+ 86400 * self.CFG['LOGGER']['INTERVAL']['days'] + 86400 * self.CFG['LOGGER']['INTERVAL']['days']
self.DATALOG_CSV_SEP = bytes(self.CFG['DATALOG']['CSV_SEP'],
'utf8').decode('unicode_escape')
def create_json(self, raw_line): def create_json(self, raw_line):
''' '''
Given the raw serial line response (CSV string), builds and returns Given the raw serial line response (CSV string), builds and returns
...@@ -128,7 +130,7 @@ class Meteorologger: ...@@ -128,7 +130,7 @@ class Meteorologger:
type_name = self.CFG['SENSORS_AVAILABLE'][nickname]['data_format'] type_name = self.CFG['SENSORS_AVAILABLE'][nickname]['data_format']
if type_name == 'datetime': if type_name == 'datetime':
if using_rtc and not v.strip().startswith('<'): if using_rtc and not 'not_found' in v.strip():
d['datetime']['value'] = datetime.strptime( d['datetime']['value'] = datetime.strptime(
v, rtc_datetime_fmt).strftime(d['datetime']['format']) v, rtc_datetime_fmt).strftime(d['datetime']['format'])
continue continue
...@@ -136,8 +138,9 @@ class Meteorologger: ...@@ -136,8 +138,9 @@ class Meteorologger:
try: try:
v = self.DATA_FORMATS[type_name](v.strip()) v = self.DATA_FORMATS[type_name](v.strip())
except: except:
logging.error("[{}]: '{}' is not a valid {}" logging.warning("[{}]: '{}' is not a valid {}"
.format(nickname, v, type_name)) .format(nickname, v, type_name))
d['sensors'][nickname] = 'NaN'
continue continue
d['sensors'][nickname] = v d['sensors'][nickname] = v
...@@ -157,19 +160,17 @@ class Meteorologger: ...@@ -157,19 +160,17 @@ class Meteorologger:
DATALOG_CSV as specficied on self.SETTINGS_FILENAME. DATALOG_CSV as specficied on self.SETTINGS_FILENAME.
''' '''
# convert raw str into normal escaped str (e.g., r'\\t' --> '\t') # convert raw str into normal escaped str (e.g., r'\\t' --> '\t')
csv_sep = bytes(self.CFG['DATALOG']['CSV_SEP'],
'utf8').decode('unicode_escape')
csv_line = json_data['datetime']['value'] + csv_sep csv_line = json_data['datetime']['value'] + self.DATALOG_CSV_SEP
for nickname in self.CFG['LOGGER']['SENSORS']: for nickname in self.CFG['LOGGER']['SENSORS']:
if nickname in json_data['sensors']: if nickname in json_data['sensors']:
csv_line += str(json_data['sensors'][nickname]) csv_line += str(json_data['sensors'][nickname])
csv_line += csv_sep csv_line += self.DATALOG_CSV_SEP
csv_line = csv_line[:-1] csv_line = csv_line[:-1]
try: try:
datalog_filename = self.CFG['DATALOG']['FILENAME'] datalog_filename = self.SESSION_DATALOG_FILENAME
with open(datalog_filename, 'a') as f: with open(datalog_filename, 'a') as f:
f.write(csv_line + '\n') f.write(csv_line + '\n')
logging.info("Updated datalog file: '{}'".format(datalog_filename)) logging.info("Updated datalog file: '{}'".format(datalog_filename))
...@@ -181,9 +182,7 @@ class Meteorologger: ...@@ -181,9 +182,7 @@ class Meteorologger:
def send_to_server(self, json_data): def send_to_server(self, json_data):
r = None r = None
outgoing_filename = os.path.join( outgoing_filename = self.SESSION_OUTGOING_FILENAME
os.path.dirname(self.CFG['DATALOG']['FILENAME']),
'outgoing.json')
URL = self.CFG['SERVER']['API_POST_URL'] URL = self.CFG['SERVER']['API_POST_URL']
try: try:
if os.path.exists(outgoing_filename): if os.path.exists(outgoing_filename):
...@@ -224,7 +223,8 @@ class Meteorologger: ...@@ -224,7 +223,8 @@ class Meteorologger:
def serial_read(self, port_index=None): def serial_read(self, port_index=None):
''' '''
Sends the 'csv_nickname_list' string to the serial port of index Sends the 'csv_nickname_list' string to the serial port of index
'port_index' (for self.SERIAL_PORTS) and returns the response line. 'port_index' (self.CFG['ARDUINO']['SERIAL_PORTS']) and returns
the response line.
csv_nickname_list example: str csv_nickname_list example: str
Example: 'DHT22_TEMP,DHT22_AH,BMP085_PRESSURE,LDR' Example: 'DHT22_TEMP,DHT22_AH,BMP085_PRESSURE,LDR'
...@@ -234,7 +234,7 @@ class Meteorologger: ...@@ -234,7 +234,7 @@ class Meteorologger:
['read'] + self.CFG['LOGGER']['SENSORS']) ['read'] + self.CFG['LOGGER']['SENSORS'])
try: try:
if isinstance(port_index, int): if isinstance(port_index, int):
serial_port = self.SERIAL_PORTS[port_index] serial_port = self.CFG['ARDUINO']['SERIAL_PORTS'][port_index]
else: else:
serial_port = self.SERIAL_PORTS[0] serial_port = self.SERIAL_PORTS[0]
...@@ -255,7 +255,7 @@ class Meteorologger: ...@@ -255,7 +255,7 @@ class Meteorologger:
logging.info("sent: '{}' ({} bytes)".format( logging.info("sent: '{}' ({} bytes)".format(
read_command, result)) read_command, result))
time.sleep(self.CFG['ARDUINO']['RESPONSE_DELAY']) time.sleep(self.BOARD_RESPONSE_DELAY)
result_line = ser.readline() result_line = ser.readline()
...@@ -285,7 +285,7 @@ class Meteorologger: ...@@ -285,7 +285,7 @@ class Meteorologger:
def sync_rtc(self, port_index=None): def sync_rtc(self, port_index=None):
if isinstance(port_index, int): if isinstance(port_index, int):
serial_port = self.SERIAL_PORTS[port_index] serial_port = self.CFG['ARDUINO']['SERIAL_PORT'][port_index]
else: else:
serial_port = self.SERIAL_PORTS[0] serial_port = self.SERIAL_PORTS[0]
...@@ -307,7 +307,7 @@ class Meteorologger: ...@@ -307,7 +307,7 @@ class Meteorologger:
print("sent: '{}' ({} bytes)".format(command_ser, result)) print("sent: '{}' ({} bytes)".format(command_ser, result))
time.sleep(self.CFG['ARDUINO']['RESPONSE_DELAY']) time.sleep(self.BOARD_RESPONSE_DELAY)
result_line = ser.readline() result_line = ser.readline()
...@@ -315,12 +315,43 @@ class Meteorologger: ...@@ -315,12 +315,43 @@ class Meteorologger:
ser.close() ser.close()
def get_serial(self):
port_index = 0
while True:
serial_port = self.SERIAL_PORTS[port_index]
try:
print("\n" + "="*60)
print("Attempting serial connection at {}".format(serial_port))
ser = serial.Serial(port=serial_port,
baudrate=self.CFG['ARDUINO']['BAUD_RATE'],
timeout=self.SERIAL_READ_TIMEOUT,
xonxoff=True)
print("Done! Check object 'ser'")
return ser
except:
print("Unable do establish the connection.")
if port_index < len(self.SERIAL_PORTS) - 1:
port_index += 1
else:
port_index = 0
print("Trying another port in about {} seconds.."
.format(self.FIND_PORT_TIMEOUT))
time.sleep(self.FIND_PORT_TIMEOUT)
def setup_logging(self): def setup_logging(self):
if self.verbose:
# logging.basicConfig(
# level=logging.DEBUG)
logging.basicConfig(
level=logging.INFO,
filename=self.SESSION_EXECUTION_LOG_FILENAME,
format='%(asctime)s : %(levelname)s : %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
if self.verbose:
root = logging.getLogger('') root = logging.getLogger('')
root.setLevel(logging.INFO) root.setLevel(logging.INFO)
console = logging.StreamHandler() console = logging.StreamHandler()
...@@ -328,12 +359,39 @@ class Meteorologger: ...@@ -328,12 +359,39 @@ class Meteorologger:
fmt='%(asctime)s : %(levelname)s : %(message)s', fmt='%(asctime)s : %(levelname)s : %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')) datefmt='%Y-%m-%d %H:%M:%S'))
root.addHandler(console) root.addHandler(console)
else:
logging.basicConfig( def setup_files(self):
level=logging.INFO, session_datetime = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
filename=self.EXECUTION_LOG_FILENAME,
format='%(asctime)s : %(levelname)s : %(message)s', try:
datefmt='%Y-%m-%d %H:%M:%S') os.mkdir(self.DATALOG_DIR)
except FileExistsError:
pass
except Exception as e:
print("Unable to create directory data/ to store datalog")
print(e)
return False
self.SESSION_DATALOG_FILENAME = self.DATALOG_DIR \
+ 'datalog-' + session_datetime + '.csv'
self.SESSION_OUTGOING_FILENAME = self.DATALOG_DIR \
+ 'outgoing/outgoing-' + session_datetime + '.json'
self.SESSION_EXECUTION_LOG_FILENAME = self.EXECUTION_LOG_PATH \
+ 'execution-' + session_datetime + '.log'
try:
with open(self.SESSION_DATALOG_FILENAME, 'w') as f:
f.write(self.DATALOG_CSV_SEP.join(
['dt' + self.CFG['DATALOG']['DATETIME_FORMAT']] +
[x for x in self.CFG['LOGGER']['SENSORS']
if x != 'RTC_DS1307'])
+ '\n')
except Exception as e:
print("Unable to write datalog file at data/")
print(e)
return False
return True
def run(self): def run(self):
''' '''
...@@ -347,13 +405,15 @@ class Meteorologger: ...@@ -347,13 +405,15 @@ class Meteorologger:
3. write_datalog() # write current data on local file for backup 3. write_datalog() # write current data on local file for backup
4. send_to_server() # try to send; if fails, save data for later 4. send_to_server() # try to send; if fails, save data for later
''' '''
if not self.setup_files():
return
self.setup_logging() self.setup_logging()
logging.info('EXECUTION START') logging.info('EXECUTION START')
port_index = 0 port_index = 0
try: try:
while True: while True:
logging.info('='*40) logging.info('='*40)
logging.debug('Attempting to read from serial') logging.debug('Attempting to serial connection')
csv_result = self.serial_read(port_index) csv_result = self.serial_read(port_index)
...@@ -370,12 +430,12 @@ class Meteorologger: ...@@ -370,12 +430,12 @@ class Meteorologger:
logging.debug('Attempting to send data to the server') logging.debug('Attempting to send data to the server')
self.send_to_server(json_data) self.send_to_server(json_data)
else: else:
if port_index < len(self.SERIAL_PORTS) - 1: if port_index < len(self.CFG['ARDUINO']['SERIAL_PORTS'])-1:
port_index += 1 port_index += 1
else: else:
port_index = 0 port_index = 0
logging.debug("Trying another port in about {} seconds.." logging.info("Trying another port in about {} seconds.."
.format(self.FIND_PORT_TIMEOUT)) .format(self.FIND_PORT_TIMEOUT))
time.sleep(self.FIND_PORT_TIMEOUT) time.sleep(self.FIND_PORT_TIMEOUT)
continue continue
......
...@@ -62,29 +62,25 @@ mapping: ...@@ -62,29 +62,25 @@ mapping:
required: true required: true
type: map type: map
mapping: mapping:
SERIAL_PORT: SERIAL_PORTS:
required: true required: true
type: str type: seq
sequence:
- type: str
BAUD_RATE: BAUD_RATE:
required: true required: true
type: int type: int
enum: [300, 1200, 2400, 4800, 9600, 14400, 19200, 28800, 38400, 57600, 115200] enum: [300, 1200, 2400, 4800, 9600, 14400, 19200, 28800, 38400, 57600, 115200]
RESPONSE_DELAY: BOARD_MODEL:
required: true required: true
type: int type: str
range:
min: 0
DATALOG: DATALOG:
required: true required: true
type: map type: map
mapping: mapping:
FILENAME:
required: true
type: str
CSV_SEP: CSV_SEP:
required: true required: true
type: str type: str
......
import serial import serial
import time import time
ser = serial.Serial(port='/dev/ttyACM0', from app import Meteorologger
baudrate=9600)
ser = Meteorologger().get_serial()
def send(command_str): def send(command_str):
'''
Send the string 'command_str' to the serial port and return the response.
'''
ser.write(bytes(command_str, encoding='utf-8')) ser.write(bytes(command_str, encoding='utf-8'))
time.sleep(1.5) time.sleep(1.5)
try: try:
......
...@@ -42,9 +42,13 @@ bool is_bmp085_connected=false; ...@@ -42,9 +42,13 @@ bool is_bmp085_connected=false;
String read_BMP085_PRESSURE() String read_BMP085_PRESSURE()
{ {
if (is_bmp085_connected) if (is_bmp085_connected)
{
return String(bmp.readPressure()); return String(bmp.readPressure());
}
else else
{
return String("<bmp085_not_found>"); return String("<bmp085_not_found>");
}
} }
// ---------------------------------------------------------------- // ----------------------------------------------------------------
...@@ -57,9 +61,13 @@ RTC_DS1307 rtc; ...@@ -57,9 +61,13 @@ RTC_DS1307 rtc;
String read_RTC_DS1307() String read_RTC_DS1307()