Commit 8b4cdf43 authored by Nelso Jost's avatar Nelso Jost

NEW: supervisor daemon deployment; better logging system

parent 7f14e322
.venv/
*.pyc
*.swp
datalog.csv
outgoing.json
*.csv
*.json
*.log
__pycache__
#-------------------------------------------------------------------------------
# Author: Nelso G. Jost (nelsojost@gmail.com)
# License: GPL
# Purpose: Get data from the board via serial and send it to the server.
#-------------------------------------------------------------------------------
from __future__ import print_function, division
from datetime import datetime
from pprint import pprint
import os
import requests
import serial
import time
import json
import yaml
def make_current_file_path(filename):
''' Append filename to the current __file__ path. '''
return os.path.join(os.path.abspath(os.path.dirname(__file__)), filename)
class Meteorologger:
'''
Provides a series of mechanisms to collect sensor data from the board
via serial port, save locally on the machine and upload to the server.
The functionality of this class relies heavily on the config file given
by the SETTINGS_FILENAME attribute, which uses YAML syntax.
Call the run() method to start the logging process.
'''
SETTINGS_FILENAME = make_current_file_path('settings.yaml')
CSV_SEP = ','
DATA_FORMATS = {'INTEGER': int, 'FLOAT': float, 'STRING': str}
SERIAL_READ_TIMEOUT = 1.5 # seconds
FIND_PORT_TIMEOUT = 2 # seconds
BOARD_RESET_TIMEOUT = 2 # seconds
def __init__(self):
self.loadSettings()
def _normalizePathFilename(self, key):
''' Appends __file__ basedir to config keys that don't have a basedir.
'''
if not os.path.dirname(self.CFG[key]):
self.CFG[key] = make_current_file_path(self.CFG[key])
def loadSettings(self):
'''
Load the configuration file onto the self.CFG attribute.
Some keys will be tested and filenames will be normalized.
'''
with open(self.SETTINGS_FILENAME) as f:
self.CFG = yaml.safe_load(f)
if not self.CFG['BASE_URL'].endswith('/'):
self.CFG['BASE_URL'] += '/'
self.URL = self.CFG['BASE_URL'] + 'api/post/rawsensordata/{}'\
.format(self.CFG['BOARD_ID'])
self.SENSORS_CSV_LINE = self.CSV_SEP.join([d['nickname'] for d in
self.CFG['SENSORS']])
# convert raw str into normal escaped str (e.g., r'\\t' --> '\t')
self.CFG['EXPORT_CSV_SEP'] = bytes(self.CFG['EXPORT_CSV_SEP'], 'utf8')\
.decode('unicode_escape')
self.SERIAL_PORTS = [p.strip() for p in self.CFG['SERIAL_PORT']
.split(',')]
self._normalizePathFilename('DATA_LOG_FILENAME')
self._normalizePathFilename('SERVER_OUTGOING_DATA_LOG_FILENAME')
def create_json_raw_sensor_data(self, raw_line):
'''
Given the raw serial line response (expected to be a CSV line), returns
a JSON dict with sensor data including the datetime field.
'''
raw_sensor_data = {'datetime': datetime.now().strftime("%Y%m%d%H%M%S"),
'sensors': {}}
for i, v in enumerate(raw_line.split(self.CSV_SEP)):
v = v.strip()
try:
v = self.DATA_FORMATS[self.CFG['SENSORS'][i]['data_format']](v)
except:
v = 'NaN'
raw_sensor_data['sensors'][self.CFG['SENSORS'][i]['nickname']] = v
print("\nJSON raw sensor data:")
pprint(raw_sensor_data)
return raw_sensor_data
def add_data_log(self, json_data):
'''
'''
csv_line = json_data['datetime'] + self.CFG['EXPORT_CSV_SEP']
for sensor in self.CFG['SENSORS']:
if sensor['nickname'] in json_data['sensors']:
csv_line += str(json_data['sensors']
[sensor['nickname']])
csv_line += self.CFG['EXPORT_CSV_SEP']
with open(self.CFG['DATA_LOG_FILENAME'], "a") as f:
f.write(csv_line[:-1] + '\n')
print("\nWrited data log onto\n {}".format(
self.CFG['DATA_LOG_FILENAME']))
def send_data_to_server(self, json_data):
print("\nSending data to the server now..", end='')
r = None
try:
if os.path.exists(self.CFG['SERVER_OUTGOING_DATA_LOG_FILENAME']):
print("\nTrying to send outgoing data first...")
with open(self.CFG['SERVER_OUTGOING_DATA_LOG_FILENAME']) as f:
for i, line in enumerate(f):
r = requests.post(self.URL, json=json.loads(line))
if r.status_code != 200:
raise Exception
print('Line {} :'.format(i), r)
os.remove(self.CFG['SERVER_OUTGOING_DATA_LOG_FILENAME'])
print("\ndone! Server data is up to date!")
r = requests.post(self.URL, json=json_data)
if r.status_code != 200:
raise Exception
except:
print("\nUnable to reach the server at\n {}".format(self.URL))
print("Request:", r)
with open(self.CFG['SERVER_OUTGOING_DATA_LOG_FILENAME'], 'a') as f:
f.write(json.dumps(json_data))
f.write('\n')
print("\nAdded to\n {}".format(
self.CFG['SERVER_OUTGOING_DATA_LOG_FILENAME']))
return
print(" done.\n ", r)
def _decode_bytes(self, raw_bytes):
result = None
try:
result = raw_bytes.decode('ascii').strip()
except:
print("Invalid bytes!")
return result
def serial_read_sensors(self, csv_nickname_list, serial_port=None):
result_line, ser = None, None
try:
if serial_port is None:
serial_port = self.SERIAL_PORTS[0]
elif isinstance(serial_port, int):
serial_port = self.SERIAL_PORTS[serial_port]
# if present, the board will be reseted
ser = serial.Serial(serial_port,
self.CFG['BAUD_RATE'],
timeout = self.SERIAL_READ_TIMEOUT,
xonxoff=True)
print(ser)
time.sleep(self.BOARD_RESET_TIMEOUT)
while bool(result_line) is False:
result = ser.write(bytes(csv_nickname_list, 'utf8'))
print("sent '{}':".format(csv_nickname_list), result)
result_line = ser.readline()
print("read: ", result_line)
result_line = self._decode_bytes(result_line)
if result_line is None:
continue
ser.close()
return result_line
except KeyboardInterrupt:
raise KeyboardInterrupt
except:
print("Unable to open serial port '{}'".format(serial_port))
return None
finally:
if ser:
ser.close()
def run(self):
'''
Starts the logger main loop, which iterate over the procedures:
1. Read sensor data via serial port;
2. If successful, save data on
'''
serial_port = 0
try:
while True:
print('\n-----------------------------------------------------')
csv_result = self.serial_read_sensors(self.SENSORS_CSV_LINE,
serial_port=serial_port)
print("csv_result: '{}'".format(csv_result))
if csv_result is not None:
json_raw_data = self.create_json_raw_sensor_data(csv_result)
self.add_data_log(json_raw_data)
self.send_data_to_server(json_raw_data)
else:
if serial_port < len(self.SERIAL_PORTS) - 1:
serial_port += 1
else:
serial_port = 0
print('Trying another port..')
time.sleep(self.FIND_PORT_TIMEOUT)
continue
print('\nGoing to sleep now for {} seconds..'.format(
self.CFG['READING_INTERVAL_SECONDS']))
time.sleep(self.CFG['READING_INTERVAL_SECONDS'])
except KeyboardInterrupt:
pass
if __name__ == '__main__':
Meteorologger().run()
......@@ -3,14 +3,18 @@ VENV := .venv
all:
@ echo "USAGE:"
@ echo " make setup -- Create a Python 3 virtual environment (only need once)"
@ echo " make log -- Launch the logger (using Python 3 from make setup)"
@ echo " make clean -- remove all the generated files"
@ echo " make deb-install -- Attempt to install required system wide"
@ echo " Debian packages via apt"
@ echo ""
@ echo " make setup -- Create a local Python virtual environment"
@ echo " that will hold requirements.pip modules"
@ echo ""
@ echo " make run -- Executes the logger "
setup: install-deb venv
install-deb:
sudo apt-get install python3 python3-pip
deb-install:
sudo apt-get install python3 python3-pip supervisor
sudo pip3 install virtualenv
venv: clean-venv
......@@ -30,12 +34,8 @@ venv: clean-venv
clean-venv:
rm -rf ${VENV}
log:
@ ${VENV}/bin/python logger.py
run:
${VENV}/bin/python3 logger.py -v
testserial:
@ ${VENV}/bin/ipython test_serial.py
clean-log:
rm -rf datalog.csv
rm -rf outgoing.json
deploy:
${VENV}/bin/python3 deploy.py
import jinja2
import os
import subprocess
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
SUPERVISOR_CONFIG_FILENAME = '/etc/supervisor/conf.d/meteorologger.conf'
def deploy_supervisor():
with open('supervisor.conf') as f_temp:
template = jinja2.Template(f_temp.read())
config_file_str = template.render(base_dir=BASE_DIR)
print('\nRegistering supervisor config at \n {}'
.format(SUPERVISOR_CONFIG_FILENAME))
print('='*60)
print(config_file_str)
print('='*60)
with open(SUPERVISOR_CONFIG_FILENAME, 'w') as f:
f.write(config_file_str + '\n')
print('\nRestarting supervisor..')
proc = subprocess.Popen('supervisorctl update', shell=True)
print("PID: ", end='')
proc = subprocess.Popen('supervisorctl pid meteorologger', shell=True,
stdout=subprocess.PIPE)
proc.wait()
print(proc.stdout.read().decode('ascii').strip(), end='')
print(' [meteorologger is running]')
print('\nYou can manage it with supervisorctl tool.')
if __name__ == '__main__':
deploy_supervisor()
This diff is collapsed.
ipython==3.1.0
Jinja2==2.7.3
MarkupSafe==0.23
pyreadline==2.0
pyserial==2.7
PyYAML==3.11
......
SERVER:
# full URL that accepts POST method for sending data to the server
# it should end with a valid board ID number (see documentation)
#
API_POST_URL: http://localhost/emm/api/post/rawsensordata/2
LOGGER:
# time between readings attempts (cycles of the logger execution)
#
READING_INTERVAL: {seconds: 10, minutes: 0, hours: 0, days: 0}
# format of the datetime column (see Python docs on datetime module)
#
DATETIME_FORMAT: '%Y%m%d%H%M%S'
# list of sensors to be read (the order reflect columns of DATALOG_CSV file)
# to ignore one, comment it out by putting # in the beginning of the line
#
SENSORS:
- {nickname: DHT22_TEMP, data_format: float}
- {nickname: DHT22_AH, data_format: float}
- {nickname: BMP085_PRESSURE, data_format: int}
- {nickname: LDR, data_format: float}
ARDUINO:
# comma-separated list of USB ports that may have the Arduino plugged in
# Arduino Uno usually mounts on /dev/ttyACMx; the others go on /dev/USBx
#
SERIAL_PORT: /dev/ttyACM0, /dev/ttyACM1
# Arduino serial communication protocol (same as in meteorolog.ino)
#
BAUD_RATE: 9600 # 115200 seems unstable with Uno + PySerial
FILES:
# file which will hold all data locally (regardless of the web server)
#
DATALOG_CSV: data/datalog.csv
# CSV delimiter to be used on the file DATA_LOG_FILENAME
# Sugestions: ',' or ';' or '\t' for tab (do not use the period '.')
#
DATALOG_CSV_SEP: '\t'
# temp file that hold data when server is off and will be sent when on
#
SERVER_OUTGOING_JSON: data/outgoing.json
[program:meteorologger]
command={{ base_dir }}/.venv/bin/python3 {{ base_dir }}/logger.py
directory={{ base_dir }}
user=root
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile={{ base_dir }}/logs/stdout.log
# server's full base address for sendind requests
BASE_URL: http://dados.cta.if.ufrgs.bt/emm
# valid board ID to which data will be sent
BOARD_ID: 2
# list of sensors that will be
SENSORS:
- {nickname: DHT22_TEMP, data_format: FLOAT}
- {nickname: DHT22_AH, data_format: FLOAT}
- {nickname: BMP085_PRESSURE, data_format: INTEGER}
- {nickname: LDR, data_format: FLOAT}
# comma-separated list of USB ports that may have the Arduino plugged in
SERIAL_PORT: /dev/ttyACM0, /dev/ttyACM1
# Arduino serial communication protocol (same as meteorolog/meteorolog.ino)
BAUD_RATE: 9600 # 115200 seems unstable with Uno + PySerial
# time to sleep between readings
READING_INTERVAL_SECONDS: 5
# CSV delimiter to be used on the file DATA_LOG_FILENAME
EXPORT_CSV_SEP: '\t'
# file which will hold all data locally (regardless of the web server)
DATA_LOG_FILENAME: datalog.csv
# temp file that hold data when server is off and will be sent when on
SERVER_OUTGOING_DATA_LOG_FILENAME: outgoing.json
import serial
import time
ser = serial.Serial('/dev/ttyACM0',
9600, # 115200 seems unstable for Arduino Uno
timeout=1.5) # timeout is important for unblock readline()
time.sleep(2) # board will be reseting
ser.write(b'LDR')
print(ser.readline())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment