Commit 1289f84a authored by Nelso Jost's avatar Nelso Jost

NEW: compatibility with new web app version

parent ea921422
EMM Data Logger
Copyright (C) 2015 Nelso G. Jost
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
...@@ -23,10 +23,12 @@ help: ...@@ -23,10 +23,12 @@ help:
@ echo "" @ echo ""
@ echo " tail-log Follow updates on the last execution log" @ echo " tail-log Follow updates on the last execution log"
@ echo " tail-data Follow updates on the last data log" @ echo " tail-data Follow updates on the last data log"
@ echo " plot-data col=x Uses Gnuplot to plot last data log col number x" @ echo " plot col=x Uses Gnuplot to plot last data log col number x"
dep: deps:
sudo apt-get install python3 python3-pip python-pip supervisor python-virtualenv sudo apt-get install python3 supervisor
sudo -H python3 scripts/get-pip.py
sudo -H pip3 install virtualenv
apt-install: apt-install:
chmod +x scripts/apt-install.sh chmod +x scripts/apt-install.sh
...@@ -40,12 +42,11 @@ ino-install: ...@@ -40,12 +42,11 @@ ino-install:
chmod +x scripts/ino-install.sh chmod +x scripts/ino-install.sh
./scripts/ino-install.sh ./scripts/ino-install.sh
setup: clean-venv create-venv setup: setup clean-venv venv
create-venv: venv:
@ echo "-------------------------------------------------------" @ echo "-------------------------------------------------------"
virtualenv -v --python='${PYBIN}' ${VENVDIR} --no-site-packages virtualenv -v --python='${PYBIN}' ${VENVDIR} --no-site-packages
${VENVDIR}/bin/pip install --upgrade pip
${VENVDIR}/bin/pip install -r logger/requirements.pip ${VENVDIR}/bin/pip install -r logger/requirements.pip
@ echo "-------------------------------------------------------" @ echo "-------------------------------------------------------"
@ echo "Required Python virtual environment installed at " @ echo "Required Python virtual environment installed at "
...@@ -67,16 +68,22 @@ firmware: ${BUILDWITH}-install ...@@ -67,16 +68,22 @@ firmware: ${BUILDWITH}-install
chmod +x scripts/ino-build.sh chmod +x scripts/ino-build.sh
./scripts/ino-build.sh ${BUILDWITH} ${INODIR} ${ARDUINOPATH} ./scripts/ino-build.sh ${BUILDWITH} ${INODIR} ${ARDUINOPATH}
serial: serial: check-venv
${VENVPY} -i scripts/init_serial.py ${VENVPY} scripts/init_serial.py
syncrtc:
${VENVPY} scripts/init_serial.py --syncrtc
run: check-venv run: check-venv
${VENVPY} logger/run.py ${VENVPY} logger/run.py
deploy: fakerun:
${VENVPY} logger/run.py --fakedata
deploy: check-venv
sudo ${VENVPY} logger/deploy.py sudo ${VENVPY} logger/deploy.py
undeploy: undeploy: check-venv
sudo ${VENVPY} logger/deploy.py -u sudo ${VENVPY} logger/deploy.py -u
tail-log: tail-log:
...@@ -96,7 +103,7 @@ tail-data: ...@@ -96,7 +103,7 @@ tail-data:
@ echo "" @ echo ""
@ tail -F data/$(TMP) @ tail -F data/$(TMP)
plot-data: plot:
@ echo "Quit by closing the window with Q and hitting Ctrl+C here to end the process" @ echo "Quit by closing the window with Q and hitting Ctrl+C here to end the process"
@ cd tools && gnuplot -persist -e "config='config.plt'; col=${col}" loop.plt @ cd tools && gnuplot -persist -e "config='config.plt'; col=${col}" loop.plt
...@@ -109,5 +116,7 @@ clean-logs: ...@@ -109,5 +116,7 @@ clean-logs:
clean-ino: clean-ino:
rm -rf ${INODIR} rm -rf ${INODIR}
clean-all: clean-data clean-logs clean-ino clean-venv clean: clean-data clean-logs clean-ino clean-venv
cd logger && sudo py3clean app find . -type f -name "*.py[co]" -delete
find . -type d -name "__pycache__" -delete
rm -rf docs/_build
***************
EMM Data Logger
***************
This applications retrieves meteorological data from Arduino boards of the project EMM, save them locally and also sent them to a remote server.
More details on the wiki:
https://git.cta.if.ufrgs.br/meteorolog/logger/wikis/home
The root directory of this repo contains a Makefile with several commands for easy installation and usage of the software. Make sure to be on this directory when executing commands `make <target>`.
Usage
=====
The following topics describe the order of the tasks needed to install and use the software along with the Arduino board.
* 1 -- Install system dependencies and prepare local Python execution environment:
$ make setup
PS: Uses apt for Debian packages and Python's pip3 will be installed if not present.
* 2 -- Install firmware on the Arduino board via Arduino IDE or the terminal (through Ino Tool) with:
$ make firmware
PS: Stil requires Arduino IDE to be installed.
* 3 -- Test the firmware by sending commands (strings) to the serial port via Arduino IDE Serial Monitor or a Python session on the terminal with:
$ make serial
Refer to the documentation for all available commands:
https://git.cta.if.ufrgs.br/meteorolog/logger/wikis/BoardCommands
* 4 -- Prepare the file `settings.ini` with desired behavior and required fields, such as `BOARD_HASH`.
* 5 -- Test the logger execution on the foreground with:
$ make run
Data will be saved on the `data/` subdirectory.
* 6 -- Install the logger for background execution and automatic initiation:
$ make deploy
* 7 -- You can monitor the last generated data or log with:|
$ make tail-log
$ make tail-data
...@@ -10,12 +10,8 @@ DEFAULT_INI=\ ...@@ -10,12 +10,8 @@ DEFAULT_INI=\
; base URL of the server ; base URL of the server
URL = http://dados.cta.if.ufrgs.br/emm URL = http://dados.cta.if.ufrgs.br/emm
; board identification number (auto generated when a new board is created) ; user board authentication token (auto generated when a new board is created)
BOARD_ID = BOARD_HASH =
; authentication token for the board's user
USER_HASH =
[reading] [reading]
; CSV list of sensor names/nicknames (order reflect columns on datalog files) ; CSV list of sensor names/nicknames (order reflect columns on datalog files)
...@@ -27,7 +23,6 @@ SLEEP_MINUTES = 5 ...@@ -27,7 +23,6 @@ SLEEP_MINUTES = 5
; true for read board clock; if fail, or false, system's time will be used ; true for read board clock; if fail, or false, system's time will be used
RTC_DS1307 = true RTC_DS1307 = true
[datalog] [datalog]
; CSV delimiter for local log files (valid options: , or ; or \t ) ; CSV delimiter for local log files (valid options: , or ; or \t )
CSV_SEP = \t CSV_SEP = \t
...@@ -36,7 +31,6 @@ CSV_SEP = \t ...@@ -36,7 +31,6 @@ CSV_SEP = \t
; %Y : years | %m : months | %d : days | %H : hours | %M : mins | %S : secs ; %Y : years | %m : months | %d : days | %H : hours | %M : mins | %S : secs
DATETIME_FORMAT = %Y-%m-%d-%H-%M-%S DATETIME_FORMAT = %Y-%m-%d-%H-%M-%S
[arduino] [arduino]
; CSV list of ports to attempt connection or blank for automatic search on ; CSV list of ports to attempt connection or blank for automatic search on
; /dev/ttyACM* and /dev/ttyUSB* (where * varies from 0 to 4) ; /dev/ttyACM* and /dev/ttyUSB* (where * varies from 0 to 4)
...@@ -57,7 +51,7 @@ class Config: ...@@ -57,7 +51,7 @@ class Config:
EXECUTION_LOG_PATH = make_path_here('../logs/') EXECUTION_LOG_PATH = make_path_here('../logs/')
OUTGOING_FILENAME = make_path_here('../../data/outgoing.json') OUTGOING_FILENAME = make_path_here('../../data/outgoing.json')
URL_API_POST_RAWSENSORDATA = '{base}api/post/rawsensordata/{bid}' URL_API_POST_RAWSENSORDATA = '{base}api/post/rawsensordata'
SERIAL_CSV_SEP = ',' SERIAL_CSV_SEP = ','
RTC_NAME = 'RTC_DS1307' RTC_NAME = 'RTC_DS1307'
...@@ -162,16 +156,9 @@ class Config: ...@@ -162,16 +156,9 @@ class Config:
self.validate_arduino() self.validate_arduino()
def validate_server(self): def validate_server(self):
try:
bid = int(self['server']['board_id'])
except:
raise self.ConfigValueError('server', 'BOARD_ID',
'TypeError: Integer number expected!', self)
self['server']['api_post_url'] = self.URL_API_POST_RAWSENSORDATA\ self['server']['api_post_url'] = self.URL_API_POST_RAWSENSORDATA\
.format(base=self['server']['url'] + .format(base=self['server']['url'] +
('' if self['server']['url'].endswith('/') else '/'), ('' if self['server']['url'].endswith('/') else '/'))
bid=bid)
def validate_reading(self): def validate_reading(self):
self['reading']['sensors'] = [x.strip() for x in self['reading']['sensors'] = [x.strip() for x in
......
...@@ -18,6 +18,7 @@ import serial ...@@ -18,6 +18,7 @@ import serial
import sys import sys
import time import time
import json import json
import random
class Meteorologger: class Meteorologger:
...@@ -37,10 +38,8 @@ class Meteorologger: ...@@ -37,10 +38,8 @@ class Meteorologger:
BOARD_RESET_TIMEOUT = 2 # seconds BOARD_RESET_TIMEOUT = 2 # seconds
BOARD_RESPONSE_DELAY = 3 # seconds BOARD_RESPONSE_DELAY = 3 # seconds
config = None def __init__(self, background=False, fakedata=False):
self.background, self.fakedata = background, fakedata
def __init__(self, background=False):
self.background = background
self.config = Config() self.config = Config()
def _decode_bytes(self, raw_bytes, encoding='ascii'): def _decode_bytes(self, raw_bytes, encoding='ascii'):
...@@ -53,7 +52,7 @@ class Meteorologger: ...@@ -53,7 +52,7 @@ class Meteorologger:
def setup_logging(self): def setup_logging(self):
''' '''
Prepares the execution log file mechanism, which uses the standard Prepares the execution log file mechanism, which uses the standard
library logging. This way de app is prepared for background execution. library logging. Thus, the app is prepared for background execution.
''' '''
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
...@@ -128,6 +127,9 @@ class Meteorologger: ...@@ -128,6 +127,9 @@ class Meteorologger:
Sends the serial command for reading sensors to the board and read Sends the serial command for reading sensors to the board and read
its response, returning the valid ASCII resulting string. its response, returning the valid ASCII resulting string.
''' '''
if self.fakedata:
return ','.join(['{}'.format(random.randint(0, 100))
for x in self.config['reading']['sensors']])
ser, result_line = self.get_serial(), None ser, result_line = self.get_serial(), None
try: try:
ser.flush() ser.flush()
...@@ -154,7 +156,7 @@ class Meteorologger: ...@@ -154,7 +156,7 @@ class Meteorologger:
if ser: if ser:
ser.close() ser.close()
def create_json(self, readline): def create_json(self, rawline):
''' '''
Given the raw serial line response (CSV string), builds and returns Given the raw serial line response (CSV string), builds and returns
a JSON dict with validated, server-ready, sensor data. a JSON dict with validated, server-ready, sensor data.
...@@ -163,7 +165,7 @@ class Meteorologger: ...@@ -163,7 +165,7 @@ class Meteorologger:
'sensors': {}} 'sensors': {}}
for name, value in zip(self.config['reading']['sensors'], for name, value in zip(self.config['reading']['sensors'],
readline.split(self.config.SERIAL_CSV_SEP)): rawline.split(self.config.SERIAL_CSV_SEP)):
if name == self.config.RTC_NAME: if name == self.config.RTC_NAME:
try: try:
d['datetime'].update( d['datetime'].update(
...@@ -235,7 +237,7 @@ class Meteorologger: ...@@ -235,7 +237,7 @@ class Meteorologger:
r = requests.post(self.config['server']['api_post_url'], r = requests.post(self.config['server']['api_post_url'],
json={'data': data, json={'data': data,
'user_hash': self.config['server']['user_hash']}) 'board_hash': self.config['server']['board_hash']})
if r.status_code == 200: if r.status_code == 200:
if 'success' in r.json(): if 'success' in r.json():
......
...@@ -2,6 +2,7 @@ from app.main import Meteorologger ...@@ -2,6 +2,7 @@ from app.main import Meteorologger
import sys import sys
try: try:
Meteorologger(background='--background' in sys.argv).run() Meteorologger(background='--background' in sys.argv,
fakedata='--fakedata' in sys.argv).run()
except Exception as e: except Exception as e:
print(e.message) print(e.message)
File mode changed from 100755 to 100644
#sudo apt-get purge python3-pip python-pip #sudo apt-get purge python3-pip python-pip
sudo apt-get update #sudo apt-get update
sudo apt-get install python python3 python3-pip supervisor python-pip #sudo apt-get install python python3 python3-pip supervisor python-pip
echo "Finding pip3 binary.." echo "Finding pip3 binary.."
PIP3=`dpkg -L python3-pip | grep /usr/bin/pip | tail -1` PIP3=`dpkg -L python3-pip | grep /usr/bin/pip | tail -1`
echo "PIP3=$PIP3" echo "PIP3=$PIP3"
......
This diff is collapsed.
import serial import serial
import sys
import time import time
from datetime import datetime from datetime import datetime
def get_serial(baudrate=9600, read_timeout=1.5, board_reset_timeout=3, def get_serial(baudrate=9600, read_timeout=1.5, board_reset_timeout=2,
find_port_timeout=0.5): find_port_timeout=0.5):
ports = [] ports = []
for i in range(5): for i in range(5):
...@@ -27,49 +28,69 @@ def get_serial(baudrate=9600, read_timeout=1.5, board_reset_timeout=3, ...@@ -27,49 +28,69 @@ def get_serial(baudrate=9600, read_timeout=1.5, board_reset_timeout=3,
i = 0 i = 0
time.sleep(find_port_timeout) time.sleep(find_port_timeout)
print("\nAttempting serial connection ...\n")
ser = get_serial()
print(ser)
ser.flush()
def send(command_str, response_wait=0): def send(command_str, response_wait=0):
''' '''
Send the string 'command_str' to the serial port and return the response. Send the string 'command_str' to the serial port and return the response.
''' '''
ser.write(bytes(command_str, encoding='utf-8')) ser.write(bytes(command_str, encoding='utf-8'))
time.sleep(response_wait) # time.sleep(response_wait)
try: try:
raw = ser.readall() raw = ser.readall()
return raw.decode('ascii').strip() return raw.decode('ascii').strip()
except: except:
print("Unable to decode raw response to ASCII:\n{}".format(raw)) print("Unable to decode raw response to ASCII:\n{}".format(raw))
def sync_rtc(): def sync_rtc():
''' '''
Synchronizes the board clock with the system's. Synchronizes the board clock with the system's.
''' '''
dt = datetime.now() dt = datetime.now()
ser.write(bytes('setRTC,{Y},{m},{d},{H},{M},{S}'.format( print('\nCurrent system time:', dt.isoformat().replace('T', ' '))
Y=dt.year, m=dt.month, d=dt.day, H=dt.hour, M=dt.minute, S=dt.second), print("\nAttempting serial connection ...", end='')
encoding='utf-8')) ser = get_serial()
print('got it!')
print(ser)
ser.flush()
command = 'setrtc,{Y},{m},{d},{H},{M},{S}'.format(
Y=dt.year, m=dt.month, d=dt.day, H=dt.hour, M=dt.minute, S=dt.second)
print('\nSending serial command:\n "{}"\n'.format(command))
ser.write(bytes(command, encoding='utf-8'))
try: try:
raw = ser.readall() raw = ser.readall()
return raw.decode('ascii').strip() return raw.decode('ascii').strip()
except: except:
print("Unable to decode raw response to ASCII:\n{}".format(raw)) print("Unable to decode raw response to ASCII:\n{}".format(raw))
print("\nSent 'help'... waiting for board response ...")
response = send('help')
print("\nBoard commands:\n" + '-'*40 + '\n' + response + '\n' + '-'*40)
print(
"""\
You are now inside the Python interpreter! The commands above
must be passed as a string to the send() function!
Examples: if '-c' in sys.argv and len(sys.argv) == 3:
>>> send('read,t,l') # for reading temp. and lum. print(send(input('> ')))
>>> send('setrtc,2015,7,15,17,45,0') # for setting the datetime on RTC elif '--syncrtc' in sys.argv:
print(sync_rtc())
else:
print("\nAttempting serial connection ...\n")
ser = get_serial()
print(ser)
ser.flush()
Hit Ctrl+D to exit!""") print("\nSent 'help'... waiting for board response ...")
print('-'*40 + '\n') response = send('help')
print("\nBoard commands:\n" + '-'*40 + '\n' + response + '\n' + '-'*40)
print(
"""\
Examples:
> read,t,l # for reading temp. and lum.
> setrtc,2015,7,15,17,45,0 # for setting RTC datetime
Hit Ctrl+C to exit!""")
print('-'*40 + '\n')
try:
while True:
print(send(input('> ')))
except:
pass
[server] [server]
; base URL of the server ; full URL of the data server
URL = http://dados.cta.if.ufrgs.br/emm URL = http://dados.cta.if.ufrgs.br/emm
; board identification number (auto generated when a new board is created) ; user board authentication token (auto generated when a new board is created)
BOARD_ID = BOARD_HASH =
; authentication token for the board's user
USER_HASH =
[reading] [reading]
; CSV list of sensor names/nicknames (order reflect columns on datalog files) ; CSV list of sensor names/nicknames (order reflect columns on datalog files)
SENSORS = DHT22_TEMP, DHT22_AH, BMP085_PRESSURE, LDR SENSORS = DHT22_TEMP, DHT22_AH, BMP085_PRESSURE, LDR
; time between logger cycles, in minutes ; time between logger cycles, in minutes
SLEEP_MINUTES = 5 SLEEP_MINUTES = 0.4
; true for read board clock; if fail, or false, system's time will be used ; true for read board clock; if fail, or false, system's time will be used
RTC_DS1307 = true RTC_DS1307 = true
[datalog] [datalog]
; CSV delimiter for local log files (valid options: , or ; or \t ) ; CSV delimiter for local log files (valid options: , or ; or \t )
CSV_SEP = ; CSV_SEP = ;
...@@ -28,7 +23,6 @@ CSV_SEP = ; ...@@ -28,7 +23,6 @@ CSV_SEP = ;
; %Y : years | %m : months | %d : days | %H : hours | %M : mins | %S : secs ; %Y : years | %m : months | %d : days | %H : hours | %M : mins | %S : secs
DATETIME_FORMAT = %Y-%m-%d-%H-%M-%S DATETIME_FORMAT = %Y-%m-%d-%H-%M-%S
[arduino] [arduino]
; CSV list of ports to attempt connection or blank for automatic search on ; CSV list of ports to attempt connection or blank for automatic search on
; /dev/ttyACM* and /dev/ttyUSB* (where * varies from 0 to 4) ; /dev/ttyACM* and /dev/ttyUSB* (where * varies from 0 to 4)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment