NEW: compatibility with new web app version

LICENSE

+
+    EMM Data Logger
+    Copyright (C) 2015  Nelso G. Jost
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.

Makefile

@@ -23,10 +23,12 @@ help:
 	@ echo ""
 	@ echo " tail-log    Follow updates on the last execution log"
 	@ echo " tail-data   Follow updates on the last data log"
-	@ echo " plot-data col=x   Uses Gnuplot to plot last data log col number x"
+	@ echo " plot col=x  Uses Gnuplot to plot last data log col number x"
 
-dep:
-	sudo apt-get install python3 python3-pip python-pip supervisor python-virtualenv
+deps:
+	sudo apt-get install python3 supervisor
+	sudo -H python3 scripts/get-pip.py
+	sudo -H pip3 install virtualenv
 
 apt-install:
 	chmod +x scripts/apt-install.sh
@@ -40,12 +42,11 @@ ino-install:
 	chmod +x scripts/ino-install.sh
 	./scripts/ino-install.sh
 
-setup: clean-venv create-venv
+setup: setup clean-venv venv
 	
-create-venv:
+venv:
 	@ echo "-------------------------------------------------------"
 	virtualenv -v --python='${PYBIN}' ${VENVDIR} --no-site-packages
-	${VENVDIR}/bin/pip install --upgrade pip
 	${VENVDIR}/bin/pip install -r logger/requirements.pip
 	@ echo "-------------------------------------------------------"
 	@ echo "Required Python virtual environment installed at "
@@ -67,16 +68,22 @@ firmware: ${BUILDWITH}-install
 	chmod +x scripts/ino-build.sh
 	./scripts/ino-build.sh ${BUILDWITH} ${INODIR} ${ARDUINOPATH}
 
-serial: 
-	${VENVPY} -i scripts/init_serial.py
+serial: check-venv
+	${VENVPY} scripts/init_serial.py
+
+syncrtc:
+	${VENVPY} scripts/init_serial.py --syncrtc
  
 run: check-venv
 	${VENVPY} logger/run.py
 
-deploy:
+fakerun:
+	${VENVPY} logger/run.py --fakedata
+
+deploy: check-venv
 	sudo ${VENVPY} logger/deploy.py
 
-undeploy:
+undeploy: check-venv
 	sudo ${VENVPY} logger/deploy.py -u
 
 tail-log:
@@ -96,7 +103,7 @@ tail-data:
 	@ echo ""
 	@ tail -F data/$(TMP)
 
-plot-data:
+plot:
 	@ echo "Quit by closing the window with Q and hitting Ctrl+C here to end the process"
 	@ cd tools && gnuplot -persist -e "config='config.plt'; col=${col}" loop.plt
 
@@ -109,5 +116,7 @@ clean-logs:
 clean-ino:
 	rm -rf ${INODIR}
 
-clean-all: clean-data clean-logs clean-ino clean-venv
-	cd logger && sudo py3clean app
+clean: clean-data clean-logs clean-ino clean-venv
+	find . -type f -name "*.py[co]" -delete
+	find . -type d -name "__pycache__" -delete	
+	rm -rf docs/_build

README

+***************
+EMM Data Logger
+***************
+
+    This applications retrieves meteorological data from Arduino boards of the project EMM, save them locally and also sent them to a remote server. 
+
+    More details on the wiki:
+        https://git.cta.if.ufrgs.br/meteorolog/logger/wikis/home
+
+    The root directory of this repo contains a Makefile with several commands for easy installation and usage of the software. Make sure to be on this directory when executing commands `make <target>`.
+
+Usage
+=====
+
+    The following topics describe the order of the tasks needed to install and use the software along with the Arduino board.
+
+
+* 1 -- Install system dependencies and prepare local Python execution environment:
+
+    $ make setup
+
+  PS: Uses apt for Debian packages and Python's pip3 will be installed if not present.
+
+
+* 2 -- Install firmware on the Arduino board via Arduino IDE or the terminal (through Ino Tool) with:
+
+    $ make firmware
+
+  PS: Stil requires Arduino IDE to be installed.
+
+
+* 3 -- Test the firmware by sending commands (strings) to the serial port via Arduino IDE Serial Monitor or a Python session on the terminal with:
+
+    $ make serial
+
+  Refer to the documentation for all available commands:
+    https://git.cta.if.ufrgs.br/meteorolog/logger/wikis/BoardCommands
+
+
+* 4 -- Prepare the file `settings.ini` with desired behavior and required fields, such as `BOARD_HASH`.
+
+
+* 5 -- Test the logger execution on the foreground with:
+
+    $ make run
+
+  Data will be saved on the `data/` subdirectory.
+
+
+* 6 -- Install the logger for background execution and automatic initiation:
+
+    $ make deploy
+
+
+* 7 -- You can monitor the last generated data or log with:|
+
+    $ make tail-log
+    $ make tail-data

logger/app/config.py

@@ -10,12 +10,8 @@ DEFAULT_INI=\
 ; base URL of the server
 URL = http://dados.cta.if.ufrgs.br/emm
 
-; board identification number (auto generated when a new board is created)
-BOARD_ID =
-
-; authentication token for the board's user
-USER_HASH =
-
+; user board authentication token (auto generated when a new board is created)
+BOARD_HASH = 
 
 [reading]
 ; CSV list of sensor names/nicknames (order reflect columns on datalog files)
@@ -27,7 +23,6 @@ SLEEP_MINUTES = 5
 ; true for read board clock; if fail, or false, system's time will be used
 RTC_DS1307 = true
 
-
 [datalog]
 ; CSV delimiter for local log files (valid options:  ,  or  ;  or  \t  )
 CSV_SEP = \t
@@ -36,7 +31,6 @@ CSV_SEP = \t
 ; %Y : years | %m : months | %d : days | %H : hours | %M : mins | %S : secs
 DATETIME_FORMAT = %Y-%m-%d-%H-%M-%S
 
-
 [arduino]
 ; CSV list of ports to attempt connection or blank for automatic search on
 ; /dev/ttyACM* and /dev/ttyUSB* (where * varies from 0 to 4)
@@ -57,7 +51,7 @@ class Config:
     EXECUTION_LOG_PATH = make_path_here('../logs/')
     OUTGOING_FILENAME = make_path_here('../../data/outgoing.json')
 
-    URL_API_POST_RAWSENSORDATA = '{base}api/post/rawsensordata/{bid}'
+    URL_API_POST_RAWSENSORDATA = '{base}api/post/rawsensordata'
     SERIAL_CSV_SEP = ','
 
     RTC_NAME = 'RTC_DS1307'
@@ -162,16 +156,9 @@ class Config:
         self.validate_arduino()
 
     def validate_server(self):
-        try:
-            bid = int(self['server']['board_id'])
-        except:
-            raise self.ConfigValueError('server', 'BOARD_ID',
-                'TypeError: Integer number expected!', self)
-
         self['server']['api_post_url'] = self.URL_API_POST_RAWSENSORDATA\
             .format(base=self['server']['url'] +
-                        ('' if self['server']['url'].endswith('/') else '/'),
-                    bid=bid)
+                        ('' if self['server']['url'].endswith('/') else '/'))
 
     def validate_reading(self):
         self['reading']['sensors'] = [x.strip() for x in

logger/app/main.py

@@ -18,6 +18,7 @@ import serial
 import sys
 import time
 import json
+import random
 
 
 class Meteorologger:
@@ -37,10 +38,8 @@ class Meteorologger:
     BOARD_RESET_TIMEOUT = 2    # seconds
     BOARD_RESPONSE_DELAY = 3   # seconds
 
-    config = None
-
-    def __init__(self, background=False):
-        self.background = background
+    def __init__(self, background=False, fakedata=False):
+        self.background, self.fakedata = background, fakedata
         self.config = Config()
 
     def _decode_bytes(self, raw_bytes, encoding='ascii'):
@@ -53,7 +52,7 @@ class Meteorologger:
     def setup_logging(self):
         '''
         Prepares the execution log file mechanism, which uses the standard
-        library logging. This way de app is prepared for background execution.
+        library logging. Thus, the app is prepared for background execution.
         '''
         logging.basicConfig(
             level=logging.INFO,
@@ -128,6 +127,9 @@ class Meteorologger:
         Sends the serial command for reading sensors to the board and read
         its response, returning the valid ASCII resulting string.
         '''
+        if self.fakedata:
+            return ','.join(['{}'.format(random.randint(0, 100))
+                    for x in self.config['reading']['sensors']])
         ser, result_line = self.get_serial(), None
         try:
             ser.flush()
@@ -154,7 +156,7 @@ class Meteorologger:
             if ser:
                 ser.close()
 
-    def create_json(self, readline):
+    def create_json(self, rawline):
         '''
         Given the raw serial line response (CSV string), builds and returns
         a JSON dict with validated, server-ready, sensor data.
@@ -163,7 +165,7 @@ class Meteorologger:
              'sensors': {}}
 
         for name, value in zip(self.config['reading']['sensors'],
-                               readline.split(self.config.SERIAL_CSV_SEP)):
+                               rawline.split(self.config.SERIAL_CSV_SEP)):
             if name == self.config.RTC_NAME:
                 try:
                     d['datetime'].update(
@@ -235,7 +237,7 @@ class Meteorologger:
 
                 r = requests.post(self.config['server']['api_post_url'],
                     json={'data': data,
-                          'user_hash': self.config['server']['user_hash']})
+                          'board_hash': self.config['server']['board_hash']})
 
             if r.status_code == 200:
                 if 'success' in r.json():

logger/run.py

@@ -2,6 +2,7 @@ from app.main import Meteorologger
 import sys
 
 try:
-    Meteorologger(background='--background' in sys.argv).run()
+    Meteorologger(background='--background' in sys.argv,
+                  fakedata='--fakedata' in sys.argv).run()
 except Exception as e:
     print(e.message)

scripts/apt-install.sh

 #sudo apt-get purge python3-pip python-pip
-sudo apt-get update
-sudo apt-get install python python3 python3-pip supervisor python-pip
+#sudo apt-get update
+#sudo apt-get install python python3 python3-pip supervisor python-pip
 echo "Finding pip3 binary.."
 PIP3=`dpkg -L python3-pip | grep /usr/bin/pip | tail -1`
 echo "PIP3=$PIP3"

scripts/init_serial.py

 import serial
+import sys
 import time
 from datetime import datetime
 
-def get_serial(baudrate=9600, read_timeout=1.5, board_reset_timeout=3,
+def get_serial(baudrate=9600, read_timeout=1.5, board_reset_timeout=2,
                find_port_timeout=0.5):
     ports = []
     for i in range(5):
@@ -27,49 +28,69 @@ def get_serial(baudrate=9600, read_timeout=1.5, board_reset_timeout=3,
                 i = 0
             time.sleep(find_port_timeout)
 
-print("\nAttempting serial connection ...\n")
-ser = get_serial()
-print(ser)
-ser.flush()
-
 def send(command_str, response_wait=0):
     '''
     Send the string 'command_str' to the serial port and return the response.
     '''
     ser.write(bytes(command_str, encoding='utf-8'))
-    time.sleep(response_wait)
+    # time.sleep(response_wait)
     try:
         raw = ser.readall()
         return raw.decode('ascii').strip()
     except:
         print("Unable to decode raw response to ASCII:\n{}".format(raw))
 
+
 def sync_rtc():
     '''
     Synchronizes the board clock with the system's.
     '''
     dt = datetime.now()
-    ser.write(bytes('setRTC,{Y},{m},{d},{H},{M},{S}'.format(
-        Y=dt.year, m=dt.month, d=dt.day, H=dt.hour, M=dt.minute, S=dt.second),
-        encoding='utf-8'))
+    print('\nCurrent system time:', dt.isoformat().replace('T', ' '))
+    print("\nAttempting serial connection ...", end='')
+    ser = get_serial()
+    print('got it!')
+    print(ser)
+    ser.flush()
+
+    command = 'setrtc,{Y},{m},{d},{H},{M},{S}'.format(
+        Y=dt.year, m=dt.month, d=dt.day, H=dt.hour, M=dt.minute, S=dt.second)
+
+    print('\nSending serial command:\n    "{}"\n'.format(command))
+
+    ser.write(bytes(command, encoding='utf-8'))
     try:
         raw = ser.readall()
         return raw.decode('ascii').strip()
     except:
         print("Unable to decode raw response to ASCII:\n{}".format(raw))
 
-print("\nSent 'help'... waiting for board response ...")
-response = send('help')
-print("\nBoard commands:\n" + '-'*40 + '\n' + response + '\n' + '-'*40)
-print(
-"""\
-You are now inside the Python interpreter! The commands above
-must be passed as a string to the send() function!
 
-Examples:
-    >>> send('read,t,l')                   # for reading temp. and lum.
-    >>> send('setrtc,2015,7,15,17,45,0')   # for setting the datetime on RTC
+if '-c' in sys.argv and len(sys.argv) == 3:
+    print(send(input('> ')))
+elif '--syncrtc' in sys.argv:
+    print(sync_rtc())
+else:
+    print("\nAttempting serial connection ...\n")
+    ser = get_serial()
+    print(ser)
+    ser.flush()
 
-Hit Ctrl+D to exit!""")
-print('-'*40 + '\n')
+    print("\nSent 'help'... waiting for board response ...")
+    response = send('help')
+    print("\nBoard commands:\n" + '-'*40 + '\n' + response + '\n' + '-'*40)
+    print(
+    """\
+    Examples:
+
+        > read,t,l                      # for reading temp. and lum.
+        > setrtc,2015,7,15,17,45,0      # for setting RTC datetime
+
+    Hit Ctrl+C to exit!""")
+    print('-'*40 + '\n')
+    try:
+        while True:
+            print(send(input('> ')))
+    except:
+        pass
 

settings.ini

 [server]
-; base URL of the server
+; full URL of the data server
 URL = http://dados.cta.if.ufrgs.br/emm
 
-; board identification number (auto generated when a new board is created)
-BOARD_ID = 
-
-; authentication token for the board's user
-USER_HASH = 
-
+; user board authentication token (auto generated when a new board is created)
+BOARD_HASH = 
 
 [reading]
 ; CSV list of sensor names/nicknames (order reflect columns on datalog files)
 SENSORS = DHT22_TEMP, DHT22_AH, BMP085_PRESSURE, LDR
 
 ; time between logger cycles, in minutes
-SLEEP_MINUTES = 5
+SLEEP_MINUTES = 0.4
 
 ; true for read board clock; if fail, or false, system's time will be used
 RTC_DS1307 = true
 
-
 [datalog]
 ; CSV delimiter for local log files (valid options:  ,  or  ;  or  \t  )
 CSV_SEP = ;
@@ -28,7 +23,6 @@ CSV_SEP = ;
 ; %Y : years | %m : months | %d : days | %H : hours | %M : mins | %S : secs 
 DATETIME_FORMAT = %Y-%m-%d-%H-%M-%S
 
-
 [arduino]
 ; CSV list of ports to attempt connection or blank for automatic search on 
 ; /dev/ttyACM* and /dev/ttyUSB* (where * varies from 0 to 4)