#------------------------------------------------------------------------------- # Author: Nelso G. Jost (nelsojost@gmail.com) # License: GPLv2 # Purpose: Get data from the board via serial and send it to the server. #------------------------------------------------------------------------------- from __future__ import print_function, division from .config import Config, RTCDateTime from datetime import datetime from pprint import pprint import logging import os import requests import subprocess import serial import sys import time import json class Meteorologger: ''' Provides a series of mechanisms to collect sensor data from the board via serial port, save locally on the machine and upload to the server. The functionality of this class relies heavily on the configuration file specified and managed by the config module, which is validated on the class instantiation and stored on self.config attribute. Call the run() method to start the logging processs. ''' FIND_PORT_TIMEOUT = 0.5 # seconds ARDUINO_BAUD_RATE = 9600 SERIAL_READ_TIMEOUT = 1.5 # seconds BOARD_RESET_TIMEOUT = 2 # seconds BOARD_RESPONSE_DELAY = 3 # seconds def __init__(self, verbose=False): self.verbose = verbose self.config = Config() def _decode_bytes(self, raw_bytes, encoding='ascii'): result = None try: result = raw_bytes.decode(encoding).strip() except: if self.verbose: print("Invalid bytes!") return result def setup_logging(self): ''' Prepares the execution log file mechanism, which uses the standard library logging. This way de app is prepared for background execution. ''' logging.basicConfig( level=logging.INFO, filename=self.SESSION_EXECUTION_LOG_FILENAME, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%Y-%m-%d %H:%M:%S') if self.verbose: root = logging.getLogger('') root.setLevel(logging.INFO) console = logging.StreamHandler() console.setFormatter(logging.Formatter( fmt='%(asctime)s : %(levelname)s : %(message)s', datefmt='%Y-%m-%d %H:%M:%S')) root.addHandler(console) def setup_session_files(self): ''' Prepare directory structure and files for execution and data logging by configuring proper FILENAME constants with current system time. ''' session_datetime = datetime.now().strftime('%Y-%m-%d-%H-%M-%S') self.SESSION_DATALOG_FILENAME = self.config.DATALOG_PATH \ + 'datalog-' + session_datetime + '.csv' self.SESSION_EXECUTION_LOG_FILENAME = self.config.EXECUTION_LOG_PATH \ + 'exec-' + session_datetime + '.log' try: with open(self.SESSION_DATALOG_FILENAME, 'w') as f: # initiate the file by writing the CSV header columns f.write(self.config['datalog']['csv_sep'].join( ['dt' + self.config['datalog']['datetime_format']] + [x for x in self.config['reading']['sensors'] if x != RTCDateTime.SENSOR_NAME]) + '\n') except Exception as e: logging.info("Unable to write at\n {}".format( self.SESSION_DATALOG_FILENAME)) raise e def sync_rtc(self): ''' TO BE IMPLEMENTED ''' pass def get_serial(self, verbose=True): ''' Keep trying to get a serial connection on any port listed on the self.config['arduino']['serial_port'] list until it succeed. Returns the serial.Serial() object. ''' i = 0 while True: serial_port = self.config['arduino']['serial_port'][i] try: ser = serial.Serial(port=serial_port, baudrate=self.ARDUINO_BAUD_RATE, timeout=self.SERIAL_READ_TIMEOUT, xonxoff=True) if verbose: logging.info(str(ser)) time.sleep(self.BOARD_RESET_TIMEOUT) return ser except Exception as e: if verbose: logging.info("SerialError: {}".format(e)) if i < len(self.config['arduino']['serial_port']) - 1: i += 1 else: i = 0 time.sleep(self.FIND_PORT_TIMEOUT) def serial_read(self): ''' Sends the serial command for reading sensors to the board and read its response, returning the valid ASCII resulting string. ''' ser, result_line = self.get_serial(), None try: ser.flush() while result_line is None: n = ser.write(bytes(self.config['reading']['command'], 'utf8')) logging.info("sent: '{}' ({} bytes)".format( self.config['reading']['command'], n)) time.sleep(self.BOARD_RESPONSE_DELAY) result_line = ser.readline() logging.info("read: {} ({} bytes)".format(result_line, len(result_line))) result_line = self._decode_bytes(result_line) if result_line is None: logging.error("DecodeError: Unable to decode resulting " " line as ASCII.") continue ser.close() return result_line except Exception as e: logging.error("{}: {}".format(e.__class__.__name__, e)) finally: if ser: ser.close() return None def create_json(self, readline): ''' Given the raw serial line response (CSV string), builds and returns a JSON dict with validated, server-ready, sensor data. ''' d = {'datetime': {'format': self.config['datalog']['datetime_format']}, 'sensors': {}} for name, value in zip(self.config['reading']['sensors'], readline.split(self.config.SERIAL_CSV_SEP)): if name == RTCDateTime.SENSOR_NAME: try: d['datetime'].update( source = RTCDateTime.SENSOR_NAME, value = datetime.strptime(v, RTCDateTime.READ_TIMESTAMP) .strftime(d['datetime']['format'])) except: logging.warning("DateTimeError: [{}]: '{}'" .format(name, value)) continue if '<' in value: logging.warning("SensorReadingError: [{}]: '{}'" .format(name, value)) value = 'NaN' d['sensors'][name] = value if not 'value' in d['datetime']: d['datetime'].update( source = 'logger_system', value = datetime.now().strftime(d['datetime']['format'])) logging.info("JSON: {}".format(d)) return d def write_datalog(self, json_data): ''' Write the json data on the local datalog file for this session. Uses CSV format with columns determined by the reading/sensors settting. ''' csv_line = self.config['datalog']['csv_sep'].join( [json_data['datetime']['value']] + [json_data['sensors'][n] for n in self.config['reading']['sensors'] if n != RTCDateTime.SENSOR_NAME]) try: with open(self.SESSION_DATALOG_FILENAME, 'a') as f: f.write(csv_line + '\n') logging.info("Updated datalog file at '{}'".format( self.SESSION_DATALOG_FILENAME)) except Exception as e: logging.error("Exception: {}".format(e)) logging.warning("Unable to write datalog at '{}'" .format(self.SESSION_DATALOG_FILENAME)) def send_to_server(self, json_data): ''' Try to send the json data to the server. If fail, data is stored on the local file outgoing.json for the next try. ''' try: with open(self.config.OUTGOING_FILENAME, 'a') as f: f.write(json.dumps(json_data) + '\n') except Exception as e: logging.error("Unable to write at '{}'. Send to server will fail. " "Exception: {}" .format(self.config.OUTGOING_FILENAME, e)) return r = None try: with open(self.config.OUTGOING_FILENAME) as f: data = [json.loads(line, encoding='utf-8') for line in f] r = requests.post(self.config['server']['api_post_url'], json={'data': data, 'user_hash': self.config['server']['user_hash']}) if r.status_code == 200: if 'success' in r.json(): logging.info("Server response: {}".format(r.json())) os.remove(self.config.OUTGOING_FILENAME) else: logging.error("Server response: {}".format(r.json())) else: raise Exception except Exception as e: logging.error("Request: {}. Unable to reach the server at '{}'. " "Exception: {}".format( r, self.config['server']['api_post_url'], e)) logging.info("Updated local file '{}'.".format( self.config.OUTGOING_FILENAME)) def run(self): ''' Starts the logger main loop, which keeps trying to read data from the serial port, save it locally and send it to the server. Basically, the loop consists of the following steps: 1. serial_read() # send a string, recieves a string 2. create_json() # validate data and make it server-ready 3. write_datalog() # write current data on local file for backup 4. send_to_server() # try to send; if fails, save data for later ''' try: self.setup_session_files() self.setup_logging() logging.info('EXECUTION START') while True: logging.info('='*40) csv_readline = self.serial_read() json_data = self.create_json(csv_readline) self.write_datalog(json_data) self.send_to_server(json_data) logging.info( "Going to sleep now for {H} hs : {M} mins : {S} secs" .format(**self.config['reading']['interval'])) time.sleep(self.config['reading']['interval_seconds']) except KeyboardInterrupt: logging.info("KeyboardInterrupt: EXECUTION FINISHED") except Exception as e: logging.info("Exception: {}".format(e))