main.py

#-------------------------------------------------------------------------------
# Author: Nelso G. Jost (nelsojost@gmail.com)
# License: GPLv2
# Purpose: Get data from the board via serial and send it to the server.
#-------------------------------------------------------------------------------
from __future__ import print_function, division

from .config import Config

from datetime import datetime
from pprint import pprint

import logging
import os
import requests
import subprocess
import serial
import sys
import time
import json


class Meteorologger:
    '''
    Provides a series of mechanisms to collect sensor data from the board
    via serial port, save locally on the machine and upload to the server.

    The functionality of this class relies heavily on the configuration file
    specified and managed by the config module, which is validated on the
    class instantiation and stored on self.config attribute.

    Call the run() method to start the logging processs.
    '''
    FIND_PORT_TIMEOUT = 0.5      # seconds
    ARDUINO_BAUD_RATE = 9600
    SERIAL_READ_TIMEOUT = 1.5  # seconds
    BOARD_RESET_TIMEOUT = 2    # seconds
    BOARD_RESPONSE_DELAY = 3   # seconds

    config = None

    def __init__(self, background=False):
        self.background = background
        self.config = Config()

    def _decode_bytes(self, raw_bytes, encoding='ascii'):
        try:
            return raw_bytes.decode(encoding).strip()
        except Exception as e:
            logging.error(" ASCIIDecodeError : {}".format(e))
            return None

    def setup_logging(self):
        '''
        Prepares the execution log file mechanism, which uses the standard
        library logging. This way de app is prepared for background execution.
        '''
        logging.basicConfig(
            level=logging.INFO,
            filename=self.SESSION_EXECUTION_LOG_FILENAME,
            format='%(asctime)s : %(levelname)s : %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')

        if not self.background:
            root = logging.getLogger('')
            root.setLevel(logging.INFO)
            console = logging.StreamHandler()
            console.setFormatter(logging.Formatter(
                fmt='%(asctime)s : %(levelname)s : %(message)s',
                datefmt='%Y-%m-%d %H:%M:%S'))
            root.addHandler(console)

    def setup_session_files(self):
        '''
        Prepare directory structure and files for execution and data logging
        by configuring proper FILENAME constants with current system time.
        '''
        session_datetime = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')

        self.SESSION_DATALOG_FILENAME = os.path.join(self.config.DATALOG_PATH,
            'datalog-' + session_datetime + '.csv')
        self.SESSION_EXECUTION_LOG_FILENAME = os.path.join(
            self.config.EXECUTION_LOG_PATH, 'exec-' + session_datetime + '.log')

        try:
            with open(self.SESSION_DATALOG_FILENAME, 'w') as f:
                # initiate the file by writing the CSV header columns
                f.write(self.config['datalog']['csv_sep'].join(
                    ['dt' + self.config['datalog']['datetime_format']] +
                    [x for x in self.config['reading']['sensors']
                                if x != self.config.RTC_NAME]) + '\n')
        except Exception as e:
            logging.info("Unable to write at\n   {}".format(
                         self.SESSION_DATALOG_FILENAME))
            raise e


    def get_serial(self, verbose=True):
        ''' Keep trying to get a serial connection on any port listed on the
            self.config['arduino']['serial_port'] list until it succeed.

        Returns the serial.Serial() object.
        '''
        i = 0
        while True:
            serial_port = self.config['arduino']['serial_port'][i]
            try:
                ser = serial.Serial(port=serial_port,
                                    baudrate=self.ARDUINO_BAUD_RATE,
                                    timeout=self.SERIAL_READ_TIMEOUT,
                                    xonxoff=True)
                if verbose:
                    logging.info(str(ser))
                time.sleep(self.BOARD_RESET_TIMEOUT)
                return ser
            except Exception as e:
                if verbose:
                    logging.error("{}".format(e))

                if i < len(self.config['arduino']['serial_port']) - 1:
                    i += 1
                else:
                    i = 0
                time.sleep(self.FIND_PORT_TIMEOUT)

    def serial_read(self):
        '''
        Sends the serial command for reading sensors to the board and read
        its response, returning the valid ASCII resulting string.
        '''
        ser, result_line = self.get_serial(), None
        try:
            ser.flush()
            while result_line is None:
                n = ser.write(bytes(self.config['reading']['command'], 'utf8'))

                logging.info("sent: '{}' ({} bytes)".format(
                    self.config['reading']['command'], n))
                time.sleep(self.BOARD_RESPONSE_DELAY)

                result_line = ser.readline()
                logging.info("read: {} ({} bytes)".format(result_line,
                                                          len(result_line)))
                result_line = self._decode_bytes(result_line)
                if result_line is None:
                    logging.error("DecodeError: Unable to decode resulting "
                                  " line as ASCII.")
                    continue
            ser.close()
            return result_line
        except Exception as e:
            logging.error("{}: {}".format(e.__class__.__name__, e))
        finally:
            if ser:
                ser.close()

    def create_json(self, readline):
        '''
        Given the raw serial line response (CSV string), builds and returns
        a JSON dict with validated, server-ready, sensor data.
        '''
        d = {'datetime': {'format': self.config['datalog']['datetime_format']},
             'sensors': {}}

        for name, value in zip(self.config['reading']['sensors'],
                               readline.split(self.config.SERIAL_CSV_SEP)):
            if name == self.config.RTC_NAME:
                try:
                    d['datetime'].update(
                        source = self.config.RTC_NAME,
                        value = datetime.strptime(value, self.config.RTC_FORMAT)
                                        .strftime(d['datetime']['format']))
                except Exception as e:
                    logging.warning("DateTimeError: [{}]: Expected format '{}'"
                                    " but was given '{}' (Exception: {})"
                                    .format(name, self.config.RTC_FORMAT,
                                            value, e))
                continue

            if '<' in value:
                logging.warning("SensorReadingError: [{}]: '{}'"
                                .format(name, value))
                value = 'NaN'

            d['sensors'][name] = value

        if not 'value' in d['datetime']:
            d['datetime'].update(
                source = 'logger_system',
                value  = datetime.now().strftime(d['datetime']['format']))

        logging.info("JSON: {}".format(d))

        return d


    def write_datalog(self, json_data):
        '''
        Write the json data on the local datalog file for this session.
        Uses CSV format with columns determined by the reading/sensors settting.
        '''
        csv_line = self.config['datalog']['csv_sep'].join(
            [json_data['datetime']['value']] +
            [json_data['sensors'][n] for n in self.config['reading']['sensors']
                                     if n != self.config.RTC_NAME])
        try:
            with open(self.SESSION_DATALOG_FILENAME, 'a') as f:
                f.write(csv_line + '\n')
            logging.info("Updated datalog file at '{}'".format(
                         self.SESSION_DATALOG_FILENAME))
        except Exception as e:
            logging.error("Exception: {}".format(e))
            logging.warning("Unable to write datalog at '{}'"
                            .format(self.SESSION_DATALOG_FILENAME))


    def send_to_server(self, json_data):
        '''
        Try to send the json data to the server. If fail, data is stored on
        the local file outgoing.json for the next try.
        '''
        try:
            with open(self.config.OUTGOING_FILENAME, 'a') as f:
                f.write(json.dumps(json_data) + '\n')
        except Exception as e:
            logging.error("Unable to write at '{}'. Send to server will fail. "
                          "Exception: {}"
                          .format(self.config.OUTGOING_FILENAME, e))
            return
        r = None
        try:
            with open(self.config.OUTGOING_FILENAME) as f:

                data = [json.loads(line, encoding='utf-8') for line in f]

                r = requests.post(self.config['server']['api_post_url'],
                    json={'data': data,
                          'user_hash': self.config['server']['user_hash']})

            if r.status_code == 200:
                if 'success' in r.json():
                    logging.info("Server response: {}".format(r.json()))
                    os.remove(self.config.OUTGOING_FILENAME)
                else:
                    logging.error("Server response: {}".format(r.json()))
                    logging.info("Updated local file '{}'.".format(
                                 self.config.OUTGOING_FILENAME))
            else:
                raise Exception
        except Exception as e:
            logging.error("Request: {}. Unable to reach the server at '{}'. "
                          "Exception: {}".format(
                          r, self.config['server']['api_post_url'], e))
            logging.info("Updated local file '{}'.".format(
                         self.config.OUTGOING_FILENAME))

    def run(self):
        '''
        Starts the logger main loop, which keeps trying to read data from the
        serial port, save it locally and send it to the server.

        Basically, the loop consists of the following steps:

            1. serial_read()     # send a string, recieves a string
            2. create_json()     # validate data and make it server-ready
            3. write_datalog()   # write current data on local file for backup
            4. send_to_server()  # try to send; if fails, save data for later
        '''
        if self.config is None:
            raise Exception("Invalid configuration!")
        try:
            self.setup_session_files()
            self.setup_logging()
            logging.info('EXECUTION START')

            while True:
                logging.info('='*40)
                csv_readline = self.serial_read()
                json_data = self.create_json(csv_readline)
                self.write_datalog(json_data)
                self.send_to_server(json_data)

                logging.info("Going to sleep now for {} minutes".format(
                    self.config['reading']['sleep_minutes']))

                time.sleep(self.config['reading']['sleep_minutes'] * 60)

        except KeyboardInterrupt:
            logging.info("KeyboardInterrupt: EXECUTION FINISHED")
        except Exception as e:
            logging.info("Exception: {}".format(e))