main.py 11.3 KB
Newer Older
1
2
#-------------------------------------------------------------------------------
# Author: Nelso G. Jost (nelsojost@gmail.com)
3
# License: GPLv2
4
5
6
7
# Purpose: Get data from the board via serial and send it to the server.
#-------------------------------------------------------------------------------
from __future__ import print_function, division

8
from .config import Config
Nelso Jost's avatar
Nelso Jost committed
9

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
from datetime import datetime
from pprint import pprint

import logging
import os
import requests
import subprocess
import serial
import sys
import time
import json


class Meteorologger:
    '''
    Provides a series of mechanisms to collect sensor data from the board
    via serial port, save locally on the machine and upload to the server.

28
29
30
    The functionality of this class relies heavily on the configuration file
    specified and managed by the config module, which is validated on the
    class instantiation and stored on self.config attribute.
31

32
    Call the run() method to start the logging processs.
33
    '''
34
35
    FIND_PORT_TIMEOUT = 0.5      # seconds
    ARDUINO_BAUD_RATE = 9600
36
37
    SERIAL_READ_TIMEOUT = 1.5  # seconds
    BOARD_RESET_TIMEOUT = 2    # seconds
38
    BOARD_RESPONSE_DELAY = 3   # seconds
39

40
41
42
43
    config = None

    def __init__(self, background=False):
        self.background = background
Nelso Jost's avatar
Nelso Jost committed
44
        self.config = Config()
45
46
47

    def _decode_bytes(self, raw_bytes, encoding='ascii'):
        try:
48
49
50
51
            return raw_bytes.decode(encoding).strip()
        except Exception as e:
            logging.error(" ASCIIDecodeError : {}".format(e))
            return None
52

53
    def setup_logging(self):
54
        '''
55
56
        Prepares the execution log file mechanism, which uses the standard
        library logging. This way de app is prepared for background execution.
57
        '''
58
59
60
61
62
        logging.basicConfig(
            level=logging.INFO,
            filename=self.SESSION_EXECUTION_LOG_FILENAME,
            format='%(asctime)s : %(levelname)s : %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')
63

64
        if not self.background:
65
66
67
68
69
70
71
            root = logging.getLogger('')
            root.setLevel(logging.INFO)
            console = logging.StreamHandler()
            console.setFormatter(logging.Formatter(
                fmt='%(asctime)s : %(levelname)s : %(message)s',
                datefmt='%Y-%m-%d %H:%M:%S'))
            root.addHandler(console)
72

73
    def setup_session_files(self):
74
        '''
75
76
        Prepare directory structure and files for execution and data logging
        by configuring proper FILENAME constants with current system time.
77
        '''
78
        session_datetime = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
79

80
81
82
83
        self.SESSION_DATALOG_FILENAME = os.path.join(self.config.DATALOG_PATH,
            'datalog-' + session_datetime + '.csv')
        self.SESSION_EXECUTION_LOG_FILENAME = os.path.join(
            self.config.EXECUTION_LOG_PATH, 'exec-' + session_datetime + '.log')
84
85

        try:
86
87
88
89
90
            with open(self.SESSION_DATALOG_FILENAME, 'w') as f:
                # initiate the file by writing the CSV header columns
                f.write(self.config['datalog']['csv_sep'].join(
                    ['dt' + self.config['datalog']['datetime_format']] +
                    [x for x in self.config['reading']['sensors']
91
                                if x != self.config.RTC_NAME]) + '\n')
92
        except Exception as e:
93
94
95
            logging.info("Unable to write at\n   {}".format(
                         self.SESSION_DATALOG_FILENAME))
            raise e
96
97


98
99
100
    def get_serial(self, verbose=True):
        ''' Keep trying to get a serial connection on any port listed on the
            self.config['arduino']['serial_port'] list until it succeed.
101

102
103
104
105
106
        Returns the serial.Serial() object.
        '''
        i = 0
        while True:
            serial_port = self.config['arduino']['serial_port'][i]
107
            try:
108
109
110
111
112
113
114
115
                ser = serial.Serial(port=serial_port,
                                    baudrate=self.ARDUINO_BAUD_RATE,
                                    timeout=self.SERIAL_READ_TIMEOUT,
                                    xonxoff=True)
                if verbose:
                    logging.info(str(ser))
                time.sleep(self.BOARD_RESET_TIMEOUT)
                return ser
116
            except Exception as e:
117
                if verbose:
118
                    logging.error("{}".format(e))
119

120
121
122
123
124
                if i < len(self.config['arduino']['serial_port']) - 1:
                    i += 1
                else:
                    i = 0
                time.sleep(self.FIND_PORT_TIMEOUT)
125

126
127
128
129
    def serial_read(self):
        '''
        Sends the serial command for reading sensors to the board and read
        its response, returning the valid ASCII resulting string.
130
        '''
131
        ser, result_line = self.get_serial(), None
132
133
        try:
            ser.flush()
134
135
            while result_line is None:
                n = ser.write(bytes(self.config['reading']['command'], 'utf8'))
136
137

                logging.info("sent: '{}' ({} bytes)".format(
138
                    self.config['reading']['command'], n))
139
                time.sleep(self.BOARD_RESPONSE_DELAY)
140
141
142

                result_line = ser.readline()
                logging.info("read: {} ({} bytes)".format(result_line,
143
                                                          len(result_line)))
144
145
                result_line = self._decode_bytes(result_line)
                if result_line is None:
146
147
                    logging.error("DecodeError: Unable to decode resulting "
                                  " line as ASCII.")
148
149
150
151
                    continue
            ser.close()
            return result_line
        except Exception as e:
152
            logging.error("{}: {}".format(e.__class__.__name__, e))
153
154
155
        finally:
            if ser:
                ser.close()
156

157
158
159
160
161
162
163
    def create_json(self, readline):
        '''
        Given the raw serial line response (CSV string), builds and returns
        a JSON dict with validated, server-ready, sensor data.
        '''
        d = {'datetime': {'format': self.config['datalog']['datetime_format']},
             'sensors': {}}
164

165
166
        for name, value in zip(self.config['reading']['sensors'],
                               readline.split(self.config.SERIAL_CSV_SEP)):
167
            if name == self.config.RTC_NAME:
168
169
                try:
                    d['datetime'].update(
170
171
                        source = self.config.RTC_NAME,
                        value = datetime.strptime(value, self.config.RTC_FORMAT)
172
                                        .strftime(d['datetime']['format']))
173
174
175
176
177
                except Exception as e:
                    logging.warning("DateTimeError: [{}]: Expected format '{}'"
                                    " but was given '{}' (Exception: {})"
                                    .format(name, self.config.RTC_FORMAT,
                                            value, e))
178
                continue
179

180
181
182
183
            if '<' in value:
                logging.warning("SensorReadingError: [{}]: '{}'"
                                .format(name, value))
                value = 'NaN'
184

185
            d['sensors'][name] = value
Nelso Jost's avatar
Nelso Jost committed
186

187
188
189
190
        if not 'value' in d['datetime']:
            d['datetime'].update(
                source = 'logger_system',
                value  = datetime.now().strftime(d['datetime']['format']))
191

192
        logging.info("JSON: {}".format(d))
193

194
        return d
195
196


197
198
199
200
201
202
203
204
    def write_datalog(self, json_data):
        '''
        Write the json data on the local datalog file for this session.
        Uses CSV format with columns determined by the reading/sensors settting.
        '''
        csv_line = self.config['datalog']['csv_sep'].join(
            [json_data['datetime']['value']] +
            [json_data['sensors'][n] for n in self.config['reading']['sensors']
205
                                     if n != self.config.RTC_NAME])
206
        try:
207
208
209
210
            with open(self.SESSION_DATALOG_FILENAME, 'a') as f:
                f.write(csv_line + '\n')
            logging.info("Updated datalog file at '{}'".format(
                         self.SESSION_DATALOG_FILENAME))
211
        except Exception as e:
212
213
214
            logging.error("Exception: {}".format(e))
            logging.warning("Unable to write datalog at '{}'"
                            .format(self.SESSION_DATALOG_FILENAME))
215
216


217
218
219
220
221
    def send_to_server(self, json_data):
        '''
        Try to send the json data to the server. If fail, data is stored on
        the local file outgoing.json for the next try.
        '''
222
        try:
223
224
            with open(self.config.OUTGOING_FILENAME, 'a') as f:
                f.write(json.dumps(json_data) + '\n')
225
        except Exception as e:
226
227
228
229
230
231
232
233
234
            logging.error("Unable to write at '{}'. Send to server will fail. "
                          "Exception: {}"
                          .format(self.config.OUTGOING_FILENAME, e))
            return
        r = None
        try:
            with open(self.config.OUTGOING_FILENAME) as f:

                data = [json.loads(line, encoding='utf-8') for line in f]
235

236
237
238
239
240
241
242
243
244
245
                r = requests.post(self.config['server']['api_post_url'],
                    json={'data': data,
                          'user_hash': self.config['server']['user_hash']})

            if r.status_code == 200:
                if 'success' in r.json():
                    logging.info("Server response: {}".format(r.json()))
                    os.remove(self.config.OUTGOING_FILENAME)
                else:
                    logging.error("Server response: {}".format(r.json()))
246
247
                    logging.info("Updated local file '{}'.".format(
                                 self.config.OUTGOING_FILENAME))
248
249
250
251
252
253
254
255
            else:
                raise Exception
        except Exception as e:
            logging.error("Request: {}. Unable to reach the server at '{}'. "
                          "Exception: {}".format(
                          r, self.config['server']['api_post_url'], e))
            logging.info("Updated local file '{}'.".format(
                         self.config.OUTGOING_FILENAME))
256
257
258

    def run(self):
        '''
259
260
        Starts the logger main loop, which keeps trying to read data from the
        serial port, save it locally and send it to the server.
261
262
263
264
265
266
267
268

        Basically, the loop consists of the following steps:

            1. serial_read()     # send a string, recieves a string
            2. create_json()     # validate data and make it server-ready
            3. write_datalog()   # write current data on local file for backup
            4. send_to_server()  # try to send; if fails, save data for later
        '''
269
270
        if self.config is None:
            raise Exception("Invalid configuration!")
271
        try:
272
273
274
275
            self.setup_session_files()
            self.setup_logging()
            logging.info('EXECUTION START')

276
277
            while True:
                logging.info('='*40)
278
279
280
281
                csv_readline = self.serial_read()
                json_data = self.create_json(csv_readline)
                self.write_datalog(json_data)
                self.send_to_server(json_data)
282

283
284
                logging.info("Going to sleep now for {} minutes".format(
                    self.config['reading']['sleep_minutes']))
285

286
                time.sleep(self.config['reading']['sleep_minutes'] * 60)
287
288

        except KeyboardInterrupt:
289
290
291
            logging.info("KeyboardInterrupt: EXECUTION FINISHED")
        except Exception as e:
            logging.info("Exception: {}".format(e))
292