main.py 11.4 KB
Newer Older
1
2
#-------------------------------------------------------------------------------
# Author: Nelso G. Jost (nelsojost@gmail.com)
3
# License: GPLv2
4
5
6
7
# Purpose: Get data from the board via serial and send it to the server.
#-------------------------------------------------------------------------------
from __future__ import print_function, division

8
from .config import Config
Nelso Jost's avatar
Nelso Jost committed
9

10
11
12
13
14
15
16
17
18
19
20
from datetime import datetime
from pprint import pprint

import logging
import os
import requests
import subprocess
import serial
import sys
import time
import json
21
import random
22
23
24
25
26
27
28


class Meteorologger:
    '''
    Provides a series of mechanisms to collect sensor data from the board
    via serial port, save locally on the machine and upload to the server.

29
30
31
    The functionality of this class relies heavily on the configuration file
    specified and managed by the config module, which is validated on the
    class instantiation and stored on self.config attribute.
32

33
    Call the run() method to start the logging processs.
34
    '''
35
36
    FIND_PORT_TIMEOUT = 0.5      # seconds
    ARDUINO_BAUD_RATE = 9600
37
38
    SERIAL_READ_TIMEOUT = 1.5  # seconds
    BOARD_RESET_TIMEOUT = 2    # seconds
39
    BOARD_RESPONSE_DELAY = 3   # seconds
40

41
42
    def __init__(self, background=False, fakedata=False):
        self.background, self.fakedata = background, fakedata
Nelso Jost's avatar
Nelso Jost committed
43
        self.config = Config()
44
45
46

    def _decode_bytes(self, raw_bytes, encoding='ascii'):
        try:
47
48
49
50
            return raw_bytes.decode(encoding).strip()
        except Exception as e:
            logging.error(" ASCIIDecodeError : {}".format(e))
            return None
51

52
    def setup_logging(self):
53
        '''
54
        Prepares the execution log file mechanism, which uses the standard
55
        library logging. Thus, the app is prepared for background execution.
56
        '''
57
58
59
60
61
        logging.basicConfig(
            level=logging.INFO,
            filename=self.SESSION_EXECUTION_LOG_FILENAME,
            format='%(asctime)s : %(levelname)s : %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')
62

63
        if not self.background:
64
65
66
67
68
69
70
            root = logging.getLogger('')
            root.setLevel(logging.INFO)
            console = logging.StreamHandler()
            console.setFormatter(logging.Formatter(
                fmt='%(asctime)s : %(levelname)s : %(message)s',
                datefmt='%Y-%m-%d %H:%M:%S'))
            root.addHandler(console)
71

72
    def setup_session_files(self):
73
        '''
74
75
        Prepare directory structure and files for execution and data logging
        by configuring proper FILENAME constants with current system time.
76
        '''
77
        session_datetime = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
78

79
80
81
82
        self.SESSION_DATALOG_FILENAME = os.path.join(self.config.DATALOG_PATH,
            'datalog-' + session_datetime + '.csv')
        self.SESSION_EXECUTION_LOG_FILENAME = os.path.join(
            self.config.EXECUTION_LOG_PATH, 'exec-' + session_datetime + '.log')
83
84

        try:
85
86
87
88
89
            with open(self.SESSION_DATALOG_FILENAME, 'w') as f:
                # initiate the file by writing the CSV header columns
                f.write(self.config['datalog']['csv_sep'].join(
                    ['dt' + self.config['datalog']['datetime_format']] +
                    [x for x in self.config['reading']['sensors']
90
                                if x != self.config.RTC_NAME]) + '\n')
91
        except Exception as e:
92
93
94
            logging.info("Unable to write at\n   {}".format(
                         self.SESSION_DATALOG_FILENAME))
            raise e
95
96


97
98
99
    def get_serial(self, verbose=True):
        ''' Keep trying to get a serial connection on any port listed on the
            self.config['arduino']['serial_port'] list until it succeed.
100

101
102
103
104
105
        Returns the serial.Serial() object.
        '''
        i = 0
        while True:
            serial_port = self.config['arduino']['serial_port'][i]
106
            try:
107
108
109
110
111
112
113
114
                ser = serial.Serial(port=serial_port,
                                    baudrate=self.ARDUINO_BAUD_RATE,
                                    timeout=self.SERIAL_READ_TIMEOUT,
                                    xonxoff=True)
                if verbose:
                    logging.info(str(ser))
                time.sleep(self.BOARD_RESET_TIMEOUT)
                return ser
115
            except Exception as e:
116
                if verbose:
117
                    logging.error("{}".format(e))
118

119
120
121
122
123
                if i < len(self.config['arduino']['serial_port']) - 1:
                    i += 1
                else:
                    i = 0
                time.sleep(self.FIND_PORT_TIMEOUT)
124

125
126
127
128
    def serial_read(self):
        '''
        Sends the serial command for reading sensors to the board and read
        its response, returning the valid ASCII resulting string.
129
        '''
130
131
132
        if self.fakedata:
            return ','.join(['{}'.format(random.randint(0, 100))
                    for x in self.config['reading']['sensors']])
133
        ser, result_line = self.get_serial(), None
134
135
        try:
            ser.flush()
136
137
            while result_line is None:
                n = ser.write(bytes(self.config['reading']['command'], 'utf8'))
138
139

                logging.info("sent: '{}' ({} bytes)".format(
140
                    self.config['reading']['command'], n))
141
                time.sleep(self.BOARD_RESPONSE_DELAY)
142
143
144

                result_line = ser.readline()
                logging.info("read: {} ({} bytes)".format(result_line,
145
                                                          len(result_line)))
146
147
                result_line = self._decode_bytes(result_line)
                if result_line is None:
148
149
                    logging.error("DecodeError: Unable to decode resulting "
                                  " line as ASCII.")
150
151
152
153
                    continue
            ser.close()
            return result_line
        except Exception as e:
154
            logging.error("{}: {}".format(e.__class__.__name__, e))
155
156
157
        finally:
            if ser:
                ser.close()
158

159
    def create_json(self, rawline):
160
161
162
163
164
165
        '''
        Given the raw serial line response (CSV string), builds and returns
        a JSON dict with validated, server-ready, sensor data.
        '''
        d = {'datetime': {'format': self.config['datalog']['datetime_format']},
             'sensors': {}}
166

167
        for name, value in zip(self.config['reading']['sensors'],
168
                               rawline.split(self.config.SERIAL_CSV_SEP)):
169
            if name == self.config.RTC_NAME:
170
171
                try:
                    d['datetime'].update(
172
173
                        source = self.config.RTC_NAME,
                        value = datetime.strptime(value, self.config.RTC_FORMAT)
174
                                        .strftime(d['datetime']['format']))
175
176
177
178
179
                except Exception as e:
                    logging.warning("DateTimeError: [{}]: Expected format '{}'"
                                    " but was given '{}' (Exception: {})"
                                    .format(name, self.config.RTC_FORMAT,
                                            value, e))
180
                continue
181

182
183
184
185
            if '<' in value:
                logging.warning("SensorReadingError: [{}]: '{}'"
                                .format(name, value))
                value = 'NaN'
186

187
            d['sensors'][name] = value
Nelso Jost's avatar
Nelso Jost committed
188

189
190
191
192
        if not 'value' in d['datetime']:
            d['datetime'].update(
                source = 'logger_system',
                value  = datetime.now().strftime(d['datetime']['format']))
193

194
        logging.info("JSON: {}".format(d))
195

196
        return d
197
198


199
200
201
202
203
204
205
206
    def write_datalog(self, json_data):
        '''
        Write the json data on the local datalog file for this session.
        Uses CSV format with columns determined by the reading/sensors settting.
        '''
        csv_line = self.config['datalog']['csv_sep'].join(
            [json_data['datetime']['value']] +
            [json_data['sensors'][n] for n in self.config['reading']['sensors']
207
                                     if n != self.config.RTC_NAME])
208
        try:
209
210
211
212
            with open(self.SESSION_DATALOG_FILENAME, 'a') as f:
                f.write(csv_line + '\n')
            logging.info("Updated datalog file at '{}'".format(
                         self.SESSION_DATALOG_FILENAME))
213
        except Exception as e:
214
215
216
            logging.error("Exception: {}".format(e))
            logging.warning("Unable to write datalog at '{}'"
                            .format(self.SESSION_DATALOG_FILENAME))
217
218


219
220
221
222
223
    def send_to_server(self, json_data):
        '''
        Try to send the json data to the server. If fail, data is stored on
        the local file outgoing.json for the next try.
        '''
224
        try:
225
226
            with open(self.config.OUTGOING_FILENAME, 'a') as f:
                f.write(json.dumps(json_data) + '\n')
227
        except Exception as e:
228
229
230
231
232
233
234
235
236
            logging.error("Unable to write at '{}'. Send to server will fail. "
                          "Exception: {}"
                          .format(self.config.OUTGOING_FILENAME, e))
            return
        r = None
        try:
            with open(self.config.OUTGOING_FILENAME) as f:

                data = [json.loads(line, encoding='utf-8') for line in f]
237

238
239
                r = requests.post(self.config['server']['api_post_url'],
                    json={'data': data,
240
                          'board_hash': self.config['server']['board_hash']})
241
242
243
244
245
246
247

            if r.status_code == 200:
                if 'success' in r.json():
                    logging.info("Server response: {}".format(r.json()))
                    os.remove(self.config.OUTGOING_FILENAME)
                else:
                    logging.error("Server response: {}".format(r.json()))
248
249
                    logging.info("Updated local file '{}'.".format(
                                 self.config.OUTGOING_FILENAME))
250
251
252
253
254
255
256
257
            else:
                raise Exception
        except Exception as e:
            logging.error("Request: {}. Unable to reach the server at '{}'. "
                          "Exception: {}".format(
                          r, self.config['server']['api_post_url'], e))
            logging.info("Updated local file '{}'.".format(
                         self.config.OUTGOING_FILENAME))
258
259
260

    def run(self):
        '''
261
262
        Starts the logger main loop, which keeps trying to read data from the
        serial port, save it locally and send it to the server.
263
264
265
266
267
268
269
270

        Basically, the loop consists of the following steps:

            1. serial_read()     # send a string, recieves a string
            2. create_json()     # validate data and make it server-ready
            3. write_datalog()   # write current data on local file for backup
            4. send_to_server()  # try to send; if fails, save data for later
        '''
271
272
        if self.config is None:
            raise Exception("Invalid configuration!")
273
        try:
274
275
276
277
            self.setup_session_files()
            self.setup_logging()
            logging.info('EXECUTION START')

278
279
            while True:
                logging.info('='*40)
280
281
282
283
                csv_readline = self.serial_read()
                json_data = self.create_json(csv_readline)
                self.write_datalog(json_data)
                self.send_to_server(json_data)
284

285
286
                logging.info("Going to sleep now for {} minutes".format(
                    self.config['reading']['sleep_minutes']))
287

288
                time.sleep(self.config['reading']['sleep_minutes'] * 60)
289
290

        except KeyboardInterrupt:
291
292
293
            logging.info("KeyboardInterrupt: EXECUTION FINISHED")
        except Exception as e:
            logging.info("Exception: {}".format(e))
294