main.py 11 KB
Newer Older
1
2
#-------------------------------------------------------------------------------
# Author: Nelso G. Jost (nelsojost@gmail.com)
3
# License: GPLv2
4
5
6
7
# Purpose: Get data from the board via serial and send it to the server.
#-------------------------------------------------------------------------------
from __future__ import print_function, division

8
from .config import Config, RTCDateTime
Nelso Jost's avatar
Nelso Jost committed
9

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
from datetime import datetime
from pprint import pprint

import logging
import os
import requests
import subprocess
import serial
import sys
import time
import json


class Meteorologger:
    '''
    Provides a series of mechanisms to collect sensor data from the board
    via serial port, save locally on the machine and upload to the server.

28
29
30
    The functionality of this class relies heavily on the configuration file
    specified and managed by the config module, which is validated on the
    class instantiation and stored on self.config attribute.
31

32
    Call the run() method to start the logging processs.
33
    '''
34
35
    FIND_PORT_TIMEOUT = 0.5      # seconds
    ARDUINO_BAUD_RATE = 9600
36
37
    SERIAL_READ_TIMEOUT = 1.5  # seconds
    BOARD_RESET_TIMEOUT = 2    # seconds
38
    BOARD_RESPONSE_DELAY = 3   # seconds
39
40
41

    def __init__(self, verbose=False):
        self.verbose = verbose
Nelso Jost's avatar
Nelso Jost committed
42
        self.config = Config()
43
44
45
46
47
48
49
50
51
52

    def _decode_bytes(self, raw_bytes, encoding='ascii'):
        result = None
        try:
            result = raw_bytes.decode(encoding).strip()
        except:
            if self.verbose:
                print("Invalid bytes!")
        return result

53
    def setup_logging(self):
54
        '''
55
56
        Prepares the execution log file mechanism, which uses the standard
        library logging. This way de app is prepared for background execution.
57
58
        '''

59
60
61
62
63
        logging.basicConfig(
            level=logging.INFO,
            filename=self.SESSION_EXECUTION_LOG_FILENAME,
            format='%(asctime)s : %(levelname)s : %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')
64

65
66
67
68
69
70
71
72
        if self.verbose:
            root = logging.getLogger('')
            root.setLevel(logging.INFO)
            console = logging.StreamHandler()
            console.setFormatter(logging.Formatter(
                fmt='%(asctime)s : %(levelname)s : %(message)s',
                datefmt='%Y-%m-%d %H:%M:%S'))
            root.addHandler(console)
73

74
    def setup_session_files(self):
75
        '''
76
77
        Prepare directory structure and files for execution and data logging
        by configuring proper FILENAME constants with current system time.
78
        '''
79
        session_datetime = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
80

81
82
83
84
        self.SESSION_DATALOG_FILENAME = self.config.DATALOG_PATH \
            + 'datalog-' + session_datetime + '.csv'
        self.SESSION_EXECUTION_LOG_FILENAME = self.config.EXECUTION_LOG_PATH \
            + 'exec-' + session_datetime + '.log'
85
86

        try:
87
88
89
90
91
92
            with open(self.SESSION_DATALOG_FILENAME, 'w') as f:
                # initiate the file by writing the CSV header columns
                f.write(self.config['datalog']['csv_sep'].join(
                    ['dt' + self.config['datalog']['datetime_format']] +
                    [x for x in self.config['reading']['sensors']
                                if x != RTCDateTime.SENSOR_NAME]) + '\n')
93
        except Exception as e:
94
95
96
            logging.info("Unable to write at\n   {}".format(
                         self.SESSION_DATALOG_FILENAME))
            raise e
97

98
99
100
101
102
    def sync_rtc(self):
        '''
        TO BE IMPLEMENTED
        '''
        pass
103

104
105
106
    def get_serial(self, verbose=True):
        ''' Keep trying to get a serial connection on any port listed on the
            self.config['arduino']['serial_port'] list until it succeed.
107

108
109
110
111
112
        Returns the serial.Serial() object.
        '''
        i = 0
        while True:
            serial_port = self.config['arduino']['serial_port'][i]
113
            try:
114
115
116
117
118
119
120
121
                ser = serial.Serial(port=serial_port,
                                    baudrate=self.ARDUINO_BAUD_RATE,
                                    timeout=self.SERIAL_READ_TIMEOUT,
                                    xonxoff=True)
                if verbose:
                    logging.info(str(ser))
                time.sleep(self.BOARD_RESET_TIMEOUT)
                return ser
122
            except Exception as e:
123
124
                if verbose:
                    logging.info("SerialError: {}".format(e))
125

126
127
128
129
130
                if i < len(self.config['arduino']['serial_port']) - 1:
                    i += 1
                else:
                    i = 0
                time.sleep(self.FIND_PORT_TIMEOUT)
131

132
133
134
135
    def serial_read(self):
        '''
        Sends the serial command for reading sensors to the board and read
        its response, returning the valid ASCII resulting string.
136
        '''
137
        ser, result_line = self.get_serial(), None
138
139
        try:
            ser.flush()
140
141
            while result_line is None:
                n = ser.write(bytes(self.config['reading']['command'], 'utf8'))
142
143

                logging.info("sent: '{}' ({} bytes)".format(
144
                    self.config['reading']['command'], n))
145
                time.sleep(self.BOARD_RESPONSE_DELAY)
146
147
148

                result_line = ser.readline()
                logging.info("read: {} ({} bytes)".format(result_line,
149
                                                          len(result_line)))
150
151
                result_line = self._decode_bytes(result_line)
                if result_line is None:
152
153
                    logging.error("DecodeError: Unable to decode resulting "
                                  " line as ASCII.")
154
155
156
157
                    continue
            ser.close()
            return result_line
        except Exception as e:
158
            logging.error("{}: {}".format(e.__class__.__name__, e))
159
160
161
162
163
        finally:
            if ser:
                ser.close()
        return None

164

165
166
167
168
169
170
171
    def create_json(self, readline):
        '''
        Given the raw serial line response (CSV string), builds and returns
        a JSON dict with validated, server-ready, sensor data.
        '''
        d = {'datetime': {'format': self.config['datalog']['datetime_format']},
             'sensors': {}}
172

173
174
175
176
177
178
179
180
181
182
183
184
        for name, value in zip(self.config['reading']['sensors'],
                               readline.split(self.config.SERIAL_CSV_SEP)):
            if name == RTCDateTime.SENSOR_NAME:
                try:
                    d['datetime'].update(
                        source = RTCDateTime.SENSOR_NAME,
                        value = datetime.strptime(v, RTCDateTime.READ_TIMESTAMP)
                                        .strftime(d['datetime']['format']))
                except:
                    logging.warning("DateTimeError: [{}]: '{}'"
                                    .format(name, value))
                continue
185

186
187
188
189
            if '<' in value:
                logging.warning("SensorReadingError: [{}]: '{}'"
                                .format(name, value))
                value = 'NaN'
190

191
            d['sensors'][name] = value
Nelso Jost's avatar
Nelso Jost committed
192

193
194
195
196
        if not 'value' in d['datetime']:
            d['datetime'].update(
                source = 'logger_system',
                value  = datetime.now().strftime(d['datetime']['format']))
197

198
        logging.info("JSON: {}".format(d))
199

200
        return d
201
202


203
204
205
206
207
208
209
210
211
    def write_datalog(self, json_data):
        '''
        Write the json data on the local datalog file for this session.
        Uses CSV format with columns determined by the reading/sensors settting.
        '''
        csv_line = self.config['datalog']['csv_sep'].join(
            [json_data['datetime']['value']] +
            [json_data['sensors'][n] for n in self.config['reading']['sensors']
                                     if n != RTCDateTime.SENSOR_NAME])
212
        try:
213
214
215
216
            with open(self.SESSION_DATALOG_FILENAME, 'a') as f:
                f.write(csv_line + '\n')
            logging.info("Updated datalog file at '{}'".format(
                         self.SESSION_DATALOG_FILENAME))
217
        except Exception as e:
218
219
220
            logging.error("Exception: {}".format(e))
            logging.warning("Unable to write datalog at '{}'"
                            .format(self.SESSION_DATALOG_FILENAME))
221
222


223
224
225
226
227
    def send_to_server(self, json_data):
        '''
        Try to send the json data to the server. If fail, data is stored on
        the local file outgoing.json for the next try.
        '''
228
        try:
229
230
            with open(self.config.OUTGOING_FILENAME, 'a') as f:
                f.write(json.dumps(json_data) + '\n')
231
        except Exception as e:
232
233
234
235
236
237
238
239
240
            logging.error("Unable to write at '{}'. Send to server will fail. "
                          "Exception: {}"
                          .format(self.config.OUTGOING_FILENAME, e))
            return
        r = None
        try:
            with open(self.config.OUTGOING_FILENAME) as f:

                data = [json.loads(line, encoding='utf-8') for line in f]
241

242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
                r = requests.post(self.config['server']['api_post_url'],
                    json={'data': data,
                          'user_hash': self.config['server']['user_hash']})

            if r.status_code == 200:
                if 'success' in r.json():
                    logging.info("Server response: {}".format(r.json()))
                    os.remove(self.config.OUTGOING_FILENAME)
                else:
                    logging.error("Server response: {}".format(r.json()))
            else:
                raise Exception
        except Exception as e:
            logging.error("Request: {}. Unable to reach the server at '{}'. "
                          "Exception: {}".format(
                          r, self.config['server']['api_post_url'], e))
            logging.info("Updated local file '{}'.".format(
                         self.config.OUTGOING_FILENAME))
260
261
262

    def run(self):
        '''
263
264
        Starts the logger main loop, which keeps trying to read data from the
        serial port, save it locally and send it to the server.
265
266
267
268
269
270
271
272
273

        Basically, the loop consists of the following steps:

            1. serial_read()     # send a string, recieves a string
            2. create_json()     # validate data and make it server-ready
            3. write_datalog()   # write current data on local file for backup
            4. send_to_server()  # try to send; if fails, save data for later
        '''
        try:
274
275
276
277
            self.setup_session_files()
            self.setup_logging()
            logging.info('EXECUTION START')

278
279
            while True:
                logging.info('='*40)
280
281
282
283
                csv_readline = self.serial_read()
                json_data = self.create_json(csv_readline)
                self.write_datalog(json_data)
                self.send_to_server(json_data)
284

285
286
287
                logging.info(
                    "Going to sleep now for {H} hs : {M} mins : {S} secs"
                    .format(**self.config['reading']['interval']))
288

289
                time.sleep(self.config['reading']['interval_seconds'])
290
291

        except KeyboardInterrupt:
292
293
294
            logging.info("KeyboardInterrupt: EXECUTION FINISHED")
        except Exception as e:
            logging.info("Exception: {}".format(e))
295