Sensor logger for Raspberry Pi in a stratospheric probe












14












$begingroup$


I'm writing a Python Script for a Raspberry Pi to measure different sensors. We are planning to send the Pi with that Script running to the stratosphere, so the power usage for the Pi is limited.



I excuse myself in advance for the code, I had no prior experience with Python.
Are there any ways I can make this code more battery friendly? Would it be beneficial to write 10 rows at once instead of writing one row at a time?



#!/usr/bin/env python3

from sense_hat import SenseHat
import time
import csv
import datetime

sense = SenseHat()
sense.clear()
sense.set_imu_config(True, True, True)
sense.low_light = True

with open('data.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Temperatur1', 'Temperatur2', 'Temperatur3', 'Luftdruck', 'Luftfeuchtigkeit', 'Yaw', 'Pitch', 'Roll', 'Compass X', 'Compass Y', 'Compass Z', 'Gyro X', 'Gyro Y', 'Gyro Z'])

with open('acc.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Acc_X','Acc_Y','Acc_Z'])

with open('log.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Fehler'])

# Farben definieren
red = (255, 0, 0)
green = (0, 255, 0)
black = (0,0,0)

def writeDataToCsv(temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z):
with open('data.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z])

def writeAccelerationToCsv(x,y,z):
with open('acc.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),x,y,z])
sense.set_pixel(0, 0, green)
time.sleep(.05)
sense.set_pixel(0, 0, black)

def main():
sense.set_pixel(0, 0, black)
counter = 0
try:
while True:
#Region Acceleration
acceleration = sense.get_accelerometer_raw()
acc_x = acceleration['x']
acc_y = acceleration['y']
acc_z = acceleration['z']
writeAccelerationToCsv(acc_x,acc_y,acc_z)
time.sleep(.250)
counter+=1

#Region Data
if(counter == 4):
temperature = sense.get_temperature()
temperature2 = sense.get_temperature_from_humidity()
temperature3 = sense.get_temperature_from_pressure()

pressure = sense.get_pressure()
humidty = sense.get_humidity()

orientation = sense.get_orientation()
yaw = orientation["yaw"]
pitch = orientation["pitch"]
roll = orientation["roll"]

mag = sense.get_compass_raw()
mag_x = mag["x"]
mag_y = mag["y"]
mag_z = mag["z"]

gyro = sense.get_gyroscope_raw()
gyro_x = gyro["x"]
gyro_y = gyro["y"]
gyro_z = gyro["z"]

writeDataToCsv(temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z)

counter = 0;
except Exception as e:
with open('log.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),str(e)])
sense.set_pixel(1, 0, red)
finally:
pass
main()

if __name__ == '__main__':
main()









share|improve this question









New contributor




Lexu is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.







$endgroup$








  • 4




    $begingroup$
    The Pi uses quite a lot of power, with only a little of it going to the CPU, so you can't make much of a difference in the code there.
    $endgroup$
    – Simon Richter
    20 hours ago










  • $begingroup$
    Is there a reason you can't use a lower power device, equip a bigger battery, or use solar panels?
    $endgroup$
    – forest
    17 hours ago
















14












$begingroup$


I'm writing a Python Script for a Raspberry Pi to measure different sensors. We are planning to send the Pi with that Script running to the stratosphere, so the power usage for the Pi is limited.



I excuse myself in advance for the code, I had no prior experience with Python.
Are there any ways I can make this code more battery friendly? Would it be beneficial to write 10 rows at once instead of writing one row at a time?



#!/usr/bin/env python3

from sense_hat import SenseHat
import time
import csv
import datetime

sense = SenseHat()
sense.clear()
sense.set_imu_config(True, True, True)
sense.low_light = True

with open('data.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Temperatur1', 'Temperatur2', 'Temperatur3', 'Luftdruck', 'Luftfeuchtigkeit', 'Yaw', 'Pitch', 'Roll', 'Compass X', 'Compass Y', 'Compass Z', 'Gyro X', 'Gyro Y', 'Gyro Z'])

with open('acc.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Acc_X','Acc_Y','Acc_Z'])

with open('log.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Fehler'])

# Farben definieren
red = (255, 0, 0)
green = (0, 255, 0)
black = (0,0,0)

def writeDataToCsv(temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z):
with open('data.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z])

def writeAccelerationToCsv(x,y,z):
with open('acc.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),x,y,z])
sense.set_pixel(0, 0, green)
time.sleep(.05)
sense.set_pixel(0, 0, black)

def main():
sense.set_pixel(0, 0, black)
counter = 0
try:
while True:
#Region Acceleration
acceleration = sense.get_accelerometer_raw()
acc_x = acceleration['x']
acc_y = acceleration['y']
acc_z = acceleration['z']
writeAccelerationToCsv(acc_x,acc_y,acc_z)
time.sleep(.250)
counter+=1

#Region Data
if(counter == 4):
temperature = sense.get_temperature()
temperature2 = sense.get_temperature_from_humidity()
temperature3 = sense.get_temperature_from_pressure()

pressure = sense.get_pressure()
humidty = sense.get_humidity()

orientation = sense.get_orientation()
yaw = orientation["yaw"]
pitch = orientation["pitch"]
roll = orientation["roll"]

mag = sense.get_compass_raw()
mag_x = mag["x"]
mag_y = mag["y"]
mag_z = mag["z"]

gyro = sense.get_gyroscope_raw()
gyro_x = gyro["x"]
gyro_y = gyro["y"]
gyro_z = gyro["z"]

writeDataToCsv(temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z)

counter = 0;
except Exception as e:
with open('log.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),str(e)])
sense.set_pixel(1, 0, red)
finally:
pass
main()

if __name__ == '__main__':
main()









share|improve this question









New contributor




Lexu is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.







$endgroup$








  • 4




    $begingroup$
    The Pi uses quite a lot of power, with only a little of it going to the CPU, so you can't make much of a difference in the code there.
    $endgroup$
    – Simon Richter
    20 hours ago










  • $begingroup$
    Is there a reason you can't use a lower power device, equip a bigger battery, or use solar panels?
    $endgroup$
    – forest
    17 hours ago














14












14








14


1



$begingroup$


I'm writing a Python Script for a Raspberry Pi to measure different sensors. We are planning to send the Pi with that Script running to the stratosphere, so the power usage for the Pi is limited.



I excuse myself in advance for the code, I had no prior experience with Python.
Are there any ways I can make this code more battery friendly? Would it be beneficial to write 10 rows at once instead of writing one row at a time?



#!/usr/bin/env python3

from sense_hat import SenseHat
import time
import csv
import datetime

sense = SenseHat()
sense.clear()
sense.set_imu_config(True, True, True)
sense.low_light = True

with open('data.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Temperatur1', 'Temperatur2', 'Temperatur3', 'Luftdruck', 'Luftfeuchtigkeit', 'Yaw', 'Pitch', 'Roll', 'Compass X', 'Compass Y', 'Compass Z', 'Gyro X', 'Gyro Y', 'Gyro Z'])

with open('acc.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Acc_X','Acc_Y','Acc_Z'])

with open('log.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Fehler'])

# Farben definieren
red = (255, 0, 0)
green = (0, 255, 0)
black = (0,0,0)

def writeDataToCsv(temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z):
with open('data.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z])

def writeAccelerationToCsv(x,y,z):
with open('acc.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),x,y,z])
sense.set_pixel(0, 0, green)
time.sleep(.05)
sense.set_pixel(0, 0, black)

def main():
sense.set_pixel(0, 0, black)
counter = 0
try:
while True:
#Region Acceleration
acceleration = sense.get_accelerometer_raw()
acc_x = acceleration['x']
acc_y = acceleration['y']
acc_z = acceleration['z']
writeAccelerationToCsv(acc_x,acc_y,acc_z)
time.sleep(.250)
counter+=1

#Region Data
if(counter == 4):
temperature = sense.get_temperature()
temperature2 = sense.get_temperature_from_humidity()
temperature3 = sense.get_temperature_from_pressure()

pressure = sense.get_pressure()
humidty = sense.get_humidity()

orientation = sense.get_orientation()
yaw = orientation["yaw"]
pitch = orientation["pitch"]
roll = orientation["roll"]

mag = sense.get_compass_raw()
mag_x = mag["x"]
mag_y = mag["y"]
mag_z = mag["z"]

gyro = sense.get_gyroscope_raw()
gyro_x = gyro["x"]
gyro_y = gyro["y"]
gyro_z = gyro["z"]

writeDataToCsv(temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z)

counter = 0;
except Exception as e:
with open('log.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),str(e)])
sense.set_pixel(1, 0, red)
finally:
pass
main()

if __name__ == '__main__':
main()









share|improve this question









New contributor




Lexu is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.







$endgroup$




I'm writing a Python Script for a Raspberry Pi to measure different sensors. We are planning to send the Pi with that Script running to the stratosphere, so the power usage for the Pi is limited.



I excuse myself in advance for the code, I had no prior experience with Python.
Are there any ways I can make this code more battery friendly? Would it be beneficial to write 10 rows at once instead of writing one row at a time?



#!/usr/bin/env python3

from sense_hat import SenseHat
import time
import csv
import datetime

sense = SenseHat()
sense.clear()
sense.set_imu_config(True, True, True)
sense.low_light = True

with open('data.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Temperatur1', 'Temperatur2', 'Temperatur3', 'Luftdruck', 'Luftfeuchtigkeit', 'Yaw', 'Pitch', 'Roll', 'Compass X', 'Compass Y', 'Compass Z', 'Gyro X', 'Gyro Y', 'Gyro Z'])

with open('acc.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Acc_X','Acc_Y','Acc_Z'])

with open('log.csv', mode='w') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['Zeit','Fehler'])

# Farben definieren
red = (255, 0, 0)
green = (0, 255, 0)
black = (0,0,0)

def writeDataToCsv(temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z):
with open('data.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z])

def writeAccelerationToCsv(x,y,z):
with open('acc.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),x,y,z])
sense.set_pixel(0, 0, green)
time.sleep(.05)
sense.set_pixel(0, 0, black)

def main():
sense.set_pixel(0, 0, black)
counter = 0
try:
while True:
#Region Acceleration
acceleration = sense.get_accelerometer_raw()
acc_x = acceleration['x']
acc_y = acceleration['y']
acc_z = acceleration['z']
writeAccelerationToCsv(acc_x,acc_y,acc_z)
time.sleep(.250)
counter+=1

#Region Data
if(counter == 4):
temperature = sense.get_temperature()
temperature2 = sense.get_temperature_from_humidity()
temperature3 = sense.get_temperature_from_pressure()

pressure = sense.get_pressure()
humidty = sense.get_humidity()

orientation = sense.get_orientation()
yaw = orientation["yaw"]
pitch = orientation["pitch"]
roll = orientation["roll"]

mag = sense.get_compass_raw()
mag_x = mag["x"]
mag_y = mag["y"]
mag_z = mag["z"]

gyro = sense.get_gyroscope_raw()
gyro_x = gyro["x"]
gyro_y = gyro["y"]
gyro_z = gyro["z"]

writeDataToCsv(temperature, temperature2, temperature3, pressure, humidty, yaw, pitch, roll, mag_x, mag_y, mag_z, gyro_x, gyro_y, gyro_z)

counter = 0;
except Exception as e:
with open('log.csv', mode='a') as file:
writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([datetime.datetime.now(),str(e)])
sense.set_pixel(1, 0, red)
finally:
pass
main()

if __name__ == '__main__':
main()






python csv logging embedded raspberry-pi






share|improve this question









New contributor




Lexu is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.











share|improve this question









New contributor




Lexu is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.









share|improve this question




share|improve this question








edited 6 hours ago









200_success

129k15153417




129k15153417






New contributor




Lexu is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.









asked yesterday









LexuLexu

1715




1715




New contributor




Lexu is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.





New contributor





Lexu is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.






Lexu is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.








  • 4




    $begingroup$
    The Pi uses quite a lot of power, with only a little of it going to the CPU, so you can't make much of a difference in the code there.
    $endgroup$
    – Simon Richter
    20 hours ago










  • $begingroup$
    Is there a reason you can't use a lower power device, equip a bigger battery, or use solar panels?
    $endgroup$
    – forest
    17 hours ago














  • 4




    $begingroup$
    The Pi uses quite a lot of power, with only a little of it going to the CPU, so you can't make much of a difference in the code there.
    $endgroup$
    – Simon Richter
    20 hours ago










  • $begingroup$
    Is there a reason you can't use a lower power device, equip a bigger battery, or use solar panels?
    $endgroup$
    – forest
    17 hours ago








4




4




$begingroup$
The Pi uses quite a lot of power, with only a little of it going to the CPU, so you can't make much of a difference in the code there.
$endgroup$
– Simon Richter
20 hours ago




$begingroup$
The Pi uses quite a lot of power, with only a little of it going to the CPU, so you can't make much of a difference in the code there.
$endgroup$
– Simon Richter
20 hours ago












$begingroup$
Is there a reason you can't use a lower power device, equip a bigger battery, or use solar panels?
$endgroup$
– forest
17 hours ago




$begingroup$
Is there a reason you can't use a lower power device, equip a bigger battery, or use solar panels?
$endgroup$
– forest
17 hours ago










4 Answers
4






active

oldest

votes


















14












$begingroup$



  • Do not call main recursively. You are setting yourself up for stack overflow. Consider instead



    def main():
    while True:
    try:
    your_logic_here
    except Exception as e:
    your_logging_here



  • Testing for counter == 4 is better done in a loop:



        for _ in range(4):
    handle_acceleration
    handle_the_rest


  • An unattended controller should handle exceptions more diligently. For sure, you want to act differently for the exceptions raised by sense (if any) vs exceptions raised by writing to the file.


  • Regarding battery, avoid binary-to-text conversions. Store your data as binary, and convert them to CSV offline, after the Pi safely returns.







share|improve this answer









$endgroup$





















    8












    $begingroup$

    Have you already executed the code to see how it performs and if the battery will last? There is that famous Donald Knuth quote saying premature optimization is the root of all evil (or at least most of it) in programming.



    I never had to think about the energy consumption of a program, so I cannot tell you about the power efficieny. But as vnp already did, I can also share my opinion about the code structure to help you to identify bottlenecks more easily. Also, a different structure should help you to still log some data even in case of exceptions.



    Here is what struck me on first read:




    • most of the code is defined in the main method

    • you overwrite the complete data files at the beginning of the program

    • very broad exception clause

    • repetition of the csv write (violates the zen of python - not dry - dont repeat yourself)


    I tried to resolve some of the issues and refactored the structure of the code:



    #!/usr/bin/env python3

    from sense_hat import SenseHat
    import time
    import csv
    import datetime

    # defined constants on moduel level and capitalized the names (pep8: https://www.python.org/dev/peps/pep-0008/#constants)
    RED = (255, 0, 0)
    GREEN = (0, 255, 0)
    BLACK = (0,0,0)


    class DataLogger(object):

    def __init__(self, init_csv_files=False):
    # initalize the commonly ued sensor
    self.sense = SenseHat()
    self.sense.clear()
    self.sense.set_imu_config(True, True, True)
    self.sense.low_light = True

    # only initialize the csv files, if intended
    # I would suggest not to init them in the same program though.
    # If - for some reasons - the python interpreter crashes and the script is restarted,
    # the init of the csv_files will overwrite all the data which was logged so far.
    if init_csv_files:
    self.init_csv_files()

    def write_data_to_file(self, data, file_name, mode='a', delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL):
    """
    Helper method to write the given data to a csv file. Using 'append' as default mode to avoid accidental overwrites.
    """
    with open(file_name, mode=mode) as file:
    writer = csv.writer(file, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
    writer.writerow(data)

    def init_csv_files(self):
    # see comment in init method
    data_headings = ['Zeit','Temperatur1', 'Temperatur2', 'Temperatur3', 'Luftdruck', 'Luftfeuchtigkeit', 'Yaw', 'Pitch', 'Roll', 'Compass X', 'Compass Y', 'Compass Z', 'Gyro X', 'Gyro Y', 'Gyro Z']
    self.write_data_to_file(data_headings, 'data.csv', 'w')

    acc_headings = ['Zeit','Acc_X','Acc_Y','Acc_Z']
    self.write_data_to_file(acc_headings, 'acc.csv', 'w')

    log_headings = ['Zeit','Fehler']
    self.write_data_to_file(log_headings, 'log.csv', 'w')

    def start_logging(self):
    # actual execution
    sense.set_pixel(0, 0, BLACK)
    counter = 0

    while True:
    # moved the accelleration logging to a different method
    # and catched possible exceptions there, so the counter will still be increase
    # and the rest of the data may still be logged even if the accelleration data
    # always raises exceptions
    self.log_accelleration()
    time.sleep(.250)
    counter += 1

    # using counter % 4 == 0 instead of counter == 4
    # this will evaluate to true for every number divisible by 4
    # If you do the strict comparision, you could find yourself in the scenario
    # where the data logging is never executed, if the counter is larger than 4
    # (in this case this is very unlikely, but in threaded scenarios it would be possible,
    # so doing modulo 4 is more defensive)
    if(counter % 4 == 0):
    self.log_data()
    counter = 0

    def log_accelleration(self):
    acceleration_data = get_accelleration()
    if acceleration_data:
    try:
    self.write_data_to_file(acceleration_data, 'acc.csv')
    except Exception as e:
    self.log_exception(e)
    pass
    else:
    # no exception occurred
    self.sense.set_pixel(0, 0, green)
    time.sleep(.05)
    finally:
    self.sense.set_pixel(0, 0, black)

    def log_data(self):
    # saving datetime first, before reading all the sensor data
    data = [datetime.datetime.now()]

    # moved each of the calls to sense in a separate method
    # exceptions will lead to empty entries being logged but
    # if e.g. get_pressure raises an exceptions, the other data may still get logged
    data += self.get_temperature()
    data += self.get_pressure()
    data += self.get_humidity()
    data += self.get_orientation()
    data += self.get_mag()
    data += self.get_gyro()

    self.write_data_to_file(data, 'data.csv')

    def log_exception(self, exception):
    sense.set_pixel(1, 0, red)
    self.write_data_to_file([datetime.datetime.now(), str(exception)], 'log.csv')
    sense.set_pixel(0, 0, black)

    def get_accelleration(self):
    try:
    acceleration = self.sense.get_accelerometer_raw()
    except Exception as e:
    self.log_exception(e)
    return

    acc_x = acceleration['x']
    acc_y = acceleration['y']
    acc_z = acceleration['z']

    return[datetime.datetime.now(), acc_x, acc_y, acc_z]

    def get_temperature(self):
    try:
    temperature1 = sense.get_temperature()
    temperature2 = sense.get_temperature_from_humidity()
    temperature3 = sense.get_temperature_from_pressure()
    except Exception as e:
    return [None, None, None]
    return [temperature1, temperature2, temperature3]

    def get_pressure(self):
    try:
    pressure = sense.get_pressure()
    except Exception as e:
    return [None]
    return [pressure]

    def get_humidity(self):
    try:
    humidty = sense.get_humidity()
    except Exception as e:
    return [None]
    return [humidty]

    def get_orientation(self):
    try:
    orientation = sense.get_orientation()
    except Exception as e:
    return [None, None, None]
    return [orientation["yaw"], orientation["pitch"], orientation["roll"]]

    def get_mag(self):
    try:
    mag = sense.get_compass_raw()
    except Exception as e:
    return [None, None, None]
    return [mag["x"], mag["y"], mag["z"]]

    def get_gyro(self):
    try:
    gyro = sense.get_gyroscope_raw()
    except Exception as e:
    return [None, None, None]
    return [gyro["x"], gyro["y"], gyro["z"]]


    if __name__ == '__main__':
    data_logger = DataLogger(init_csv_files=True)
    try:
    data_logger.start_logging()
    except Exception as e:
    data_logger.log_exception(e)


    Further steps for improvements:




    • Catch specific exceptions (e.g. IOErrors in the write csv, or SenseHat specific exceptions

    • Log exceptions (where needed) and return different defaults in cases of error

    • Refactor the write to - as you suggested - log the data in memory and only write every 10th entry to the csv. Attention: If you only log every 10th or even every 100th data entry and the python interpreter crashes, the recently logged data will be lost

    • Don't write the csv headers in code, but manually prepare the csv files and put them next to the script

    • Use a sqlite database and log the data here instead of in CSVs


    In order to figure out where to start with the optimizations, you can now profile the helper methods (write_data_to_file, get_temperature and the other get_... methods) and derive appropriate measurements to take.



    PS. Fair warning: I never executed the code in a python shell, so it may not be free from bugs :see_no_evil:.






    share|improve this answer










    New contributor




    Kim is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
    Check out our Code of Conduct.






    $endgroup$













    • $begingroup$
      In particular, batteries tend to perform poorly when cold, and need to be tested specifically for that.
      $endgroup$
      – 200_success
      6 hours ago



















    5












    $begingroup$

    Opening and closing files takes resources:



    with open('babar.txt', 'a') as f: f.write('a'*10000)


    takes 300 micro-seconds while:



    for _ in range(10000):
    with open('babar.txt', 'a') as f: f.write('a')


    takes 648000 micro-seconds



    So to answer your question Would it be beneficial to write 10 rows at once instead of writing one row at a time?. The answer, as always is YES, but...



    You shouldn't implement a buffer yourself instead use the third argument of open:



    f = open('babar.txt', 'a', 500)
    for _ in range(10000):
    f.write('a')
    f.close()
    # takes 2200 micro-seconds for a 500 buffer
    # and 3660 micro-seconds for a 50 buffer


    It is the buffer-size (4096 chars by default I think). Put the close() in a finally block to avoid corruption of your files.



    I think less opening and closing would take a lot less resources but implementing a buffer yourself is less safe then letting the built-in function handle it for you. Beware of the risks you take, not writing data mean your data is lost if power goes down, and as you can see dividing the buffer by 10 doesn't necessarily mean it takes 10x more resources.



    note: battery consumption is hard to measure and is not directly related to cpu time.






    share|improve this answer








    New contributor




    Benoît Pilatte is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
    Check out our Code of Conduct.






    $endgroup$





















      1












      $begingroup$

      For a system that will be in stratosphere, you dont need any color or light mechanism. Get rid of all code about visualisation, e.g. Setting colors or setting light value. That way light also wont consume your battery. Once your code is cleared, apply other answers.






      share|improve this answer









      $endgroup$













      • $begingroup$
        Absolutely. And do the same with the hardware. Go for the bare minimum, there are stripped versions of the Raspberry Pi available.
        $endgroup$
        – Mast
        5 hours ago











      Your Answer





      StackExchange.ifUsing("editor", function () {
      return StackExchange.using("mathjaxEditing", function () {
      StackExchange.MarkdownEditor.creationCallbacks.add(function (editor, postfix) {
      StackExchange.mathjaxEditing.prepareWmdForMathJax(editor, postfix, [["\$", "\$"]]);
      });
      });
      }, "mathjax-editing");

      StackExchange.ifUsing("editor", function () {
      StackExchange.using("externalEditor", function () {
      StackExchange.using("snippets", function () {
      StackExchange.snippets.init();
      });
      });
      }, "code-snippets");

      StackExchange.ready(function() {
      var channelOptions = {
      tags: "".split(" "),
      id: "196"
      };
      initTagRenderer("".split(" "), "".split(" "), channelOptions);

      StackExchange.using("externalEditor", function() {
      // Have to fire editor after snippets, if snippets enabled
      if (StackExchange.settings.snippets.snippetsEnabled) {
      StackExchange.using("snippets", function() {
      createEditor();
      });
      }
      else {
      createEditor();
      }
      });

      function createEditor() {
      StackExchange.prepareEditor({
      heartbeatType: 'answer',
      autoActivateHeartbeat: false,
      convertImagesToLinks: false,
      noModals: true,
      showLowRepImageUploadWarning: true,
      reputationToPostImages: null,
      bindNavPrevention: true,
      postfix: "",
      imageUploader: {
      brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
      contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
      allowUrls: true
      },
      onDemand: true,
      discardSelector: ".discard-answer"
      ,immediatelyShowMarkdownHelp:true
      });


      }
      });






      Lexu is a new contributor. Be nice, and check out our Code of Conduct.










      draft saved

      draft discarded


















      StackExchange.ready(
      function () {
      StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fcodereview.stackexchange.com%2fquestions%2f214112%2fsensor-logger-for-raspberry-pi-in-a-stratospheric-probe%23new-answer', 'question_page');
      }
      );

      Post as a guest















      Required, but never shown

























      4 Answers
      4






      active

      oldest

      votes








      4 Answers
      4






      active

      oldest

      votes









      active

      oldest

      votes






      active

      oldest

      votes









      14












      $begingroup$



      • Do not call main recursively. You are setting yourself up for stack overflow. Consider instead



        def main():
        while True:
        try:
        your_logic_here
        except Exception as e:
        your_logging_here



      • Testing for counter == 4 is better done in a loop:



            for _ in range(4):
        handle_acceleration
        handle_the_rest


      • An unattended controller should handle exceptions more diligently. For sure, you want to act differently for the exceptions raised by sense (if any) vs exceptions raised by writing to the file.


      • Regarding battery, avoid binary-to-text conversions. Store your data as binary, and convert them to CSV offline, after the Pi safely returns.







      share|improve this answer









      $endgroup$


















        14












        $begingroup$



        • Do not call main recursively. You are setting yourself up for stack overflow. Consider instead



          def main():
          while True:
          try:
          your_logic_here
          except Exception as e:
          your_logging_here



        • Testing for counter == 4 is better done in a loop:



              for _ in range(4):
          handle_acceleration
          handle_the_rest


        • An unattended controller should handle exceptions more diligently. For sure, you want to act differently for the exceptions raised by sense (if any) vs exceptions raised by writing to the file.


        • Regarding battery, avoid binary-to-text conversions. Store your data as binary, and convert them to CSV offline, after the Pi safely returns.







        share|improve this answer









        $endgroup$
















          14












          14








          14





          $begingroup$



          • Do not call main recursively. You are setting yourself up for stack overflow. Consider instead



            def main():
            while True:
            try:
            your_logic_here
            except Exception as e:
            your_logging_here



          • Testing for counter == 4 is better done in a loop:



                for _ in range(4):
            handle_acceleration
            handle_the_rest


          • An unattended controller should handle exceptions more diligently. For sure, you want to act differently for the exceptions raised by sense (if any) vs exceptions raised by writing to the file.


          • Regarding battery, avoid binary-to-text conversions. Store your data as binary, and convert them to CSV offline, after the Pi safely returns.







          share|improve this answer









          $endgroup$





          • Do not call main recursively. You are setting yourself up for stack overflow. Consider instead



            def main():
            while True:
            try:
            your_logic_here
            except Exception as e:
            your_logging_here



          • Testing for counter == 4 is better done in a loop:



                for _ in range(4):
            handle_acceleration
            handle_the_rest


          • An unattended controller should handle exceptions more diligently. For sure, you want to act differently for the exceptions raised by sense (if any) vs exceptions raised by writing to the file.


          • Regarding battery, avoid binary-to-text conversions. Store your data as binary, and convert them to CSV offline, after the Pi safely returns.








          share|improve this answer












          share|improve this answer



          share|improve this answer










          answered yesterday









          vnpvnp

          39.9k232102




          39.9k232102

























              8












              $begingroup$

              Have you already executed the code to see how it performs and if the battery will last? There is that famous Donald Knuth quote saying premature optimization is the root of all evil (or at least most of it) in programming.



              I never had to think about the energy consumption of a program, so I cannot tell you about the power efficieny. But as vnp already did, I can also share my opinion about the code structure to help you to identify bottlenecks more easily. Also, a different structure should help you to still log some data even in case of exceptions.



              Here is what struck me on first read:




              • most of the code is defined in the main method

              • you overwrite the complete data files at the beginning of the program

              • very broad exception clause

              • repetition of the csv write (violates the zen of python - not dry - dont repeat yourself)


              I tried to resolve some of the issues and refactored the structure of the code:



              #!/usr/bin/env python3

              from sense_hat import SenseHat
              import time
              import csv
              import datetime

              # defined constants on moduel level and capitalized the names (pep8: https://www.python.org/dev/peps/pep-0008/#constants)
              RED = (255, 0, 0)
              GREEN = (0, 255, 0)
              BLACK = (0,0,0)


              class DataLogger(object):

              def __init__(self, init_csv_files=False):
              # initalize the commonly ued sensor
              self.sense = SenseHat()
              self.sense.clear()
              self.sense.set_imu_config(True, True, True)
              self.sense.low_light = True

              # only initialize the csv files, if intended
              # I would suggest not to init them in the same program though.
              # If - for some reasons - the python interpreter crashes and the script is restarted,
              # the init of the csv_files will overwrite all the data which was logged so far.
              if init_csv_files:
              self.init_csv_files()

              def write_data_to_file(self, data, file_name, mode='a', delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL):
              """
              Helper method to write the given data to a csv file. Using 'append' as default mode to avoid accidental overwrites.
              """
              with open(file_name, mode=mode) as file:
              writer = csv.writer(file, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
              writer.writerow(data)

              def init_csv_files(self):
              # see comment in init method
              data_headings = ['Zeit','Temperatur1', 'Temperatur2', 'Temperatur3', 'Luftdruck', 'Luftfeuchtigkeit', 'Yaw', 'Pitch', 'Roll', 'Compass X', 'Compass Y', 'Compass Z', 'Gyro X', 'Gyro Y', 'Gyro Z']
              self.write_data_to_file(data_headings, 'data.csv', 'w')

              acc_headings = ['Zeit','Acc_X','Acc_Y','Acc_Z']
              self.write_data_to_file(acc_headings, 'acc.csv', 'w')

              log_headings = ['Zeit','Fehler']
              self.write_data_to_file(log_headings, 'log.csv', 'w')

              def start_logging(self):
              # actual execution
              sense.set_pixel(0, 0, BLACK)
              counter = 0

              while True:
              # moved the accelleration logging to a different method
              # and catched possible exceptions there, so the counter will still be increase
              # and the rest of the data may still be logged even if the accelleration data
              # always raises exceptions
              self.log_accelleration()
              time.sleep(.250)
              counter += 1

              # using counter % 4 == 0 instead of counter == 4
              # this will evaluate to true for every number divisible by 4
              # If you do the strict comparision, you could find yourself in the scenario
              # where the data logging is never executed, if the counter is larger than 4
              # (in this case this is very unlikely, but in threaded scenarios it would be possible,
              # so doing modulo 4 is more defensive)
              if(counter % 4 == 0):
              self.log_data()
              counter = 0

              def log_accelleration(self):
              acceleration_data = get_accelleration()
              if acceleration_data:
              try:
              self.write_data_to_file(acceleration_data, 'acc.csv')
              except Exception as e:
              self.log_exception(e)
              pass
              else:
              # no exception occurred
              self.sense.set_pixel(0, 0, green)
              time.sleep(.05)
              finally:
              self.sense.set_pixel(0, 0, black)

              def log_data(self):
              # saving datetime first, before reading all the sensor data
              data = [datetime.datetime.now()]

              # moved each of the calls to sense in a separate method
              # exceptions will lead to empty entries being logged but
              # if e.g. get_pressure raises an exceptions, the other data may still get logged
              data += self.get_temperature()
              data += self.get_pressure()
              data += self.get_humidity()
              data += self.get_orientation()
              data += self.get_mag()
              data += self.get_gyro()

              self.write_data_to_file(data, 'data.csv')

              def log_exception(self, exception):
              sense.set_pixel(1, 0, red)
              self.write_data_to_file([datetime.datetime.now(), str(exception)], 'log.csv')
              sense.set_pixel(0, 0, black)

              def get_accelleration(self):
              try:
              acceleration = self.sense.get_accelerometer_raw()
              except Exception as e:
              self.log_exception(e)
              return

              acc_x = acceleration['x']
              acc_y = acceleration['y']
              acc_z = acceleration['z']

              return[datetime.datetime.now(), acc_x, acc_y, acc_z]

              def get_temperature(self):
              try:
              temperature1 = sense.get_temperature()
              temperature2 = sense.get_temperature_from_humidity()
              temperature3 = sense.get_temperature_from_pressure()
              except Exception as e:
              return [None, None, None]
              return [temperature1, temperature2, temperature3]

              def get_pressure(self):
              try:
              pressure = sense.get_pressure()
              except Exception as e:
              return [None]
              return [pressure]

              def get_humidity(self):
              try:
              humidty = sense.get_humidity()
              except Exception as e:
              return [None]
              return [humidty]

              def get_orientation(self):
              try:
              orientation = sense.get_orientation()
              except Exception as e:
              return [None, None, None]
              return [orientation["yaw"], orientation["pitch"], orientation["roll"]]

              def get_mag(self):
              try:
              mag = sense.get_compass_raw()
              except Exception as e:
              return [None, None, None]
              return [mag["x"], mag["y"], mag["z"]]

              def get_gyro(self):
              try:
              gyro = sense.get_gyroscope_raw()
              except Exception as e:
              return [None, None, None]
              return [gyro["x"], gyro["y"], gyro["z"]]


              if __name__ == '__main__':
              data_logger = DataLogger(init_csv_files=True)
              try:
              data_logger.start_logging()
              except Exception as e:
              data_logger.log_exception(e)


              Further steps for improvements:




              • Catch specific exceptions (e.g. IOErrors in the write csv, or SenseHat specific exceptions

              • Log exceptions (where needed) and return different defaults in cases of error

              • Refactor the write to - as you suggested - log the data in memory and only write every 10th entry to the csv. Attention: If you only log every 10th or even every 100th data entry and the python interpreter crashes, the recently logged data will be lost

              • Don't write the csv headers in code, but manually prepare the csv files and put them next to the script

              • Use a sqlite database and log the data here instead of in CSVs


              In order to figure out where to start with the optimizations, you can now profile the helper methods (write_data_to_file, get_temperature and the other get_... methods) and derive appropriate measurements to take.



              PS. Fair warning: I never executed the code in a python shell, so it may not be free from bugs :see_no_evil:.






              share|improve this answer










              New contributor




              Kim is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
              Check out our Code of Conduct.






              $endgroup$













              • $begingroup$
                In particular, batteries tend to perform poorly when cold, and need to be tested specifically for that.
                $endgroup$
                – 200_success
                6 hours ago
















              8












              $begingroup$

              Have you already executed the code to see how it performs and if the battery will last? There is that famous Donald Knuth quote saying premature optimization is the root of all evil (or at least most of it) in programming.



              I never had to think about the energy consumption of a program, so I cannot tell you about the power efficieny. But as vnp already did, I can also share my opinion about the code structure to help you to identify bottlenecks more easily. Also, a different structure should help you to still log some data even in case of exceptions.



              Here is what struck me on first read:




              • most of the code is defined in the main method

              • you overwrite the complete data files at the beginning of the program

              • very broad exception clause

              • repetition of the csv write (violates the zen of python - not dry - dont repeat yourself)


              I tried to resolve some of the issues and refactored the structure of the code:



              #!/usr/bin/env python3

              from sense_hat import SenseHat
              import time
              import csv
              import datetime

              # defined constants on moduel level and capitalized the names (pep8: https://www.python.org/dev/peps/pep-0008/#constants)
              RED = (255, 0, 0)
              GREEN = (0, 255, 0)
              BLACK = (0,0,0)


              class DataLogger(object):

              def __init__(self, init_csv_files=False):
              # initalize the commonly ued sensor
              self.sense = SenseHat()
              self.sense.clear()
              self.sense.set_imu_config(True, True, True)
              self.sense.low_light = True

              # only initialize the csv files, if intended
              # I would suggest not to init them in the same program though.
              # If - for some reasons - the python interpreter crashes and the script is restarted,
              # the init of the csv_files will overwrite all the data which was logged so far.
              if init_csv_files:
              self.init_csv_files()

              def write_data_to_file(self, data, file_name, mode='a', delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL):
              """
              Helper method to write the given data to a csv file. Using 'append' as default mode to avoid accidental overwrites.
              """
              with open(file_name, mode=mode) as file:
              writer = csv.writer(file, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
              writer.writerow(data)

              def init_csv_files(self):
              # see comment in init method
              data_headings = ['Zeit','Temperatur1', 'Temperatur2', 'Temperatur3', 'Luftdruck', 'Luftfeuchtigkeit', 'Yaw', 'Pitch', 'Roll', 'Compass X', 'Compass Y', 'Compass Z', 'Gyro X', 'Gyro Y', 'Gyro Z']
              self.write_data_to_file(data_headings, 'data.csv', 'w')

              acc_headings = ['Zeit','Acc_X','Acc_Y','Acc_Z']
              self.write_data_to_file(acc_headings, 'acc.csv', 'w')

              log_headings = ['Zeit','Fehler']
              self.write_data_to_file(log_headings, 'log.csv', 'w')

              def start_logging(self):
              # actual execution
              sense.set_pixel(0, 0, BLACK)
              counter = 0

              while True:
              # moved the accelleration logging to a different method
              # and catched possible exceptions there, so the counter will still be increase
              # and the rest of the data may still be logged even if the accelleration data
              # always raises exceptions
              self.log_accelleration()
              time.sleep(.250)
              counter += 1

              # using counter % 4 == 0 instead of counter == 4
              # this will evaluate to true for every number divisible by 4
              # If you do the strict comparision, you could find yourself in the scenario
              # where the data logging is never executed, if the counter is larger than 4
              # (in this case this is very unlikely, but in threaded scenarios it would be possible,
              # so doing modulo 4 is more defensive)
              if(counter % 4 == 0):
              self.log_data()
              counter = 0

              def log_accelleration(self):
              acceleration_data = get_accelleration()
              if acceleration_data:
              try:
              self.write_data_to_file(acceleration_data, 'acc.csv')
              except Exception as e:
              self.log_exception(e)
              pass
              else:
              # no exception occurred
              self.sense.set_pixel(0, 0, green)
              time.sleep(.05)
              finally:
              self.sense.set_pixel(0, 0, black)

              def log_data(self):
              # saving datetime first, before reading all the sensor data
              data = [datetime.datetime.now()]

              # moved each of the calls to sense in a separate method
              # exceptions will lead to empty entries being logged but
              # if e.g. get_pressure raises an exceptions, the other data may still get logged
              data += self.get_temperature()
              data += self.get_pressure()
              data += self.get_humidity()
              data += self.get_orientation()
              data += self.get_mag()
              data += self.get_gyro()

              self.write_data_to_file(data, 'data.csv')

              def log_exception(self, exception):
              sense.set_pixel(1, 0, red)
              self.write_data_to_file([datetime.datetime.now(), str(exception)], 'log.csv')
              sense.set_pixel(0, 0, black)

              def get_accelleration(self):
              try:
              acceleration = self.sense.get_accelerometer_raw()
              except Exception as e:
              self.log_exception(e)
              return

              acc_x = acceleration['x']
              acc_y = acceleration['y']
              acc_z = acceleration['z']

              return[datetime.datetime.now(), acc_x, acc_y, acc_z]

              def get_temperature(self):
              try:
              temperature1 = sense.get_temperature()
              temperature2 = sense.get_temperature_from_humidity()
              temperature3 = sense.get_temperature_from_pressure()
              except Exception as e:
              return [None, None, None]
              return [temperature1, temperature2, temperature3]

              def get_pressure(self):
              try:
              pressure = sense.get_pressure()
              except Exception as e:
              return [None]
              return [pressure]

              def get_humidity(self):
              try:
              humidty = sense.get_humidity()
              except Exception as e:
              return [None]
              return [humidty]

              def get_orientation(self):
              try:
              orientation = sense.get_orientation()
              except Exception as e:
              return [None, None, None]
              return [orientation["yaw"], orientation["pitch"], orientation["roll"]]

              def get_mag(self):
              try:
              mag = sense.get_compass_raw()
              except Exception as e:
              return [None, None, None]
              return [mag["x"], mag["y"], mag["z"]]

              def get_gyro(self):
              try:
              gyro = sense.get_gyroscope_raw()
              except Exception as e:
              return [None, None, None]
              return [gyro["x"], gyro["y"], gyro["z"]]


              if __name__ == '__main__':
              data_logger = DataLogger(init_csv_files=True)
              try:
              data_logger.start_logging()
              except Exception as e:
              data_logger.log_exception(e)


              Further steps for improvements:




              • Catch specific exceptions (e.g. IOErrors in the write csv, or SenseHat specific exceptions

              • Log exceptions (where needed) and return different defaults in cases of error

              • Refactor the write to - as you suggested - log the data in memory and only write every 10th entry to the csv. Attention: If you only log every 10th or even every 100th data entry and the python interpreter crashes, the recently logged data will be lost

              • Don't write the csv headers in code, but manually prepare the csv files and put them next to the script

              • Use a sqlite database and log the data here instead of in CSVs


              In order to figure out where to start with the optimizations, you can now profile the helper methods (write_data_to_file, get_temperature and the other get_... methods) and derive appropriate measurements to take.



              PS. Fair warning: I never executed the code in a python shell, so it may not be free from bugs :see_no_evil:.






              share|improve this answer










              New contributor




              Kim is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
              Check out our Code of Conduct.






              $endgroup$













              • $begingroup$
                In particular, batteries tend to perform poorly when cold, and need to be tested specifically for that.
                $endgroup$
                – 200_success
                6 hours ago














              8












              8








              8





              $begingroup$

              Have you already executed the code to see how it performs and if the battery will last? There is that famous Donald Knuth quote saying premature optimization is the root of all evil (or at least most of it) in programming.



              I never had to think about the energy consumption of a program, so I cannot tell you about the power efficieny. But as vnp already did, I can also share my opinion about the code structure to help you to identify bottlenecks more easily. Also, a different structure should help you to still log some data even in case of exceptions.



              Here is what struck me on first read:




              • most of the code is defined in the main method

              • you overwrite the complete data files at the beginning of the program

              • very broad exception clause

              • repetition of the csv write (violates the zen of python - not dry - dont repeat yourself)


              I tried to resolve some of the issues and refactored the structure of the code:



              #!/usr/bin/env python3

              from sense_hat import SenseHat
              import time
              import csv
              import datetime

              # defined constants on moduel level and capitalized the names (pep8: https://www.python.org/dev/peps/pep-0008/#constants)
              RED = (255, 0, 0)
              GREEN = (0, 255, 0)
              BLACK = (0,0,0)


              class DataLogger(object):

              def __init__(self, init_csv_files=False):
              # initalize the commonly ued sensor
              self.sense = SenseHat()
              self.sense.clear()
              self.sense.set_imu_config(True, True, True)
              self.sense.low_light = True

              # only initialize the csv files, if intended
              # I would suggest not to init them in the same program though.
              # If - for some reasons - the python interpreter crashes and the script is restarted,
              # the init of the csv_files will overwrite all the data which was logged so far.
              if init_csv_files:
              self.init_csv_files()

              def write_data_to_file(self, data, file_name, mode='a', delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL):
              """
              Helper method to write the given data to a csv file. Using 'append' as default mode to avoid accidental overwrites.
              """
              with open(file_name, mode=mode) as file:
              writer = csv.writer(file, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
              writer.writerow(data)

              def init_csv_files(self):
              # see comment in init method
              data_headings = ['Zeit','Temperatur1', 'Temperatur2', 'Temperatur3', 'Luftdruck', 'Luftfeuchtigkeit', 'Yaw', 'Pitch', 'Roll', 'Compass X', 'Compass Y', 'Compass Z', 'Gyro X', 'Gyro Y', 'Gyro Z']
              self.write_data_to_file(data_headings, 'data.csv', 'w')

              acc_headings = ['Zeit','Acc_X','Acc_Y','Acc_Z']
              self.write_data_to_file(acc_headings, 'acc.csv', 'w')

              log_headings = ['Zeit','Fehler']
              self.write_data_to_file(log_headings, 'log.csv', 'w')

              def start_logging(self):
              # actual execution
              sense.set_pixel(0, 0, BLACK)
              counter = 0

              while True:
              # moved the accelleration logging to a different method
              # and catched possible exceptions there, so the counter will still be increase
              # and the rest of the data may still be logged even if the accelleration data
              # always raises exceptions
              self.log_accelleration()
              time.sleep(.250)
              counter += 1

              # using counter % 4 == 0 instead of counter == 4
              # this will evaluate to true for every number divisible by 4
              # If you do the strict comparision, you could find yourself in the scenario
              # where the data logging is never executed, if the counter is larger than 4
              # (in this case this is very unlikely, but in threaded scenarios it would be possible,
              # so doing modulo 4 is more defensive)
              if(counter % 4 == 0):
              self.log_data()
              counter = 0

              def log_accelleration(self):
              acceleration_data = get_accelleration()
              if acceleration_data:
              try:
              self.write_data_to_file(acceleration_data, 'acc.csv')
              except Exception as e:
              self.log_exception(e)
              pass
              else:
              # no exception occurred
              self.sense.set_pixel(0, 0, green)
              time.sleep(.05)
              finally:
              self.sense.set_pixel(0, 0, black)

              def log_data(self):
              # saving datetime first, before reading all the sensor data
              data = [datetime.datetime.now()]

              # moved each of the calls to sense in a separate method
              # exceptions will lead to empty entries being logged but
              # if e.g. get_pressure raises an exceptions, the other data may still get logged
              data += self.get_temperature()
              data += self.get_pressure()
              data += self.get_humidity()
              data += self.get_orientation()
              data += self.get_mag()
              data += self.get_gyro()

              self.write_data_to_file(data, 'data.csv')

              def log_exception(self, exception):
              sense.set_pixel(1, 0, red)
              self.write_data_to_file([datetime.datetime.now(), str(exception)], 'log.csv')
              sense.set_pixel(0, 0, black)

              def get_accelleration(self):
              try:
              acceleration = self.sense.get_accelerometer_raw()
              except Exception as e:
              self.log_exception(e)
              return

              acc_x = acceleration['x']
              acc_y = acceleration['y']
              acc_z = acceleration['z']

              return[datetime.datetime.now(), acc_x, acc_y, acc_z]

              def get_temperature(self):
              try:
              temperature1 = sense.get_temperature()
              temperature2 = sense.get_temperature_from_humidity()
              temperature3 = sense.get_temperature_from_pressure()
              except Exception as e:
              return [None, None, None]
              return [temperature1, temperature2, temperature3]

              def get_pressure(self):
              try:
              pressure = sense.get_pressure()
              except Exception as e:
              return [None]
              return [pressure]

              def get_humidity(self):
              try:
              humidty = sense.get_humidity()
              except Exception as e:
              return [None]
              return [humidty]

              def get_orientation(self):
              try:
              orientation = sense.get_orientation()
              except Exception as e:
              return [None, None, None]
              return [orientation["yaw"], orientation["pitch"], orientation["roll"]]

              def get_mag(self):
              try:
              mag = sense.get_compass_raw()
              except Exception as e:
              return [None, None, None]
              return [mag["x"], mag["y"], mag["z"]]

              def get_gyro(self):
              try:
              gyro = sense.get_gyroscope_raw()
              except Exception as e:
              return [None, None, None]
              return [gyro["x"], gyro["y"], gyro["z"]]


              if __name__ == '__main__':
              data_logger = DataLogger(init_csv_files=True)
              try:
              data_logger.start_logging()
              except Exception as e:
              data_logger.log_exception(e)


              Further steps for improvements:




              • Catch specific exceptions (e.g. IOErrors in the write csv, or SenseHat specific exceptions

              • Log exceptions (where needed) and return different defaults in cases of error

              • Refactor the write to - as you suggested - log the data in memory and only write every 10th entry to the csv. Attention: If you only log every 10th or even every 100th data entry and the python interpreter crashes, the recently logged data will be lost

              • Don't write the csv headers in code, but manually prepare the csv files and put them next to the script

              • Use a sqlite database and log the data here instead of in CSVs


              In order to figure out where to start with the optimizations, you can now profile the helper methods (write_data_to_file, get_temperature and the other get_... methods) and derive appropriate measurements to take.



              PS. Fair warning: I never executed the code in a python shell, so it may not be free from bugs :see_no_evil:.






              share|improve this answer










              New contributor




              Kim is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
              Check out our Code of Conduct.






              $endgroup$



              Have you already executed the code to see how it performs and if the battery will last? There is that famous Donald Knuth quote saying premature optimization is the root of all evil (or at least most of it) in programming.



              I never had to think about the energy consumption of a program, so I cannot tell you about the power efficieny. But as vnp already did, I can also share my opinion about the code structure to help you to identify bottlenecks more easily. Also, a different structure should help you to still log some data even in case of exceptions.



              Here is what struck me on first read:




              • most of the code is defined in the main method

              • you overwrite the complete data files at the beginning of the program

              • very broad exception clause

              • repetition of the csv write (violates the zen of python - not dry - dont repeat yourself)


              I tried to resolve some of the issues and refactored the structure of the code:



              #!/usr/bin/env python3

              from sense_hat import SenseHat
              import time
              import csv
              import datetime

              # defined constants on moduel level and capitalized the names (pep8: https://www.python.org/dev/peps/pep-0008/#constants)
              RED = (255, 0, 0)
              GREEN = (0, 255, 0)
              BLACK = (0,0,0)


              class DataLogger(object):

              def __init__(self, init_csv_files=False):
              # initalize the commonly ued sensor
              self.sense = SenseHat()
              self.sense.clear()
              self.sense.set_imu_config(True, True, True)
              self.sense.low_light = True

              # only initialize the csv files, if intended
              # I would suggest not to init them in the same program though.
              # If - for some reasons - the python interpreter crashes and the script is restarted,
              # the init of the csv_files will overwrite all the data which was logged so far.
              if init_csv_files:
              self.init_csv_files()

              def write_data_to_file(self, data, file_name, mode='a', delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL):
              """
              Helper method to write the given data to a csv file. Using 'append' as default mode to avoid accidental overwrites.
              """
              with open(file_name, mode=mode) as file:
              writer = csv.writer(file, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
              writer.writerow(data)

              def init_csv_files(self):
              # see comment in init method
              data_headings = ['Zeit','Temperatur1', 'Temperatur2', 'Temperatur3', 'Luftdruck', 'Luftfeuchtigkeit', 'Yaw', 'Pitch', 'Roll', 'Compass X', 'Compass Y', 'Compass Z', 'Gyro X', 'Gyro Y', 'Gyro Z']
              self.write_data_to_file(data_headings, 'data.csv', 'w')

              acc_headings = ['Zeit','Acc_X','Acc_Y','Acc_Z']
              self.write_data_to_file(acc_headings, 'acc.csv', 'w')

              log_headings = ['Zeit','Fehler']
              self.write_data_to_file(log_headings, 'log.csv', 'w')

              def start_logging(self):
              # actual execution
              sense.set_pixel(0, 0, BLACK)
              counter = 0

              while True:
              # moved the accelleration logging to a different method
              # and catched possible exceptions there, so the counter will still be increase
              # and the rest of the data may still be logged even if the accelleration data
              # always raises exceptions
              self.log_accelleration()
              time.sleep(.250)
              counter += 1

              # using counter % 4 == 0 instead of counter == 4
              # this will evaluate to true for every number divisible by 4
              # If you do the strict comparision, you could find yourself in the scenario
              # where the data logging is never executed, if the counter is larger than 4
              # (in this case this is very unlikely, but in threaded scenarios it would be possible,
              # so doing modulo 4 is more defensive)
              if(counter % 4 == 0):
              self.log_data()
              counter = 0

              def log_accelleration(self):
              acceleration_data = get_accelleration()
              if acceleration_data:
              try:
              self.write_data_to_file(acceleration_data, 'acc.csv')
              except Exception as e:
              self.log_exception(e)
              pass
              else:
              # no exception occurred
              self.sense.set_pixel(0, 0, green)
              time.sleep(.05)
              finally:
              self.sense.set_pixel(0, 0, black)

              def log_data(self):
              # saving datetime first, before reading all the sensor data
              data = [datetime.datetime.now()]

              # moved each of the calls to sense in a separate method
              # exceptions will lead to empty entries being logged but
              # if e.g. get_pressure raises an exceptions, the other data may still get logged
              data += self.get_temperature()
              data += self.get_pressure()
              data += self.get_humidity()
              data += self.get_orientation()
              data += self.get_mag()
              data += self.get_gyro()

              self.write_data_to_file(data, 'data.csv')

              def log_exception(self, exception):
              sense.set_pixel(1, 0, red)
              self.write_data_to_file([datetime.datetime.now(), str(exception)], 'log.csv')
              sense.set_pixel(0, 0, black)

              def get_accelleration(self):
              try:
              acceleration = self.sense.get_accelerometer_raw()
              except Exception as e:
              self.log_exception(e)
              return

              acc_x = acceleration['x']
              acc_y = acceleration['y']
              acc_z = acceleration['z']

              return[datetime.datetime.now(), acc_x, acc_y, acc_z]

              def get_temperature(self):
              try:
              temperature1 = sense.get_temperature()
              temperature2 = sense.get_temperature_from_humidity()
              temperature3 = sense.get_temperature_from_pressure()
              except Exception as e:
              return [None, None, None]
              return [temperature1, temperature2, temperature3]

              def get_pressure(self):
              try:
              pressure = sense.get_pressure()
              except Exception as e:
              return [None]
              return [pressure]

              def get_humidity(self):
              try:
              humidty = sense.get_humidity()
              except Exception as e:
              return [None]
              return [humidty]

              def get_orientation(self):
              try:
              orientation = sense.get_orientation()
              except Exception as e:
              return [None, None, None]
              return [orientation["yaw"], orientation["pitch"], orientation["roll"]]

              def get_mag(self):
              try:
              mag = sense.get_compass_raw()
              except Exception as e:
              return [None, None, None]
              return [mag["x"], mag["y"], mag["z"]]

              def get_gyro(self):
              try:
              gyro = sense.get_gyroscope_raw()
              except Exception as e:
              return [None, None, None]
              return [gyro["x"], gyro["y"], gyro["z"]]


              if __name__ == '__main__':
              data_logger = DataLogger(init_csv_files=True)
              try:
              data_logger.start_logging()
              except Exception as e:
              data_logger.log_exception(e)


              Further steps for improvements:




              • Catch specific exceptions (e.g. IOErrors in the write csv, or SenseHat specific exceptions

              • Log exceptions (where needed) and return different defaults in cases of error

              • Refactor the write to - as you suggested - log the data in memory and only write every 10th entry to the csv. Attention: If you only log every 10th or even every 100th data entry and the python interpreter crashes, the recently logged data will be lost

              • Don't write the csv headers in code, but manually prepare the csv files and put them next to the script

              • Use a sqlite database and log the data here instead of in CSVs


              In order to figure out where to start with the optimizations, you can now profile the helper methods (write_data_to_file, get_temperature and the other get_... methods) and derive appropriate measurements to take.



              PS. Fair warning: I never executed the code in a python shell, so it may not be free from bugs :see_no_evil:.







              share|improve this answer










              New contributor




              Kim is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
              Check out our Code of Conduct.









              share|improve this answer



              share|improve this answer








              edited yesterday









              Cris Luengo

              2,524319




              2,524319






              New contributor




              Kim is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
              Check out our Code of Conduct.









              answered yesterday









              KimKim

              1812




              1812




              New contributor




              Kim is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
              Check out our Code of Conduct.





              New contributor





              Kim is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
              Check out our Code of Conduct.






              Kim is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
              Check out our Code of Conduct.












              • $begingroup$
                In particular, batteries tend to perform poorly when cold, and need to be tested specifically for that.
                $endgroup$
                – 200_success
                6 hours ago


















              • $begingroup$
                In particular, batteries tend to perform poorly when cold, and need to be tested specifically for that.
                $endgroup$
                – 200_success
                6 hours ago
















              $begingroup$
              In particular, batteries tend to perform poorly when cold, and need to be tested specifically for that.
              $endgroup$
              – 200_success
              6 hours ago




              $begingroup$
              In particular, batteries tend to perform poorly when cold, and need to be tested specifically for that.
              $endgroup$
              – 200_success
              6 hours ago











              5












              $begingroup$

              Opening and closing files takes resources:



              with open('babar.txt', 'a') as f: f.write('a'*10000)


              takes 300 micro-seconds while:



              for _ in range(10000):
              with open('babar.txt', 'a') as f: f.write('a')


              takes 648000 micro-seconds



              So to answer your question Would it be beneficial to write 10 rows at once instead of writing one row at a time?. The answer, as always is YES, but...



              You shouldn't implement a buffer yourself instead use the third argument of open:



              f = open('babar.txt', 'a', 500)
              for _ in range(10000):
              f.write('a')
              f.close()
              # takes 2200 micro-seconds for a 500 buffer
              # and 3660 micro-seconds for a 50 buffer


              It is the buffer-size (4096 chars by default I think). Put the close() in a finally block to avoid corruption of your files.



              I think less opening and closing would take a lot less resources but implementing a buffer yourself is less safe then letting the built-in function handle it for you. Beware of the risks you take, not writing data mean your data is lost if power goes down, and as you can see dividing the buffer by 10 doesn't necessarily mean it takes 10x more resources.



              note: battery consumption is hard to measure and is not directly related to cpu time.






              share|improve this answer








              New contributor




              Benoît Pilatte is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
              Check out our Code of Conduct.






              $endgroup$


















                5












                $begingroup$

                Opening and closing files takes resources:



                with open('babar.txt', 'a') as f: f.write('a'*10000)


                takes 300 micro-seconds while:



                for _ in range(10000):
                with open('babar.txt', 'a') as f: f.write('a')


                takes 648000 micro-seconds



                So to answer your question Would it be beneficial to write 10 rows at once instead of writing one row at a time?. The answer, as always is YES, but...



                You shouldn't implement a buffer yourself instead use the third argument of open:



                f = open('babar.txt', 'a', 500)
                for _ in range(10000):
                f.write('a')
                f.close()
                # takes 2200 micro-seconds for a 500 buffer
                # and 3660 micro-seconds for a 50 buffer


                It is the buffer-size (4096 chars by default I think). Put the close() in a finally block to avoid corruption of your files.



                I think less opening and closing would take a lot less resources but implementing a buffer yourself is less safe then letting the built-in function handle it for you. Beware of the risks you take, not writing data mean your data is lost if power goes down, and as you can see dividing the buffer by 10 doesn't necessarily mean it takes 10x more resources.



                note: battery consumption is hard to measure and is not directly related to cpu time.






                share|improve this answer








                New contributor




                Benoît Pilatte is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
                Check out our Code of Conduct.






                $endgroup$
















                  5












                  5








                  5





                  $begingroup$

                  Opening and closing files takes resources:



                  with open('babar.txt', 'a') as f: f.write('a'*10000)


                  takes 300 micro-seconds while:



                  for _ in range(10000):
                  with open('babar.txt', 'a') as f: f.write('a')


                  takes 648000 micro-seconds



                  So to answer your question Would it be beneficial to write 10 rows at once instead of writing one row at a time?. The answer, as always is YES, but...



                  You shouldn't implement a buffer yourself instead use the third argument of open:



                  f = open('babar.txt', 'a', 500)
                  for _ in range(10000):
                  f.write('a')
                  f.close()
                  # takes 2200 micro-seconds for a 500 buffer
                  # and 3660 micro-seconds for a 50 buffer


                  It is the buffer-size (4096 chars by default I think). Put the close() in a finally block to avoid corruption of your files.



                  I think less opening and closing would take a lot less resources but implementing a buffer yourself is less safe then letting the built-in function handle it for you. Beware of the risks you take, not writing data mean your data is lost if power goes down, and as you can see dividing the buffer by 10 doesn't necessarily mean it takes 10x more resources.



                  note: battery consumption is hard to measure and is not directly related to cpu time.






                  share|improve this answer








                  New contributor




                  Benoît Pilatte is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
                  Check out our Code of Conduct.






                  $endgroup$



                  Opening and closing files takes resources:



                  with open('babar.txt', 'a') as f: f.write('a'*10000)


                  takes 300 micro-seconds while:



                  for _ in range(10000):
                  with open('babar.txt', 'a') as f: f.write('a')


                  takes 648000 micro-seconds



                  So to answer your question Would it be beneficial to write 10 rows at once instead of writing one row at a time?. The answer, as always is YES, but...



                  You shouldn't implement a buffer yourself instead use the third argument of open:



                  f = open('babar.txt', 'a', 500)
                  for _ in range(10000):
                  f.write('a')
                  f.close()
                  # takes 2200 micro-seconds for a 500 buffer
                  # and 3660 micro-seconds for a 50 buffer


                  It is the buffer-size (4096 chars by default I think). Put the close() in a finally block to avoid corruption of your files.



                  I think less opening and closing would take a lot less resources but implementing a buffer yourself is less safe then letting the built-in function handle it for you. Beware of the risks you take, not writing data mean your data is lost if power goes down, and as you can see dividing the buffer by 10 doesn't necessarily mean it takes 10x more resources.



                  note: battery consumption is hard to measure and is not directly related to cpu time.







                  share|improve this answer








                  New contributor




                  Benoît Pilatte is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
                  Check out our Code of Conduct.









                  share|improve this answer



                  share|improve this answer






                  New contributor




                  Benoît Pilatte is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
                  Check out our Code of Conduct.









                  answered yesterday









                  Benoît PilatteBenoît Pilatte

                  2898




                  2898




                  New contributor




                  Benoît Pilatte is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
                  Check out our Code of Conduct.





                  New contributor





                  Benoît Pilatte is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
                  Check out our Code of Conduct.






                  Benoît Pilatte is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
                  Check out our Code of Conduct.























                      1












                      $begingroup$

                      For a system that will be in stratosphere, you dont need any color or light mechanism. Get rid of all code about visualisation, e.g. Setting colors or setting light value. That way light also wont consume your battery. Once your code is cleared, apply other answers.






                      share|improve this answer









                      $endgroup$













                      • $begingroup$
                        Absolutely. And do the same with the hardware. Go for the bare minimum, there are stripped versions of the Raspberry Pi available.
                        $endgroup$
                        – Mast
                        5 hours ago
















                      1












                      $begingroup$

                      For a system that will be in stratosphere, you dont need any color or light mechanism. Get rid of all code about visualisation, e.g. Setting colors or setting light value. That way light also wont consume your battery. Once your code is cleared, apply other answers.






                      share|improve this answer









                      $endgroup$













                      • $begingroup$
                        Absolutely. And do the same with the hardware. Go for the bare minimum, there are stripped versions of the Raspberry Pi available.
                        $endgroup$
                        – Mast
                        5 hours ago














                      1












                      1








                      1





                      $begingroup$

                      For a system that will be in stratosphere, you dont need any color or light mechanism. Get rid of all code about visualisation, e.g. Setting colors or setting light value. That way light also wont consume your battery. Once your code is cleared, apply other answers.






                      share|improve this answer









                      $endgroup$



                      For a system that will be in stratosphere, you dont need any color or light mechanism. Get rid of all code about visualisation, e.g. Setting colors or setting light value. That way light also wont consume your battery. Once your code is cleared, apply other answers.







                      share|improve this answer












                      share|improve this answer



                      share|improve this answer










                      answered 9 hours ago







                      user193727



















                      • $begingroup$
                        Absolutely. And do the same with the hardware. Go for the bare minimum, there are stripped versions of the Raspberry Pi available.
                        $endgroup$
                        – Mast
                        5 hours ago


















                      • $begingroup$
                        Absolutely. And do the same with the hardware. Go for the bare minimum, there are stripped versions of the Raspberry Pi available.
                        $endgroup$
                        – Mast
                        5 hours ago
















                      $begingroup$
                      Absolutely. And do the same with the hardware. Go for the bare minimum, there are stripped versions of the Raspberry Pi available.
                      $endgroup$
                      – Mast
                      5 hours ago




                      $begingroup$
                      Absolutely. And do the same with the hardware. Go for the bare minimum, there are stripped versions of the Raspberry Pi available.
                      $endgroup$
                      – Mast
                      5 hours ago










                      Lexu is a new contributor. Be nice, and check out our Code of Conduct.










                      draft saved

                      draft discarded


















                      Lexu is a new contributor. Be nice, and check out our Code of Conduct.













                      Lexu is a new contributor. Be nice, and check out our Code of Conduct.












                      Lexu is a new contributor. Be nice, and check out our Code of Conduct.
















                      Thanks for contributing an answer to Code Review Stack Exchange!


                      • Please be sure to answer the question. Provide details and share your research!

                      But avoid



                      • Asking for help, clarification, or responding to other answers.

                      • Making statements based on opinion; back them up with references or personal experience.


                      Use MathJax to format equations. MathJax reference.


                      To learn more, see our tips on writing great answers.




                      draft saved


                      draft discarded














                      StackExchange.ready(
                      function () {
                      StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fcodereview.stackexchange.com%2fquestions%2f214112%2fsensor-logger-for-raspberry-pi-in-a-stratospheric-probe%23new-answer', 'question_page');
                      }
                      );

                      Post as a guest















                      Required, but never shown





















































                      Required, but never shown














                      Required, but never shown












                      Required, but never shown







                      Required, but never shown

































                      Required, but never shown














                      Required, but never shown












                      Required, but never shown







                      Required, but never shown







                      Popular posts from this blog

                      How to label and detect the document text images

                      Vallis Paradisi

                      Tabula Rosettana