S0ImpulseCounter/S0Meters_py/s0meters.py

#!/usr/bin/python3 -u
#
import serial
#from time import sleep
import time
import os
import sys
import paho.mqtt.client as mqtt
import json
import yaml
from yaml.constructor import ConstructorError
from threading import Thread
from influxdb import InfluxDBClient
import configparser
#from datetime import datetime
import datetime

# Change working dir to the same dir as this script
os.chdir(sys.path[0])

config = configparser.ConfigParser()
config.read('s0meters.ini')

quiet = False
verbose = False
debug = False

if len(sys.argv) >= 2:
    if sys.argv[1] == "-q":
        verbose = False
        quiet = True
        debug = False
    elif sys.argv[1] == "-v":
        verbose = True
        quiet = False
        debug = False
        print("VERBOSE=ON")
    elif sys.argv[1] == "-d":
        verbose = True
        quiet = False
        debug = True
        print("DEBUG=ON")


consoleAddTimestamp = config['main'].get('consoleAddTimestamp')


conf_storage_path = config['filelog'].get('storage_path')
if conf_storage_path[-1:] != '/':
    conf_storage_path += '/'


ser = serial.Serial(port=config['hardware'].get('serialPort'),
    baudrate = config['hardware'].get('serialBaud'),
    parity=serial.PARITY_NONE,
    stopbits=serial.STOPBITS_ONE,
    bytesize=serial.EIGHTBITS,
    timeout=config['hardware'].getint('serialTout'))



if not quiet:
    print("S0MetersLog by Flo Kra")
    print("=======================================================================")

meters_yaml = yaml.load(open(config['main'].get('meters_config_yml')), Loader=yaml.SafeLoader)
if not quiet:
    print("Meters configuration:")
    print(json.dumps(meters_yaml, indent=2))
    
    print("loading InfluxDB configuration...")
    
influxdb_yaml = yaml.load(open(config['main'].get('influx_config_yml')), Loader=yaml.SafeLoader)

if not quiet:
    print("InfluxDB Instances:")
    print(json.dumps(influxdb_yaml, indent=4))

influxclient = dict()
for instance in influxdb_yaml:
    i_host = influxdb_yaml[instance].get('host', None)
    i_port = int(influxdb_yaml[instance].get('port', 8086))
    i_username = influxdb_yaml[instance].get('username', None)
    i_password = influxdb_yaml[instance].get('password', None)
    i_database = influxdb_yaml[instance].get('database', None)
    if i_host != None and i_database != None:
        if i_username != None and i_password != None:
            influxclient[instance] = InfluxDBClient(i_host, i_port, i_username, i_password, i_database)
        else:
            influxclient[instance] = InfluxDBClient(i_host, i_port, i_database)


data_energy = dict()
data_momentary = dict()
saved_today_date = dict()
influxdb_energy_lastWriteTime = dict()
energy_history_lastPublished = dict()

saved_energy_today_min = dict()
saved_energy_yesterday_total = dict()

for meter in meters_yaml:
    data_energy[meter] = dict()
    data_momentary[meter] = dict()


MQTTenabled = config['mqtt'].getboolean('enable', False)

if MQTTenabled:
    def on_connect(client, userdata, flags, rc):
        if not quiet:
            print("MQTT: connected with result code " + str(rc))
        client.subscribe(config['mqtt'].get('topic_cmd'))
        if not quiet:
            print("MQTT: subscribed topic:", config['mqtt'].get('topic_cmd'))
        
    def on_message(client, userdata, msg):    
        if not quiet:
            print("MQTT incoming on topic", msg.topic)
        if msg.topic == config['mqtt'].get('topic_cmd'):
            if not quiet:
                print("MQTT CMD:", msg.payload)
            cmd = msg.payload.decode('ascii') + '\n'
            ser.write(cmd.encode('ascii'))

    mqttc = mqtt.Client()
    mqttc.on_connect = on_connect
    #mqttc.on_disconnect = on_disconnect
    mqttc.on_message = on_message
    
    if config['mqtt'].get('user') != "" and config['mqtt'].get('password') != "":
        mqttc.username_pw_set(config['mqtt'].get('user'), config['mqtt'].get('password'))
    mqttc.connect(config['mqtt'].get('server'), config['mqtt'].getint('port'), config['mqtt'].getint('keepalive'))
    
    mqttc.loop_start()


def processMeterData(data):
    try:
        cJson = json.loads(data)
    except:
        cJson = False
    #print(json.dumps(cJson, indent=1))
    #print(cJson['C'])
    #print(json.dumps(meters_yaml[cJson['C']]))
    #print(json.dumps(cJson))
    
    # parse JSON object received from DualS0ImpCounter hardware
    if cJson:
    
        # extract variables from JSON
        cNum = cJson.get('C')
        #print(cNum)
        cReading = float(cJson.get('reading', None))
        dTime = cJson.get('dTime', None)
        
        # declare variables for program sequence control
        write_energy_to_influxdb = False
        
        # get config data for this meter number
        cName = meters_yaml[cNum].get('name', None)
        statTopic = meters_yaml[cNum].get('statTopic', None)
        unit = meters_yaml[cNum].get('unit', None)
        publishJSON = meters_yaml[cNum].get('MQTTPublishJSON', False)
        
        extendJSON = False
        if publishJSON or verbose: extendJSON = True
        
        conv_unit = meters_yaml[cNum].get('conv_unit', None)
        conv_factor = meters_yaml[cNum].get('conv_factor', None)
        conv_digits = meters_yaml[cNum].get('conv_digits', None)
        if conv_digits is None: conv_digits = 2
        
        cost_unit = meters_yaml[cNum].get('cost_unit', None)
        cost_per_unit = meters_yaml[cNum].get('cost_per_unit', None)
        cost_from_conv = meters_yaml[cNum].get('cost_from_conv', False)
        
        momUnit = meters_yaml[cNum].get('momUnit', None)
        momType = meters_yaml[cNum].get('momType', None)
        
        momUnit_conv1 = meters_yaml[cNum].get('momUnit_conv1', None)
        momType_conv1 = meters_yaml[cNum].get('momType_conv1', None)
        
        momUnit_conv2 = meters_yaml[cNum].get('momUnit_conv2', None)
        momType_conv2 = meters_yaml[cNum].get('momType_conv2', None)
        
        impPerUnit = meters_yaml[cNum].get('impPerUnit', None)
        
        momFactor = meters_yaml[cNum].get('momFactor', None)
        if momFactor is None:
            momFactor = 1
        
        momDigits = meters_yaml[cNum].get('momDigits', None)
        if momDigits is None:
            momDigits = 3
            
        momFactor_conv1 = meters_yaml[cNum].get('momFactor_conv1', None)
        if momFactor_conv1 is None:
            momFactor_conv1 = 1
        
        momDigits_conv1 = meters_yaml[cNum].get('momDigits_conv1', None)
        if momDigits_conv1 is None:
            momDigits_conv1 = 3
            
        momFactor_conv2 = meters_yaml[cNum].get('momFactor_conv2', None)
        if momFactor_conv2 is None:
            momFactor_conv2 = 1    
        
        momDigits_conv2 = meters_yaml[cNum].get('momDigits_conv2', None)
        if momDigits_conv2 is None:
            momDigits_conv2 = 3
        
        digits = meters_yaml[cNum].get('digits', None)
        
        influxMinWriteInterval = meters_yaml[cNum].get('influxMinWriteInterval_energy', None)
        if influxMinWriteInterval == None: influxMinWriteInterval = 0
        
        historyPublishInterval = meters_yaml[cNum].get('historyPublishInterval', 300)
        
        billingStartDate = meters_yaml[cNum].get('billingStartDate', None)
        
        # add meter name to JSON object
        if cName is not None and extendJSON:
            cJson['name'] = cName
        
        
        # check for date rollover since last impulse
        today = datetime.date.today()
        today_str = today.strftime('%Y%m%d')
        today_year_str = today.strftime('%Y')
        today_mon_str = today.strftime('%m')
        
        yesterday = today - datetime.timedelta(days = 1)
        yesterday_str = yesterday.strftime('%Y%m%d')
        yesterday_year_str = yesterday.strftime('%Y')
        yesterday_mon_str = yesterday.strftime('%m')
        
        
        # needed for calendar week
        isocal = datetime.date.fromisoformat(today.strftime('%Y-%m-%d')).isocalendar()
        week = isocal[1]
        year = isocal[0]
        #print(year)
        #print(week)
        
        # this week
        ## week_day1 = datetime.date.fromisocalendar(year, week, 1)  # works only from python 3.8 - RPi only has 3.7.3
        week_d = str(year) + "-W" + str(week)
        #print(week_d)
        week_day1 = datetime.datetime.strptime(week_d + '-1', "%Y-W%W-%w") # for python 3.7
        week_day1_str = week_day1.strftime('%Y%m%d')
        #print(week_day1_str)
        week_day1_year_str = week_day1.strftime('%Y')
        week_day1_mon_str = week_day1.strftime('%m')
        
        
        # last week
        if week > 1:
            lastweek = week - 1
            lastweek_year = year
        else: 
            # get highest week number of last year
            lastweek_year = year - 1
            lastyearlastweek = datetime.date.fromisoformat(str(lastweek_year) + "-12-31").isocalendar().week
            lastweek = lastyearlastweek
        lastweek_d = str(lastweek_year) + "-W" + str(lastweek)
        #lastweek_day1 = datetime.date.fromisocalendar(lastweek_year, lastweek, 1)  # works only from python 3.8 - RPi only has 3.7.3
        lastweek_day1 = datetime.datetime.strptime(lastweek_d + '-1', "%Y-W%W-%w") # for python 3.7
        lastweek_day1_str = lastweek_day1.strftime('%Y%m%d')
        lastweek_day1_year_str = lastweek_day1.strftime('%Y')
        lastweek_day1_mon_str = lastweek_day1.strftime('%m')
        
        # this month
        month = int(today.strftime('%m'))
        month_day1_str = str(year) + str(month) + "01"
        
        
        # last month
        lastmonth_year = None
        lastmonth = None
        if month > 1:
            lastmonth = month - 1
            lastmonth_year = year
        else:
            lastmonth = 12
            lastmonth_year = year - 1
        lastmonth_day1_str = str(lastmonth_year) + str(lastmonth) + "01"
        
        
        # billing start date
        if billingStartDate is not None:
            tmp_billdate = billingStartDate.split("-")
            bill_year = tmp_billdate[0]
            bill_month = tmp_billdate[1]
            bill_day = tmp_billdate[2]
            ##print("bill_year=" + str(bill_year))
            ##print("bill_month=" + str(bill_month))
            ##print("bill_day=" + str(bill_day))
        
        
        dateRollover = False
        savedtodaydate = saved_today_date.get(cName, False)
        if not savedtodaydate or savedtodaydate != today:
            if debug:
                print("date rollover happened or no date has been saved yet for meter " + str(cName))
            if savedtodaydate and savedtodaydate == yesterday:
                # date rollover just happened, so change todays date to current and proceed
                dateRollover = True
                #log.debug(savedtodaydate)
            saved_today_date[cName] = today
        
        
        strformat = "{:.3f}"
        cReading_formatted = None
        if digits is not None:
            strformat = "{:."+str(digits)+"f}"
            cReading_formatted = strformat.format(cReading)
            if extendJSON: cJson['reading'] = round(cReading, digits)
        
        # add unit to JSON object
        if unit is not None and extendJSON:
            cJson['unit'] = unit
        
        # calculate momentane usage 
        # if impulses per unit are set and measured time between last impulses is known
        if impPerUnit and dTime is not None:
            if dTime == 0:
                momValue = 0.0
            else:
                momValue = (3600000 / dTime / impPerUnit) * momFactor
                
            # conversions of momValue
            momValue_conv1 = 0.0
            momValue_conv2 = 0.0
            
            if momType_conv1 is not None:
                momValue_conv1 = momValue * momFactor_conv1
                
            if momType_conv2 is not None:
                momValue_conv2 = momValue * momFactor_conv2
            
            # round value of momValue
            if momDigits > 0:
                momValue = round(momValue, momDigits)
            else:
                momValue = round(momValue)
                
            if momDigits_conv1 > 0:
                momValue_conv1 = round(momValue_conv1, momDigits_conv1)
            else:
                momValue_conv1 = round(momValue_conv1)
                
            if momDigits_conv2 > 0:
                momValue_conv2 = round(momValue_conv2, momDigits_conv2)
            else:
                momValue_conv2 = round(momValue_conv2)
                
                
            if momType is not None and extendJSON:
                cJson[momType] = momValue
            
            if momType_conv1 is not None and extendJSON:
                cJson[momType_conv1] = momValue_conv1
            
            if momType_conv2 is not None and extendJSON:
                cJson[momType_conv2] = momValue_conv2
            
            if statTopic and MQTTenabled:
                if momType is not None:
                    mqttc.publish(statTopic + "/" + momType, str(momValue), qos=0, retain=False)
                    
                    if momType_conv1 is not None:
                        mqttc.publish(statTopic + "/" + momType_conv1, str(momValue_conv1), qos=0, retain=False)
                    
                    if momType_conv2 is not None:
                        mqttc.publish(statTopic + "/" + momType_conv2, str(momValue_conv2), qos=0, retain=False)
                        
        
        # publish current reading to MQTT
        if statTopic:
            if MQTTenabled and cReading_formatted is not None:
                mqttc.publish(statTopic + "/reading", str(cReading_formatted), qos=0, retain=False)
        
        
        data_energy[cNum][meters_yaml[cNum].get('influxFieldName_energy', 'energyTotal')] = round(float(cReading), digits)
        #if conv_unit is not None and conv_factor is not None and meters_yaml[cNum].get('influxFieldName_energy_conv', None) is not None::
        #    data_energy[cNum][meters_yaml[cNum].get('influxFieldName_energy_conv', 'energyTotal_conv')] = round(momValue_conv1, digits)
        
        data_momentary[cNum][meters_yaml[cNum].get('influxFieldName_mom', 'momentaryUsage')] = round(float(momValue), momDigits)
        if momType_conv2 is not None and meters_yaml[cNum].get('influxFieldName_mom_conv1', None) is not None:
            data_momentary[cNum][meters_yaml[cNum].get('influxFieldName_mom_conv1', 'momentaryUsage_conv1')] = round(momValue_conv1, momDigits_conv1)
            
        if momType_conv2 is not None and meters_yaml[cNum].get('influxFieldName_mom_conv2', None) is not None:
            data_momentary[cNum][meters_yaml[cNum].get('influxFieldName_mom_conv2', 'momentaryUsage_conv2')] = round(momValue_conv2, momDigits_conv2)
        
        #print()
        #print("data_energy[cNum]")
        #print(data_energy[cNum])
        #print("data_momentary[cNum]")
        #print(data_momentary[cNum])
        #print()
        
        
                
        # check history publish interval - only publish if interval elapsed or first run after program start
        publishEnergyHistory = False
        ts = int(time.time())
        energyHistoryPublish_elapsedTime = 0
        if energy_history_lastPublished.get(cNum, None) != None:
            energyHistoryPublish_elapsedTime = ts - energy_history_lastPublished.get(cNum)
            if energyHistoryPublish_elapsedTime >= historyPublishInterval:
                # write interval elapsed -> write to InfluxDB
                publishEnergyHistory = True
                energy_history_lastPublished[cNum] = ts
        else:
            # first run -> write to InfluxDB immediately
            publishEnergyHistory = True
            energy_history_lastPublished[cNum] = ts
        

        
        
        # InfluxDB
        t_utc = datetime.datetime.utcnow()
        t_str = t_utc.isoformat() + 'Z'
        
        
        # write to InfluxDB after date rollover
        if dateRollover:
            write_energy_to_influxdb = True
            
        # check write interval - only write if interval elapsed or first run after program start
        ts = int(time.time())
        influxEnergyWrite_elapsedTime = 0
        if influxdb_energy_lastWriteTime.get(cNum, None) != None:
            influxEnergyWrite_elapsedTime = ts - influxdb_energy_lastWriteTime.get(cNum)
            if influxEnergyWrite_elapsedTime >= influxMinWriteInterval:
                # write interval elapsed -> write to InfluxDB
                write_energy_to_influxdb = True
        else:
            # first run -> write to InfluxDB immediately
            write_energy_to_influxdb = True
            
        
        # InfluxDB - energy readings
        influxInstance_energy = meters_yaml[cNum].get('influxInstance_energy', None)
        if influxInstance_energy is not None:
            if write_energy_to_influxdb:
                influx_measurement = meters_yaml[cNum].get('influxMeasurement_energy', None)
                if influx_measurement == None:
                    influx_measurement = influxdb_yaml[influxInstance_energy].get('defaultMeasurement', 'energy')
                    
                jsondata_energy = [
                    {
                        'measurement': influx_measurement,
                        'tags': {
                            'meter': cName,
                        },
                        'time': t_str,
                        'fields': data_energy[cNum]
                    }
                ]
                if verbose:
                    print("Writing ENERGY to InfluxDB:")
                    print(json.dumps(jsondata_energy, indent = 4))
                try:
                    influxclient[influxInstance_energy].write_points(jsondata_energy)
                    
                    # remember write time to maintain interval
                    influxdb_energy_lastWriteTime[cNum] = int(time.time())
                    
                except Exception as e:
                    print('Data not written!')
                    #log.error('Data not written!')
                    print(e)
                    #log.error(e)
            else:
                if verbose:
                    print("Writing ENERGY to InfluxDB skipped (influxMinWriteInterval="+ str(influxMinWriteInterval) + ", elapsedTime=" + str(influxEnergyWrite_elapsedTime) + ")")
        
        
        # InfluxDB - momentary values
        influxInstance_mom = meters_yaml[cNum].get('influxInstance_mom', None)
        if influxInstance_mom is not None:
            influx_measurement = meters_yaml[cNum].get('influxMeasurement_mom', None)
            if influx_measurement == None:
                influx_measurement = influxdb_yaml[influxInstance_mom].get('defaultMeasurement', 'energy')
                
            jsondata_momentary = [
                {
                    'measurement': influx_measurement,
                    'tags': {
                        'meter': cName,
                    },
                    'time': t_str,
                    'fields': data_momentary[cNum]
                }
            ]
            if verbose:
                print("Writing MOMENTARY to InfluxDB:")
                print(json.dumps(jsondata_momentary, indent = 4))
            try:
                influxclient[influxInstance_mom].write_points(jsondata_momentary)
            except Exception as e:
                print('Data not written!')
                #log.error('Data not written!')
                print(e)
                #log.error(e)
                
        
        # file log and calculation of today/yesterday values
        if config['filelog'].getboolean('enable'):
            # save and restore yesterday´s total energy to calculate today´s energy
            # check if total energy from yesterday is stored in memory, if not try to get it from saved file
            
            file_path_meter_base = conf_storage_path + cName + "/" 
            
            file_path_meter_year = conf_storage_path + cName + "/" + today_year_str + "/"
            file_path_meter = conf_storage_path + cName + "/" + today_year_str + "/" + today_mon_str + "/"
            file_today_min = file_path_meter + today_str + "_min.txt"
            
            file_path_meter_year_yday = conf_storage_path + cName + "/" + yesterday_year_str + "/"
            file_path_meter_yday = conf_storage_path + cName + "/" + yesterday_year_str + "/" + yesterday_mon_str + "/"
            
            file_yesterday_total = file_path_meter_yday + yesterday_str + "_total.txt"
            file_yesterday_min = file_path_meter_yday + yesterday_str + "_min.txt"
            
            energy_today_min = saved_energy_today_min.get(cName, None)
            energy_today_total = 0
            energy_yesterday_min = 0
            
            energy_yesterday_total = None
            
            try:
                # handle energy_today_min
                if dateRollover:
                    # after date rollover set to None in order to read from last saved file
                    energy_today_min = None
                
                if energy_today_min == None:
                    exists = os.path.isfile(file_today_min)
                    if exists:
                        # load energy_today_min from file if exists
                        f = open(file_today_min, "r")
                        if f.mode == 'r':
                            contents = f.read()
                            f.close()
                        if contents != '':
                            energy_today_min = float(contents)
                            saved_energy_today_min[cName] = energy_today_min
                            if verbose:
                                print(cName + " - Energy Today min read from file -> = " + str(energy_today_min))
                        else: 
                            #energy_today_min = None
                            energy_today_min = cReading # file was empty - take current reading as value
                    else:
                        # save current energy_today to min-file
                        try:
                            if not os.path.exists(file_path_meter_base):
                                os.mkdir(file_path_meter_base)
                            if not os.path.exists(file_path_meter_year):
                                os.mkdir(file_path_meter_year)
                            if not os.path.exists(file_path_meter):
                                os.mkdir(file_path_meter)
                        except:
                                e = sys.exc_info()[0]
                                print( "Error creating directory: %s" % e )
                        f = open(file_today_min, "w+")
                        energy_today_min = cReading
                        saved_energy_today_min[cName] = energy_today_min
                        #f.write(str('{0:.3f}'.format(energy_today_min)))
                        #f.write(str(energy_today_min))
                        f.write(strformat.format(energy_today_min))
                        f.close()
                        
                # calculate energy_today_total
                if energy_today_min != None:
                    energy_today_total = cReading - energy_today_min
                    if verbose:
                        print(cName + " - Energy Today total: " + str('{0:.3f}'.format(energy_today_total)))
                
                # handle energy_yesterday_total
                energy_yesterday_total = saved_energy_yesterday_total.get(cName, None)
                if dateRollover:
                    # after date rollover set to None in order to read from last saved file
                    energy_yesterday_total = None
                    
                if energy_yesterday_total == None:
                    exists = os.path.isfile(file_yesterday_total)
                    if exists:
                        # load energy_yesterday_total from file if exists
                        f = open(file_yesterday_total, "r")
                        if f.mode == 'r':
                            contents = f.read()
                            f.close()
                        if contents != '':
                            energy_yesterday_total = float(contents)
                            saved_energy_yesterday_total[cName] = energy_yesterday_total
                            if debug:
                                print(cName + " - Energy Yesterday total read from file -> = " + str(energy_yesterday_total))
                        else: energy_yesterday_total = None
                    else:
                        # no yesterday_total file exists - create one using file_yesterday_min
                        exists = os.path.isfile(file_yesterday_min)
                        if exists:
                            # load yesterday_min from file
                            f = open(file_yesterday_min, "r")
                            if f.mode == 'r':
                                contents =f.read()
                                f.close()
                            
                            if contents != '':
                                energy_yesterday_min = float(contents)
                                if debug:
                                    print(cName + " - Energy yesterday min: " + str(energy_yesterday_min))
                            else: energy_yesterday_min = None
                            
                            if energy_yesterday_min != None:
                                energy_yesterday_total = round(energy_today_min - energy_yesterday_min, 3)
                                saved_energy_yesterday_total[cName] = energy_yesterday_total
                                try:
                                    if not os.path.exists(file_path_meter_base):
                                        os.mkdir(file_path_meter_base)
                                    if not os.path.exists(file_path_meter_year_yday):
                                        os.mkdir(file_path_meter_year_yday)
                                    if not os.path.exists(file_path_meter):
                                        os.mkdir(file_path_meter)
                                except:
                                    e = sys.exc_info()[0]
                                    print( "Error creating directory: %s" % e )
                                f = open(file_yesterday_total, "w+")
                                f.write(strformat.format(energy_yesterday_total))
                                f.close()
                            
            except:
                e = sys.exc_info()[0]
                print( "Error in file log: %s" % e )
            
            
            # calculate and MQTT publish energy_today and conversions/cost
            if energy_today_total is not None and MQTTenabled:
                # today in base unit
                mqttc.publish(statTopic + "/Today__" + unit, str(round(energy_today_total, digits)), qos=0, retain=False)
                if extendJSON: cJson['Today__' + unit] = round(energy_today_total, digits)
                
                # conversion
                if conv_unit and conv_factor is not None:
                    conv_value = energy_today_total * conv_factor
                    mqttc.publish(statTopic + "/Today__" + conv_unit, str(round(conv_value, conv_digits)), qos=0, retain=False)
                    if extendJSON: cJson['Today__' + conv_unit] = round(conv_value, conv_digits)
                    
                    # cost from conversion
                    if cost_unit and cost_per_unit is not None and cost_from_conv:
                        cost_value = round(conv_value * cost_per_unit, 2)
                        mqttc.publish(statTopic + "/Today_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                        if extendJSON: cJson['Today_cost__' + cost_unit] = round(cost_value, 2)
                
                # cost from base unit, not conversion
                if not cost_from_conv:
                    cost_value = round(energy_today_total * cost_per_unit, 2)                                
                    mqttc.publish(statTopic + "/Today_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                    if extendJSON: cJson['Today_cost__' + cost_unit] = round(cost_value, 2)
            
            
            # calculate and MQTT publish energy_yesterday and conversions/cost
            if energy_yesterday_total is not None and MQTTenabled and publishEnergyHistory:
                # yesterday in base unit
                mqttc.publish(statTopic + "/Yesterday__" + unit, str(round(energy_yesterday_total, digits)), qos=0, retain=False)
                if extendJSON: cJson['Yesterday__' + unit] = round(energy_yesterday_total, digits)
                
                # conversion
                if conv_unit and conv_factor is not None:
                    conv_value = energy_yesterday_total * conv_factor
                    mqttc.publish(statTopic + "/Yesterday__" + conv_unit, str(round(conv_value, conv_digits)), qos=0, retain=False)
                    if extendJSON: cJson['Yesterday__' + conv_unit] = round(conv_value, conv_digits)
                    
                    # cost from conversion
                    if cost_unit and cost_per_unit is not None and cost_from_conv:
                        cost_value = round(conv_value * cost_per_unit, 2)
                        mqttc.publish(statTopic + "/Yesterday_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                        if extendJSON: cJson['Yesterday_cost__' + cost_unit] = round(cost_value, 2)
                
                # cost from base unit, not conversion
                if not cost_from_conv:
                    cost_value = round(energy_yesterday_total * cost_per_unit, 2)                                
                    mqttc.publish(statTopic + "/Yesterday_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                    if extendJSON: cJson['Yesterday_cost__' + cost_unit] = round(cost_value, 2)
            
            
            # calculate this weeks total
            energy_week_minfile = file_path_meter_base + week_day1_year_str + "/" + week_day1_mon_str + "/" + week_day1_str + "_min.txt"
            energy_week_min = None
            if os.path.isfile(energy_week_minfile) and MQTTenabled and publishEnergyHistory:
                # load energy_week_minfile from file if exists
                f = open(energy_week_minfile, "r")
                if f.mode == 'r':
                    contents = f.read()
                    f.close()
                if contents != '':
                    energy_week_min = float(contents)
                    energy_week_total = cReading - energy_week_min
                    if verbose:
                        print(cName + " - Energy Week min read from file '" + energy_week_minfile + "'")
                        print("        -> = " + str(energy_week_min))
                    mqttc.publish(statTopic + "/Week__" + unit, str(round(energy_week_total, digits)), qos=0, retain=False)
                    if extendJSON: cJson['Week__' + unit] = round(energy_week_total, digits)
                    if conv_unit and conv_factor is not None:
                        conv_value = energy_week_total * conv_factor
                        mqttc.publish(statTopic + "/Week__" + conv_unit, str(round(conv_value, conv_digits)), qos=0, retain=False)
                        if extendJSON: cJson['Week__' + conv_unit] = round(conv_value, conv_digits)
                        
                        # cost from conversion
                        if cost_unit and cost_per_unit is not None and cost_from_conv:
                            cost_value = round(conv_value * cost_per_unit, 2)
                            mqttc.publish(statTopic + "/Week_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                            if extendJSON: cJson['Week_cost__' + cost_unit] = round(cost_value, 2)
                    
                    # cost from base unit, not conversion
                    if not cost_from_conv:
                        cost_value = round(energy_week_total * cost_per_unit, 2)                                
                        mqttc.publish(statTopic + "/Week_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                        if extendJSON: cJson['Week_cost__' + cost_unit] = round(cost_value, 2)
                        
                        
            # calculate last weeks total
            energy_lastweek_minfile = file_path_meter_base + lastweek_day1_year_str + "/" + lastweek_day1_mon_str + "/" + lastweek_day1_str + "_min.txt"
            if os.path.isfile(energy_lastweek_minfile) and MQTTenabled and energy_week_min is not None and publishEnergyHistory:
                # load energy_lastweek_minfile from file if exists
                f = open(energy_lastweek_minfile, "r")
                if f.mode == 'r':
                    contents = f.read()
                    f.close()
                if contents != '':
                    energy_lastweek_min = float(contents)
                    energy_lastweek_total = energy_week_min - energy_lastweek_min
                    if verbose:
                        print(cName + " - Energy LastWeek min read from file '" + energy_lastweek_minfile + "'")
                        print("        -> = " + str(energy_week_min))
                    mqttc.publish(statTopic + "/LastWeek__" + unit, str(round(energy_lastweek_total, digits)), qos=0, retain=False)
                    if extendJSON: cJson['LastWeek__' + unit] = round(energy_lastweek_total, digits)
                    if conv_unit and conv_factor is not None:
                        conv_value = energy_lastweek_total * conv_factor
                        mqttc.publish(statTopic + "/LastWeek__" + conv_unit, str(round(conv_value, conv_digits)), qos=0, retain=False)
                        if extendJSON: cJson['LastWeek__' + conv_unit] = round(conv_value, conv_digits)
                        
                        # cost from conversion
                        if cost_unit and cost_per_unit is not None and cost_from_conv:
                            cost_value = round(conv_value * cost_per_unit, 2)
                            mqttc.publish(statTopic + "/LastWeek_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                            if extendJSON: cJson['LastWeek_cost__' + cost_unit] = round(cost_value, 2)
                    
                    # cost from base unit, not conversion
                    if not cost_from_conv:
                        cost_value = round(energy_lastweek_total * cost_per_unit, 2)                                
                        mqttc.publish(statTopic + "/LastWeek_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                        if extendJSON: cJson['LastWeek_cost__' + cost_unit] = round(cost_value, 2)
                        
                        
                        
            # calculate this months total
            energy_month_minfile = file_path_meter_base + str(year) + "/" + str(month) + "/" + month_day1_str + "_min.txt"
            energy_month_min = None
            if os.path.isfile(energy_month_minfile) and MQTTenabled and publishEnergyHistory:
                # load energy_month_minfile from file if exists
                f = open(energy_month_minfile, "r")
                if f.mode == 'r':
                    contents = f.read()
                    f.close()
                if contents != '':
                    energy_month_min = float(contents)
                    energy_month_total = cReading - energy_month_min
                    if verbose:
                        print(cName + " - Energy Month min read from file '" + energy_month_minfile + "'")
                        print("        -> = " + str(energy_month_min))
                    mqttc.publish(statTopic + "/Month__" + unit, str(round(energy_month_total, digits)), qos=0, retain=False)
                    if extendJSON: cJson['Month__' + unit] = round(energy_month_total, digits)
                    if conv_unit and conv_factor is not None:
                        conv_value = energy_month_total * conv_factor
                        mqttc.publish(statTopic + "/Month__" + conv_unit, str(round(conv_value, conv_digits)), qos=0, retain=False)
                        if extendJSON: cJson['Month__' + conv_unit] = round(conv_value, conv_digits)
                        
                        # cost from conversion
                        if cost_unit and cost_per_unit is not None and cost_from_conv:
                            cost_value = round(conv_value * cost_per_unit, 2)
                            mqttc.publish(statTopic + "/Month_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                            if extendJSON: cJson['Month_cost__' + cost_unit] = round(cost_value, 2)
                    
                    # cost from base unit, not conversion
                    if not cost_from_conv:
                        cost_value = round(energy_month_total * cost_per_unit, 2)                                
                        mqttc.publish(statTopic + "/Month_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                        if extendJSON: cJson['Month_cost__' + cost_unit] = round(cost_value, 2)
            
            
            # calculate last months total
            energy_lastmonth_minfile = file_path_meter_base + str(lastmonth_year) + "/" + str(lastmonth) + "/" + lastmonth_day1_str + "_min.txt"
            if os.path.isfile(energy_lastmonth_minfile) and MQTTenabled and energy_month_min is not None and publishEnergyHistory:
                # load energy_lastmonth_minfile from file if exists
                f = open(energy_lastmonth_minfile, "r")
                if f.mode == 'r':
                    contents = f.read()
                    f.close()
                if contents != '':
                    energy_lastmonth_min = float(contents)
                    energy_lastmonth_total = energy_month_min - energy_lastmonth_min
                    if verbose:
                        print(cName + " - Energy LastMonth min read from file '" + energy_lastmonth_minfile + "'")
                        print("        -> = " + str(energy_lastmonth_min))
                    mqttc.publish(statTopic + "/LastMonth__" + unit, str(round(energy_lastmonth_total, digits)), qos=0, retain=False)
                    if extendJSON: cJson['LastMonth__' + unit] = round(energy_lastmonth_total, digits)
                    if conv_unit and conv_factor is not None:
                        conv_value = energy_lastmonth_total * conv_factor
                        mqttc.publish(statTopic + "/LastMonth__" + conv_unit, str(round(conv_value, conv_digits)), qos=0, retain=False)
                        if extendJSON: cJson['LastMonth__' + conv_unit] = round(conv_value, conv_digits)
                        
                        # cost from conversion
                        if cost_unit and cost_per_unit is not None and cost_from_conv:
                            cost_value = round(conv_value * cost_per_unit, 2)
                            mqttc.publish(statTopic + "/LastMonth_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                            if extendJSON: cJson['LastMonth_cost__' + cost_unit] = round(cost_value, 2)
                    
                    # cost from base unit, not conversion
                    if not cost_from_conv:
                        cost_value = round(energy_lastmonth_total * cost_per_unit, 2)                                
                        mqttc.publish(statTopic + "/LastMonth_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                        if extendJSON: cJson['LastMonth_cost__' + cost_unit] = round(cost_value, 2)
            
            
            # calculate this years total
            energy_year_minfile = file_path_meter_base + str(year) + "/01/" + str(year) + "0101_min.txt"
            energy_year_min = None
            if os.path.isfile(energy_year_minfile) and MQTTenabled and publishEnergyHistory:
                # load energy_year_minfile from file if exists
                f = open(energy_year_minfile, "r")
                if f.mode == 'r':
                    contents = f.read()
                    f.close()
                if contents != '':
                    energy_year_min = float(contents)
                    energy_year_total = cReading - energy_year_min
                    if verbose:
                        print(cName + " - Energy Year min read from file '" + energy_year_minfile + "'")
                        print("        -> = " + str(energy_year_min))
                    mqttc.publish(statTopic + "/Year__" + unit, str(round(energy_year_total, digits)), qos=0, retain=False)
                    if extendJSON: cJson['Year__' + unit] = round(energy_year_total, digits)
                    if conv_unit and conv_factor is not None:
                        conv_value = energy_year_total * conv_factor
                        mqttc.publish(statTopic + "/Year__" + conv_unit, str(round(conv_value, conv_digits)), qos=0, retain=False)
                        if extendJSON: cJson['Year__' + conv_unit] = round(conv_value, conv_digits)
                        
                        # cost from conversion
                        if cost_unit and cost_per_unit is not None and cost_from_conv:
                            cost_value = round(conv_value * cost_per_unit, 2)
                            mqttc.publish(statTopic + "/Year_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                            if extendJSON: cJson['Year_cost__' + cost_unit] = round(cost_value, 2)
                    
                    # cost from base unit, not conversion
                    if not cost_from_conv:
                        cost_value = round(energy_year_total * cost_per_unit, 2)                                
                        mqttc.publish(statTopic + "/Year_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                        if extendJSON: cJson['Year_cost__' + cost_unit] = round(cost_value, 2)
                        
            # calculate last years total
            energy_lastyear_minfile = file_path_meter_base + str(year-1) + "/01/" + str(year-1) + "0101_min.txt"
            if os.path.isfile(energy_lastyear_minfile) and MQTTenabled and energy_year_min is not None and publishEnergyHistory:
                # load energy_lastyear_minfile from file if exists
                f = open(energy_lastyear_minfile, "r")
                if f.mode == 'r':
                    contents = f.read()
                    f.close()
                if contents != '':
                    energy_lastyear_min = float(contents)
                    energy_lastyear_total = energy_year_min - energy_lastyear_min
                    if verbose:
                        print(cName + " - Energy LastYear min read from file '" + energy_lastyear_minfile + "'")
                        print("        -> = " + str(energy_lastyear_min))
                    mqttc.publish(statTopic + "/LastYear__" + unit, str(round(energy_lastyear_total, digits)), qos=0, retain=False)
                    if extendJSON: cJson['LastYear__' + unit] = round(energy_lastyear_total, digits)
                    if conv_unit and conv_factor is not None:
                        conv_value = energy_lastyear_total * conv_factor
                        mqttc.publish(statTopic + "/LastYear__" + conv_unit, str(round(conv_value, conv_digits)), qos=0, retain=False)
                        if extendJSON: cJson['LastYear__' + conv_unit] = round(conv_value, conv_digits)
                        
                        # cost from conversion
                        if cost_unit and cost_per_unit is not None and cost_from_conv:
                            cost_value = round(conv_value * cost_per_unit, 2)
                            mqttc.publish(statTopic + "/LastYear_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                            if extendJSON: cJson['LastYear_cost__' + cost_unit] = round(cost_value, 2)
                    
                    # cost from base unit, not conversion
                    if not cost_from_conv:
                        cost_value = round(energy_lastyear_total * cost_per_unit, 2)                                
                        mqttc.publish(statTopic + "/LastYear_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                        if extendJSON: cJson['LastYear_cost__' + cost_unit] = round(cost_value, 2)
            
            # calculate total since last billing
            energy_bill_minfile = file_path_meter_base + str(bill_year) + "/" + str(bill_month) + "/" + str(bill_year) + str(bill_month) + str(bill_day) + "_min.txt"
            if os.path.isfile(energy_bill_minfile) and MQTTenabled and publishEnergyHistory and billingStartDate is not None:
                # load energy_bill_minfile from file if exists
                f = open(energy_bill_minfile, "r")
                if f.mode == 'r':
                    contents = f.read()
                    f.close()
                if contents != '':
                    energy_bill_min = float(contents)
                    energy_bill_total = cReading - energy_bill_min
                    if verbose:
                        print(cName + " - Energy Year min read from file '" + energy_bill_minfile + "'")
                        print("        -> = " + str(energy_bill_min))
                    mqttc.publish(statTopic + "/SinceLastBill__" + unit, str(round(energy_bill_total, digits)), qos=0, retain=False)
                    if extendJSON: cJson['SinceLastBill__' + unit] = round(energy_bill_total, digits)
                    if conv_unit and conv_factor is not None:
                        conv_value = energy_bill_total * conv_factor
                        mqttc.publish(statTopic + "/SinceLastBill__" + conv_unit, str(round(conv_value, conv_digits)), qos=0, retain=False)
                        if extendJSON: cJson['SinceLastBill__' + conv_unit] = round(conv_value, conv_digits)
                        
                        # cost from conversion
                        if cost_unit and cost_per_unit is not None and cost_from_conv:
                            cost_value = round(conv_value * cost_per_unit, 2)
                            mqttc.publish(statTopic + "/SinceLastBill_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                            if extendJSON: cJson['SinceLastBill_cost__' + cost_unit] = round(cost_value, 2)
                    
                    # cost from base unit, not conversion
                    if not cost_from_conv:
                        cost_value = round(energy_bill_total * cost_per_unit, 2)                                
                        mqttc.publish(statTopic + "/SinceLastBill_cost__" + cost_unit, str(cost_value), qos=0, retain=False)
                        if extendJSON: cJson['SinceLastBill_cost__' + cost_unit] = round(cost_value, 2)
                        
                        
        # END file log 
        
        if verbose:
            if consoleAddTimestamp:
                print ("[" + str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]) + "] meterData:", json.dumps(cJson))
            else:
                print("meterData:", json.dumps(cJson))
        
        if MQTTenabled and publishJSON: 
            mqttc.publish(statTopic + "/json", json.dumps(cJson), qos=0, retain=False)

def publishStatMsg(data):
    if MQTTenabled: 
        mqttc.publish(config['mqtt'].get('topic_stat'), data, qos=0, retain=False)

def publishCmdResponseMsg(data):
    if MQTTenabled: 
        mqttc.publish(config['mqtt'].get('topic_cmdresponse'), data, qos=0, retain=False)


try:
    while True:
        serLine = ser.readline().strip()
        # catch exception on invalid char coming in: UnicodeDecodeError: 'ascii' codec can't decode byte 0xf4 in position 6: ordinal not in range(128)
        try: 
            serLine = serLine.decode('ascii')
        except:
            serLine = ""

        if(serLine):
            if verbose:
                if consoleAddTimestamp:
                    print ("[" + str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]) + "] DEV_SENT: ", serLine)
                else:
                    print ("DEV_SENT: ", serLine)    #Echo the serial buffer bytes up to the CRLF back to screen
            
            # check if data looks like meter readings JSON - then process
            if serLine.startswith('{"C":'):
                Thread(target=processMeterData, args=(serLine,)).start()
            
            # response to a command
            elif serLine.startswith('cmd:'):
                Thread(target=publishCmdResponseMsg, args=(serLine,)).start()
            
            # other cases it is a normal "stat" message
            else:
                Thread(target=publishStatMsg, args=(serLine,)).start()
            
except KeyboardInterrupt:
    print('\n')
    exit()