Mercurial > ~darius > hgwebdir.cgi > vanlogger

comparison graph.py @ 4:a7e9775b33f6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add graph script based on AGL code (but better)
author Daniel O'Connor <darius@dons.net.au>
date Fri, 22 Dec 2017 13:03:06 +0100
parents
children 30e7adf283ca
comparison
equal deleted inserted replaced
3:e29a8fcdbd57 4:a7e9775b33f6
1 #!/usr/bin/env python
2
3 import argparse
4 import datetime
5 import dateutil
6 import exceptions
7 import matplotlib
8 import matplotlib.dates
9 import numpy
10 import os
11 import requests
12 import sqlite3
13 import tzlocal
14
15 class Column(object):
16 def __init__(self, rowname, title, table, units, limits = (None, None), conv = None):
17 self.rowname = rowname
18 self.title = title
19 self.table = table
20 self.units = units
21 self.limits = limits
22 self.conv = None
23
24 columns = [
25 Column('main_voltage', 'Battery Voltage', 'eprolog', 'Vdc'),
26 Column('aux_voltage', 'Aux Voltage', 'eprolog', 'Vdc'),
27 Column('battery_curr', 'Battery Current', 'eprolog', 'A'),
28 Column('amp_hours', 'Battery Amp Hours', 'eprolog', 'Ah'),
29 Column('state_of_charge', 'State of Charge', 'eprolog', '%', (0, 100)),
30 Column('time_remaining', 'Time Remaining', 'eprolog', 'min'),
31 Column('battery_temp', 'Battery Temperature', 'eprolog', 'C'),
32
33 Column('ac_act_power', 'Active Power', 'giantlog', 'W'),
34 Column('ac_app_power', 'Apparent Power', 'giantlog', 'W'),
35 Column('ac_frequency', 'AC Frequency', 'giantlog', 'Hz'),
36 Column('ac_volts', 'AC Voltage', 'giantlog', 'Vac'),
37 Column('batt_chr_curr', 'Discharge Current', 'giantlog', 'A'),
38 Column('batt_dis_curr', 'Charge Current', 'giantlog', 'A'),
39 Column('battery_cap', 'Battery Capacity', 'giantlog', '%', (0, 100)),
40 Column('battery_volts', 'Battery Voltage', 'giantlog', 'Vdc'),
41 Column('grid_frequency', 'Grid Frequency', 'giantlog', 'Hz'),
42 Column('grid_volts', 'Grid Voltage', 'giantlog', 'Vac'),
43 Column('hs_temperature', 'HS Temperature', 'giantlog', 'C'),
44 Column('load_pct', 'Load', 'giantlog', '%', (0, 100)),
45 ]
46
47 def valid_date(s):
48 try:
49 return datetime.datetime.strptime(s, "%Y-%m-%d")
50 except ValueError:
51 raise argparse.ArgumentTypeError("Not a valid date: '{0}'.".format(s))
52
53 def main():
54 parser = argparse.ArgumentParser()
55 parser.add_argument('-f', '--filename', help = 'Path to database', type = str, required = True)
56 parser.add_argument('-g', '--graphfn', help = 'File to write graph to', type = str)
57 parser.add_argument('-d', '--days', help = 'Days ago to graph', type = int)
58 parser.add_argument('-s', '--start', help = 'Start date for graph (YYYY-MM-DD)', type = valid_date)
59 parser.add_argument('-e', '--end', help = 'End date for graph (YYYY-MM-DD)', type = valid_date)
60 parser.add_argument('-c', '--column', help = 'Column to plot (can be specified multiple times)', type = str, action = 'append')
61
62 args = parser.parse_args()
63
64 if args.days is not None and args.days < 0:
65 parser.error('days must be non-negative')
66
67 # Can specify..
68 # Start and end
69 # Start and days
70 # End and days
71 # Nothing
72 # Want to end up with a start & end
73 if args.start is not None and args.end is not None:
74 pass
75 elif args.start is not None and args.days is not None:
76 args.end = args.start + datetime.timedelta(days = args.days)
77 elif args.end is not None and args.days is not None:
78 args.start = args.end - datetime.timedelta(days = args.days)
79 elif args.start is None and args.end is None and args.days is None:
80 end = datetime.date.today()
81 end = datetime.datetime(start.year, start.month, start.day)
82 args.start = args.end - datetime.timedelta(days = args.days)
83 else:
84 parser.error('can\'t specify days, start and end simultaneously')
85
86 if args.start >= args.end:
87 parser.error('Start must be before end')
88
89 cols = args.column
90 if cols == None:
91 cols = ['main_voltage', 'aux_voltage', 'ac_app_power']
92
93 dbh = sqlite3.connect(args.filename, detect_types = sqlite3.PARSE_DECLTYPES)
94 cur = dbh.cursor()
95
96 # Get local timezone name and convert start/end to it
97 # Why is this so hard...
98 ltname = tzlocal.get_localzone().zone
99 ltname = 'Australia/Adelaide'
100 lt = dateutil.tz.gettz(ltname)
101 utc = dateutil.tz.gettz('UTC')
102 matplotlib.rcParams['timezone'] = ltname
103
104 if args.start.tzinfo == None:
105 args.start = args.start.replace(tzinfo = lt)
106 if args.end.tzinfo == None:
107 args.end = args.end.replace(tzinfo = lt)
108 startlt = args.start
109 endlt = args.end
110 args.start = args.start.astimezone(utc)
111 args.end = args.end.astimezone(utc)
112 graph(args.graphfn, cur, cols, int(args.start.strftime('%s')), int(args.end.strftime('%s')), lt, utc)
113
114 def graph(fname, cur, _cols, start, end, lt, utc):
115 import numpy
116 import matplotlib
117 import matplotlib.dates
118
119 startdt = datetime.datetime.fromtimestamp(start).replace(tzinfo = utc).astimezone(lt)
120 enddt = datetime.datetime.fromtimestamp(end).replace(tzinfo = utc).astimezone(lt)
121
122 colourlist = ['b','g','r','c','m','y','k']
123
124 cols = []
125
126 yaxisunits1 = None
127 yaxisunits2 = None
128 ax1lines = []
129 ax2lines = []
130 colouridx = 0
131 for col in _cols:
132 # Check the column exists
133 for c in columns:
134 if col == c.rowname:
135 cols.append(c)
136 break
137 else:
138 raise exceptions.Exception('Unknown column name ' + c)
139
140 # Work out what axes we are using
141 if yaxisunits1 == None:
142 yaxisunits1 = c.units
143 if yaxisunits2 == None:
144 if c.units != yaxisunits1:
145 yaxisunits2 = c.units
146 else:
147 if c.units != yaxisunits1 and c.units != yaxisunits2:
148 raise exceptions.Exception('Asked to graph >2 different units')
149
150 for c in cols:
151 # Get the data
152 cur.execute('SELECT tstamp, ' + c.rowname + ' FROM ' + c.table + ' WHERE tstamp > ? AND tstamp < ? ORDER BY tstamp',
153 (start, end))
154 ary = numpy.array(cur.fetchall())
155 if ary.shape[0] == 0:
156 print('No data for ' + c.rowname)
157 return
158
159 # Create TZ naive from POSIX stamp, then convert to TZ aware UTC then adjust to local time
160 c.xdata = map(lambda f: datetime.datetime.fromtimestamp(f).replace(tzinfo = utc).astimezone(lt), ary[:,0])
161 c.ydata = ary[:,1]
162 if c.conv != None:
163 c.ydata = map(c.conv, c.ydata)
164
165 scale_min, scale_max = c.limits
166
167 # DoD?
168 c.annotation = None
169
170 # Work out which axis to plot on
171 if c.units == yaxisunits1:
172 ax = ax1lines
173 else:
174 ax = ax2lines
175 c.colour = colourlist[colouridx]
176 colouridx += 1
177 ax.append(c)
178
179 # Load the right backend for display or save
180 if fname == None:
181 import matplotlib.pylab
182 fig = matplotlib.pylab.figure()
183 else:
184 import matplotlib.backends.backend_agg
185 fig = matplotlib.figure.Figure(figsize = (12, 6), dpi = 75)
186
187 # Do the plot
188 ax1 = fig.add_subplot(111)
189 ax1.set_ylabel(yaxisunits1)
190
191 annotations = []
192 for line in ax1lines:
193 ax1.plot(line.xdata, line.ydata, label = line.title, color = line.colour)
194 if line.limits[0] != None or line.limits[1] != None:
195 ax1.set_ylim(line.limits[0], line.limits[1])
196 if line.annotation != None:
197 annotations.append(line.annotation)
198 ax1.legend(loc = 'upper left')
199
200 if len(ax2lines) > 0:
201 ax2 = ax1.twinx()
202 ax2.set_ylabel(yaxisunits2)
203
204 for line in ax2lines:
205 ax2.plot(line.xdata, line.ydata, label = line.title, color = line.colour)
206 if line.limits[0] != None or line.limits[1] != None:
207 ax2.set_ylim(line.limits[0], line.limits[1])
208 if line.annotation != None:
209 annotations.append(line.annotation)
210
211 ax2.legend(loc = 'upper right')
212
213 if len(annotations) > 0:
214 ax1.text(0.02, 0.9, reduce(lambda a, b: a + '\n' + b, annotations),
215 transform = ax1.transAxes, bbox = dict(facecolor = 'red', alpha = 0.5),
216 ha = 'left', va = 'top')
217 ndays = int(max(1, round((end - start) / 86400)))
218 for ax in fig.get_axes():
219 if (enddt - startdt).total_seconds() > 86400:
220 ax.set_title('%s to %s' % (startdt.strftime('%Y-%m-%d'), enddt.strftime('%Y-%m-%d')))
221 else:
222 ax.set_title('%s' % (startdt.strftime('%Y-%m-%d')))
223 ax.set_xlim([startdt, enddt])
224 ax.format_xdata = lambda d: matplotlib.dates.num2date(d).strftime('%d %b %H:%M')
225 ax.xaxis.grid(True)
226 ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter('%d %b\n%H:%M'))
227 ax.xaxis.set_major_locator(matplotlib.dates.HourLocator(interval = 2 * ndays))
228 ax.xaxis.set_minor_locator(matplotlib.dates.MinuteLocator(interval = 5 * ndays))
229 for label in ax.get_xticklabels():
230 label.set_ha('center')
231 label.set_rotation(90)
232
233 # Fudge margins to give more graph and less space
234 fig.subplots_adjust(left = 0.10, right = 0.88, top = 0.95, bottom = 0.15)
235 if fname == None:
236 matplotlib.pyplot.show()
237 else:
238 canvas = matplotlib.backends.backend_agg.FigureCanvasAgg(fig) # Sets canvas in fig too
239 fig.savefig(startlt.strftime(fname))
240
241 def updatedb(cur, data):
242 mkdb(cur)
243 for d in data['reads']['data']:
244 ts = datetime.datetime.strptime(d['t_stamp'], '%Y-%m-%dT%H:%M:%SZ')
245 # Note we rename *energy* to *power* here to match what it actually means
246 vals = [ts, d['battery_charge'], d['battery_energy'], d['energy_consumed'], d['energy_expected'], d['energy_exported'], d['energy_generated'],
247 d['energy_imported'], d['estimated_savings'], d['pv_forecast'], d['pv_generation']['battery_energy'],
248 d['pv_generation']['grid_energy'], d['pv_generation']['site_energy'], d['site_consumption']['battery_energy'],
249 d['site_consumption']['grid_energy'], d['site_consumption']['pv_energy']]
250 skip = True
251 for v in vals[1:]:
252 if v != None:
253 skip = False
254 break
255 if skip:
256 print('Skipping empty record at ' + str(ts))
257 continue
258 cur.execute('INSERT OR IGNORE INTO agl(t_stamp, battery_charge, battery_power, power_consumed, power_expected, power_exported, power_generated, power_imported, estimated_savings, pv_forecast, pv_gen_battery, pv_gen_grid, pv_gen_site, site_cons_battery, site_cons_grid, site_cons_pv) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', vals)
259
260 def gettoken(username, password):
261 authblob = json.encoder.JSONEncoder().encode({'email' : username, 'password' : password})
262 reply = requests.request('POST', loginurl, data = authblob, headers = {'Content-Type' : 'application/json'})
263 if reply.status_code != 200:
264 return None
265 return json.decoder.JSONDecoder().decode(reply.content)['access_token']
266
267 def getdata(token, startdate, enddate):
268 #print('getting ' + startdate.strftime('%Y-%m-%d'))
269 reply = requests.request('GET', dataurl, params = {
270 'startDate' : startdate.strftime('%Y-%m-%d'),
271 'endDate' : enddate.strftime('%Y-%m-%d'),
272 'granularity' : 'Minute',
273 'metrics' : 'read',
274 'units' : 'W',
275 }, headers = { 'Authorization' : 'Bearer ' + token})
276
277 if reply.status_code != 200:
278 return None
279
280 return json.decoder.JSONDecoder().decode(reply.content)
281
282 def logout(token):
283 reply = requests.request('GET', logouturl, headers = { 'Authorization' : 'Bearer ' + token})
284 return reply.status_code == 200
285
286 if __name__ == '__main__':
287 main()