Skip to content

Commit c3de1e8

Browse files
pgalkopgalko
committed
Create consolidated summary (day) and streams(minute) views.
This is to tie together the data from different tables and present it in a consolidated form.
1 parent 609cc9a commit c3de1e8

10 files changed

+714
-11
lines changed

‎src/athletedataapp_apache.py

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from mind_monitor_data_download_db_insert import mm_data_insert
2525
from cstm_data_download_db_insert import cstm_data_insert
2626
from weather import get_weather
27+
from time_interval_streams_data import update_intervals_range
2728
from send_email import send_email
2829
from oura_data_download import dwnld_insert_oura_data
2930
from strava_data_download import dwnld_insert_strava_data
@@ -487,8 +488,8 @@ def index():
487488
progress_error = True
488489
with StdoutRedirection(ath_un):
489490
print(gc_login_progress)
490-
flash(' There was a problem logging in to Garmin Connect. Please try again later','warning')
491-
return render_template("index.html",signin_valid=signin_valid,admin_email=admin_email,integrated_with_dropbox=integrated_with_dropbox,diasend_enabled=diasend_enabled,oura_enabled=oura_enabled,strava_enabled=strava_enabled)
491+
flash(' There was a problem logging in to Garmin Connect. The script will now continue, but will skip the Garmin data. Please try again later','warning')
492+
#return render_template("index.html",signin_valid=signin_valid,admin_email=admin_email,integrated_with_dropbox=integrated_with_dropbox,diasend_enabled=diasend_enabled,oura_enabled=oura_enabled,strava_enabled=strava_enabled)
492493

493494
#---------------------------------- GC Activities ---------------------------------------
494495

@@ -763,6 +764,16 @@ def index():
763764
with ErrorStdoutRedirection(ath_un):
764765
print((str(datetime.datetime.now()) + ' [' + sys._getframe().f_code.co_name + ']' + ' Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + ' ' + strava_activ_progress))
765766

767+
#---------------------- Generate Time intervals, and populate "time_interval_min" table -------------------------
768+
769+
#PG:Call to execute "update intervals range" script
770+
try:
771+
flash(' The data download has completed. Setting up and updating the db views. This process might take a while.','info')
772+
update_intervals_range(ath_un,db_host,db_name,superuser_un,superuser_pw)
773+
except Exception as e:
774+
with ErrorStdoutRedirection(ath_un):
775+
print((str(datetime.datetime.now()) + ' [' + sys._getframe().f_code.co_name + ']' + ' Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + ' ' + str(e)))
776+
766777
#------Archive DB to Dropbox-------
767778
try:
768779
if archive_to_dropbox == True:

‎src/athletedataapp_flask.py

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from mind_monitor_data_download_db_insert import mm_data_insert
2525
from cstm_data_download_db_insert import cstm_data_insert
2626
from weather import get_weather
27+
from time_interval_streams_data import update_intervals_range
2728
from send_email import send_email
2829
from oura_data_download import dwnld_insert_oura_data
2930
from strava_data_download import dwnld_insert_strava_data
@@ -483,8 +484,8 @@ def index():
483484
progress_error = True
484485
with StdoutRedirection(ath_un):
485486
print(gc_login_progress)
486-
flash(' There was a problem logging in to Garmin Connect. Please try again later','warning')
487-
return render_template("index.html",signin_valid=signin_valid,admin_email=admin_email,integrated_with_dropbox=integrated_with_dropbox,diasend_enabled=diasend_enabled,oura_enabled=oura_enabled,strava_enabled=strava_enabled)
487+
flash(' There was a problem logging in to Garmin Connect. The script will now continue, but will skip the Garmin data. Please try again later','warning')
488+
#return render_template("index.html",signin_valid=signin_valid,admin_email=admin_email,integrated_with_dropbox=integrated_with_dropbox,diasend_enabled=diasend_enabled,oura_enabled=oura_enabled,strava_enabled=strava_enabled)
488489

489490
#---------------------------------- GC Activities ---------------------------------------
490491

@@ -758,6 +759,16 @@ def index():
758759
time.sleep(1)
759760
with ErrorStdoutRedirection(ath_un):
760761
print((str(datetime.datetime.now()) + ' [' + sys._getframe().f_code.co_name + ']' + ' Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + ' ' + strava_activ_progress))
762+
763+
#---------------------- Generate Time intervals, and populate "time_interval_min" table -------------------------
764+
765+
#PG:Call to execute "update intervals range" script
766+
try:
767+
flash(' The data download has completed. Setting up and updating the db views. This process might take a while.','info')
768+
update_intervals_range(ath_un,db_host,db_name,superuser_un,superuser_pw)
769+
except Exception as e:
770+
with ErrorStdoutRedirection(ath_un):
771+
print((str(datetime.datetime.now()) + ' [' + sys._getframe().f_code.co_name + ']' + ' Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + ' ' + str(e)))
761772

762773
#------Archive DB to Dropbox-------
763774
try:

‎src/db_create_user_database.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,11 +313,19 @@ def restore_db_schema(ath_un,db_host,db_name,superuser_un,superuser_pw):
313313
pidfile = PID_FILE_DIR + ath_un + '_PID.txt'
314314
open(pidfile, 'w').write(pid)
315315

316-
317316
#PG: Read SQL query to restore db schema from file
318317
sql_file = open('db_schema.sql', 'r')
319318
sql_restore_db_schema = s = " ".join(sql_file.readlines())
320319

320+
#PG: Read SQL query to create 1min data view from file
321+
sql_file_view_streams = open('db_create_view_streams.sql', 'r')
322+
sql_create_view_streams = s = " ".join(sql_file_view_streams.readlines())
323+
324+
#PG: Read SQL query to create 1day data view from file
325+
sql_file_view_summary = open('db_create_view_summary.sql', 'r')
326+
sql_create_view_summary = s = " ".join(sql_file_view_summary.readlines())
327+
328+
321329
sql_grant_userpriv = "GRANT ALL PRIVILEGES ON DATABASE \""+ db_name +"\" to \""+db_username+"\";"
322330
sql_grant_table_permissions = "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO \""+db_username+"\";"
323331
sql_revoke_public = "REVOKE ALL ON DATABASE \""+ db_name +"\" FROM PUBLIC;"
@@ -342,9 +350,14 @@ def restore_db_schema(ath_un,db_host,db_name,superuser_un,superuser_pw):
342350
cur.execute(sql_grant_table_permissions)
343351
cur.execute(sql_revoke_public)
344352
cur.execute(sql_revoke_public_1)
353+
cur.execute(sql_create_view_streams)
354+
cur.execute(sql_create_view_summary)
345355

346356
# close the communication with the PostgreSQL
347357
cur.close()
358+
sql_file.close()
359+
sql_file_view_streams.close()
360+
sql_create_view_summary.close()
348361
except (Exception, psycopg2.DatabaseError) as error:
349362
with ErrorStdoutRedirection(ath_un):
350363
print((str(datetime.datetime.now()) + ' [' + sys._getframe().f_code.co_name + ']' + ' Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + ' ' + str(error)))

‎src/db_create_view_streams.sql

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
2+
--
3+
-- CREATE CONSOLIDATED 1MIN INTERVALS DATA VIEW
4+
--
5+
CREATE OR REPLACE VIEW public.view_consolidated_data_streams AS
6+
SELECT DISTINCT athlete.id AS athlete_id,
7+
time_interval_min.timestamp_gmt,
8+
gmt_local_time_difference.gmt_local_difference as utc_offset,
9+
-- Garmin Connect Activity
10+
garmin_connect_original_record.heart_rate as gc_actvt_hr,
11+
garmin_connect_original_record.enhanced_speed as gc_actvt_speed,
12+
garmin_connect_original_record.distance as gc_actvt_distance,
13+
garmin_connect_original_record.enhanced_altitude as gc_actvt_altitude,
14+
garmin_connect_original_record.cadence as gc_actvt_cadence,
15+
garmin_connect_original_record.power as gc_actvt_power,
16+
garmin_connect_original_record.temperature as gc_actvt_temperature,
17+
-- Strava Activity
18+
strava_activity_streams.heartrate as strava_actvt_hr,
19+
strava_activity_streams.velocity_smooth as strava_actvt_speed,
20+
strava_activity_streams.distance as strava_actvt_distance,
21+
strava_activity_streams.altitude as strava_actvt_altitude,
22+
strava_activity_streams.cadence as strava_actvt_cadence,
23+
strava_activity_streams.watts as strava_actvt_power,
24+
strava_activity_streams.temp as strava_actvt_temperature,
25+
strava_activity_streams.grade_smooth as strava_actv_grade,
26+
-- Garmin Connect Wellness
27+
gc_original_wellness_activity_tracking.activity_type as gc_well_actvt_type,
28+
gc_original_wellness_activity_tracking.intensity as gc_well_intensity,
29+
gc_original_wellness_hr_tracking.heart_rate as gc_well_hr,
30+
gc_original_wellness_stress_tracking.stress_level_value as gc_well_stress,
31+
CASE
32+
WHEN gc_original_wellness_sleep_tracking.timestamp_gmt IS NOT NULL THEN True
33+
ELSE False
34+
END AS gc_well_sleeping,
35+
-- Oura
36+
oura_activity_detail.met_1min as oura_act_met,
37+
oura_activity_detail.class_5min as oura_act_intensity,
38+
oura_sleep_detail.hypnogram_5min as oura_hypnogram,
39+
oura_sleep_detail.hr_5min as oura_sleep_hr,
40+
oura_sleep_detail.rmssd_5min as oura_rmssd,
41+
-- MyFitnessPall
42+
STRING_AGG(mfp_nutrition.food_item,',') as nutr_food_items,
43+
SUM(mfp_nutrition.calories) as nutr_calories,
44+
SUM(mfp_nutrition.protein) as nutr_protein,
45+
SUM(mfp_nutrition.carbohydrates) as nutr_carbs,
46+
SUM(mfp_nutrition.fat) as nutr_fat,
47+
SUM(mfp_nutrition.fiber) as nutr_fiber,
48+
SUM(mfp_nutrition.sodium) as nutr_sodium,
49+
-- Glimp, FreestyleLibre
50+
diasend_cgm.glucose_nmol_l as glu_glucose,
51+
diasend_cgm.glucose_nmol_l_15min_avrg as glu_glucose_15m_avrg,
52+
diasend_cgm.ketone_nmol_l as glu_ketones,
53+
--EEG
54+
to_char((((mind_monitor_eeg.alpha_tp9+mind_monitor_eeg.alpha_af7+mind_monitor_eeg.alpha_af8+mind_monitor_eeg.alpha_tp10)/4)+1)*50,'FM999.00')::real as eeg_avg_alpha,
55+
to_char((((mind_monitor_eeg.beta_tp9+mind_monitor_eeg.beta_af7+mind_monitor_eeg.beta_af8+mind_monitor_eeg.beta_tp10)/4)+1)*50,'FM999.00')::real as eeg_avg_beta,
56+
to_char((((mind_monitor_eeg.gamma_tp9+mind_monitor_eeg.gamma_af7+mind_monitor_eeg.gamma_af8+mind_monitor_eeg.gamma_tp10)/4)+1)*50,'FM999.00')::real eeg_avg_gamma,
57+
to_char((((mind_monitor_eeg.delta_tp9+mind_monitor_eeg.delta_af7+mind_monitor_eeg.delta_af8+mind_monitor_eeg.delta_tp10)/4)+1)*50,'FM999.00')::real as eeg_avg_delta,
58+
-- Weather
59+
weather.temperature as wthr_temp,
60+
weather.dew_point as wthr_dew_point,
61+
weather.relative_humidity as wthr_humidity,
62+
weather.precipitation as wthr_precip,
63+
weather.snow as wthr_snow,
64+
weather.wind_direction as wthr_wind_dir,
65+
weather.wind_speed as wthr_wind_speed,
66+
weather.wind_gust as wthr_wind_gust,
67+
weather.sea_air_pressure as wthr_air_press,
68+
weather.total_sunshine as wthr_tot_sunsh,
69+
weather.condition_code as wthr_cond_code
70+
FROM public.athlete
71+
--Timestamps
72+
LEFT JOIN public.time_interval_min ON time_interval_min.athlete_id = athlete.id
73+
LEFT JOIN public.gmt_local_time_difference ON gmt_local_time_difference.local_date::text = left(time_interval_min.timestamp_gmt,-9)
74+
--Garmin Connect Wellness
75+
LEFT JOIN public.gc_original_wellness_activity_tracking ON gc_original_wellness_activity_tracking.timestamp = time_interval_min.timestamp_gmt
76+
LEFT JOIN public.gc_original_wellness_hr_tracking ON gc_original_wellness_hr_tracking.timestamp = time_interval_min.timestamp_gmt
77+
LEFT JOIN public.gc_original_wellness_stress_tracking ON gc_original_wellness_stress_tracking.timestamp = time_interval_min.timestamp_gmt
78+
LEFT JOIN public.gc_original_wellness_sleep_tracking ON gc_original_wellness_sleep_tracking.timestamp_gmt = time_interval_min.timestamp_gmt
79+
--Garmin Connect Activity
80+
LEFT JOIN public.garmin_connect_original_record ON garmin_connect_original_record.timestamp = time_interval_min.timestamp_gmt
81+
--Strava Activity
82+
LEFT JOIN public.strava_activity_streams ON strava_activity_streams.time_gmt = time_interval_min.timestamp_gmt
83+
-- Oura
84+
LEFT JOIN public.oura_activity_detail ON oura_activity_detail.timestamp_gmt = time_interval_min.timestamp_gmt
85+
LEFT JOIN public.oura_sleep_detail ON CONCAT(left(oura_sleep_detail.timestamp_gmt, -2), '00') = time_interval_min.timestamp_gmt
86+
-- Nutrition
87+
-- This is based on MFP defaults (breakfast,lunch,dinner etc) or customised meal labeling (eg '6am-9am','9am-12pm','12pm-3pm'etc).
88+
-- Can be customised in MFP settings and they allow up to 6 meal labels per day.
89+
LEFT JOIN public.mfp_nutrition ON CASE
90+
WHEN meal = '6am-9am' THEN CONCAT(date::text,' 07:30:00')
91+
WHEN meal = 'Breakfast' THEN CONCAT(date::text,' 07:30:00')
92+
WHEN meal = '9am-12pm' THEN CONCAT(date::text,' 10:30:00')
93+
WHEN meal = '12pm-3pm' THEN CONCAT(date::text,' 13:30:00')
94+
WHEN meal = 'Lunch' THEN CONCAT(date::text,' 12:30:00')
95+
WHEN meal = '3pm-6pm' THEN CONCAT(date::text,' 16:30:00')
96+
WHEN meal = 'Snack' THEN CONCAT(date::text,' 15:30:00')
97+
WHEN meal = '6pm-9pm' THEN CONCAT(date::text,' 19:30:00')
98+
WHEN meal = 'Dinner' THEN CONCAT(date::text,' 19:30:00')
99+
WHEN meal = '9pm-12am' THEN CONCAT(date::text,' 22:30:00')
100+
-- Convert GMT timestamp to local
101+
END = (time_interval_min.timestamp_gmt::timestamp + gmt_local_time_difference.gmt_local_difference)::text
102+
-- Glimp, FreestyleLibre
103+
LEFT JOIN public.diasend_cgm ON concat(left(diasend_cgm.timestamp_gmt, -2), '00') = time_interval_min.timestamp_gmt
104+
-- EEG
105+
LEFT JOIN public.mind_monitor_eeg ON left(mind_monitor_eeg.timestamp_gmt, -7) = time_interval_min.timestamp_gmt
106+
-- Weather
107+
LEFT JOIN public.weather ON weather.timestamp_gmt = time_interval_min.timestamp_gmt
108+
--WHERE time_interval_min.timestamp_gmt::timestamp > '2021-10-01 00:00:00'::timestamp
109+
GROUP BY athlete.id,time_interval_min.timestamp_gmt,gmt_local_time_difference.gmt_local_difference,gc_original_wellness_activity_tracking.activity_type,gc_original_wellness_activity_tracking.intensity,
110+
gc_original_wellness_hr_tracking.heart_rate,gc_original_wellness_stress_tracking.stress_level_value,oura_activity_detail.met_1min,
111+
oura_activity_detail.class_5min,oura_sleep_detail.hypnogram_5min,oura_sleep_detail.hr_5min,oura_sleep_detail.rmssd_5min,
112+
gc_original_wellness_sleep_tracking.timestamp_gmt,diasend_cgm.glucose_nmol_l,diasend_cgm.glucose_nmol_l_15min_avrg,diasend_cgm.glucose_nmol_l_15min_avrg,
113+
diasend_cgm.ketone_nmol_l, weather.temperature,weather.dew_point,weather.relative_humidity,weather.precipitation,weather.snow,weather.wind_direction,
114+
weather.wind_speed,weather.wind_gust,weather.sea_air_pressure,weather.total_sunshine,weather.condition_code,garmin_connect_original_record.heart_rate,
115+
garmin_connect_original_record.enhanced_speed,garmin_connect_original_record.distance,garmin_connect_original_record.enhanced_altitude,garmin_connect_original_record.cadence,
116+
garmin_connect_original_record.power,garmin_connect_original_record.temperature,mind_monitor_eeg.alpha_tp9,mind_monitor_eeg.alpha_af7,mind_monitor_eeg.alpha_af8,
117+
mind_monitor_eeg.alpha_tp10,mind_monitor_eeg.beta_tp9,mind_monitor_eeg.beta_af7,mind_monitor_eeg.beta_af8,mind_monitor_eeg.beta_tp10,mind_monitor_eeg.gamma_tp9,
118+
mind_monitor_eeg.gamma_af7,mind_monitor_eeg.gamma_af8,mind_monitor_eeg.gamma_tp10,mind_monitor_eeg.delta_tp9,mind_monitor_eeg.delta_af7,mind_monitor_eeg.delta_af8,
119+
mind_monitor_eeg.delta_tp10,strava_activity_streams.heartrate,strava_activity_streams.velocity_smooth,strava_activity_streams.distance,strava_activity_streams.altitude,strava_activity_streams.cadence,
120+
strava_activity_streams.watts,strava_activity_streams.temp,strava_activity_streams.grade_smooth
121+
ORDER BY time_interval_min.timestamp_gmt DESC --LIMIT 525600 ; -- 1 years of data at 1 min intervals

0 commit comments

Comments
 (0)