timezone fixes and moving window on beginning of period

grafana/dashboards/charge-level.json

@@ -177,7 +177,7 @@
           "hide": false,
           "metricColumn": "none",
           "rawQuery": true,
-          "rawSql": "SELECT\n\tdate_bin('2 minutes'::interval, date, to_timestamp(${__from:date:seconds})) as time,\n\tavg(battery_level) AS \"Battery Level\",\n\tavg(usable_battery_level) AS \"Usable Battery Level\"\nfrom positions\n\tWHERE $__timeFilter(date) AND car_id = $car_id\n\tgroup by time\n\tORDER BY time ASC\n;",
+          "rawSql": "SELECT\n\tdate_bin('2 minutes'::interval, date at time zone 'UTC', to_timestamp(${__from:date:seconds})) as time,\n\tavg(battery_level) AS \"Battery Level\",\n\tavg(usable_battery_level) AS \"Usable Battery Level\"\nfrom positions\n\tWHERE $__timeFilter(date) AND car_id = $car_id\n\tgroup by time\n\tORDER BY time ASC\n;",
           "refId": "A",
           "select": [
             [
@@ -253,7 +253,7 @@
           "format": "table",
           "hide": false,
           "rawQuery": true,
-          "rawSql": "-- To be able to calculate percentiles for unevenly sampled values we are bucketing & gapfilling values before running calculations\r\nwith positions_filtered as (\r\n    select\r\n        date,\r\n        battery_level\r\n    from\r\n        positions p\r\n    where\r\n        p.car_id = $car_id\r\n        -- p.ideal_battery_range_km condition is added to reduce overall amount of data and avoid data biases while driving (unevenly sampled data)\r\n        and p.ideal_battery_range_km is not null\r\n        and 1 = $include_average_percentiles\r\n),\r\ngen_date_series as (\r\n    select\r\n        -- series is used to bucket data and avoid gaps in series used to determine percentiles\r\n        generate_series(to_timestamp(${__from:date:seconds}), to_timestamp(${__to:date:seconds}), concat($bucket_width, ' seconds')::INTERVAL) as series_id\r\n),\r\ndate_series as (\r\n    select\r\n        series_id,\r\n        -- before joining, get beginning of next series to be able to left join `positions_filtered`\r\n        lead(series_id) over (order by series_id asc) as next_series_id\r\n    from\r\n        gen_date_series\r\n),\r\npositions_bucketed as (\r\n    select\r\n        series_id,\r\n        -- simple average can result in loss of accuracy, see https://www.timescale.com/blog/what-time-weighted-averages-are-and-why-you-should-care/ for details\r\n        avg(battery_level) as battery_level\r\n    from\r\n        date_series\r\n    left join positions_filtered on\r\n        positions_filtered.date >= date_series.series_id\r\n        and positions_filtered.date < date_series.next_series_id\r\n    group by\r\n        series_id\r\n),\r\n-- PostgreSQL cannot IGNORE NULLS via Window Functions LAST_VALUE - therefore use natural behavior of COUNT & MAX, see https://www.reddit.com/r/SQL/comments/wb949v/comment/ii5mmmi/ for details\r\npositions_bucketed_gapfilling_locf_intermediate as (\r\n    select\r\n        series_id,\r\n        battery_level,\r\n        count(battery_level) over (order by series_id) as i\r\n    from\r\n        positions_bucketed\r\n\r\n),\r\npositions_bucketed_gapfilled_locf as (\r\n    select\r\n        series_id,\r\n        max(battery_level) over (partition by i) as battery_level\r\n    from\r\n        positions_bucketed_gapfilling_locf_intermediate\r\n),\r\n-- PostgreSQL cannot use PERCENTILE_DISC as Window Function - therefore use ARRAY_AGG and UNNEST, see https://stackoverflow.com/a/72718604 for details\r\npositions_bucketed_gapfilled_locf_percentile_intermediate as (\r\n    select\r\n        series_id,\r\n        battery_level,\r\n        array_agg(battery_level) over w as arr,\r\n        avg(battery_level) over w as battery_level_avg\r\n    from\r\n        positions_bucketed_gapfilled_locf\r\n    window w as (rows between (86400 / $bucket_width) * ($days_moving_average_percentiles / 2) preceding and (86400 / $bucket_width) * ($days_moving_average_percentiles / 2) following)\r\n)\r\n\r\nselect\r\n    series_id,\r\n    case when battery_level is null then null else (select percentile_cont(0.075) within group (order by s) from unnest(arr) trick(s)) end as \"$days_moving_average_percentiles Day Moving 7,5% Percentile (${bucket_width:text} buckets)\",\r\n    case when battery_level is null then null else (battery_level_avg) end as \"$days_moving_average_percentiles Day Moving Average (${bucket_width:text} buckets)\",\r\n    case when battery_level is null then null else (select percentile_cont(0.5) within group (order by s) from unnest(arr) trick(s)) end as \"$days_moving_average_percentiles Day Moving Median (${bucket_width:text} buckets)\",\r\n    case when battery_level is null then null else (select percentile_cont(0.925) within group (order by s) from unnest(arr) trick(s)) end as \"$days_moving_average_percentiles Day Moving 92,5% Percentile (${bucket_width:text} buckets)\"\r\nfrom\r\n    positions_bucketed_gapfilled_locf_percentile_intermediate;",
+          "rawSql": "-- To be able to calculate percentiles for unevenly sampled values we are bucketing & gapfilling values before running calculations\r\nwith positions_filtered as (\r\n    select\r\n        date,\r\n        battery_level\r\n    from\r\n        positions p\r\n    where\r\n        p.car_id = $car_id\r\n        -- p.ideal_battery_range_km condition is added to reduce overall amount of data and avoid data biases while driving (unevenly sampled data)\r\n        and p.ideal_battery_range_km is not null\r\n        and 1 = $include_average_percentiles\r\n),\r\ngen_date_series as (\r\n    select\r\n        -- series is used to bucket data and avoid gaps in series used to determine percentiles\r\n        generate_series(to_timestamp(${__from:date:seconds} - (86400 * $days_moving_average_percentiles / 2)), to_timestamp(${__to:date:seconds}), concat($bucket_width, ' seconds')::INTERVAL) as series_id\r\n),\r\ndate_series as (\r\n    select\r\n        series_id at time zone 'UTC' as series_id,\r\n        -- before joining, get beginning of next series to be able to left join `positions_filtered`\r\n        lead(series_id) over (order by series_id asc) at time zone 'UTC' as next_series_id\r\n    from\r\n        gen_date_series\r\n),\r\npositions_bucketed as (\r\n    select\r\n        series_id,\r\n        -- simple average can result in loss of accuracy, see https://www.timescale.com/blog/what-time-weighted-averages-are-and-why-you-should-care/ for details\r\n        avg(battery_level) as battery_level,\r\n        min(positions_filtered.date) as series_min_date\r\n    from\r\n        date_series\r\n    left join positions_filtered on\r\n        positions_filtered.date >= date_series.series_id\r\n        and positions_filtered.date < date_series.next_series_id\r\n    group by\r\n        series_id\r\n),\r\n-- PostgreSQL cannot IGNORE NULLS via Window Functions LAST_VALUE - therefore use natural behavior of COUNT & MAX, see https://www.reddit.com/r/SQL/comments/wb949v/comment/ii5mmmi/ for details\r\npositions_bucketed_gapfilling_locf_intermediate as (\r\n    select\r\n        series_id,\r\n        battery_level,\r\n        series_min_date,\r\n        count(battery_level) over (order by series_id) as i\r\n    from\r\n        positions_bucketed\r\n\r\n),\r\npositions_bucketed_gapfilled_locf as (\r\n    select\r\n        series_id,\r\n        series_min_date,\r\n        max(battery_level) over (partition by i) as battery_level_locf\r\n    from\r\n        positions_bucketed_gapfilling_locf_intermediate\r\n),\r\n-- PostgreSQL cannot use PERCENTILE_DISC as Window Function - therefore use ARRAY_AGG and UNNEST, see https://stackoverflow.com/a/72718604 for details\r\npositions_bucketed_gapfilled_locf_percentile_intermediate as (\r\n    select\r\n        series_id,\r\n        series_min_date,\r\n        min(series_min_date) over () as min_date,\r\n        array_agg(battery_level_locf) over w as arr,\r\n        avg(battery_level_locf) over w as battery_level_avg\r\n    from\r\n        positions_bucketed_gapfilled_locf\r\n    window w as (rows between (86400 / $bucket_width) * ($days_moving_average_percentiles / 2) preceding and (86400 / $bucket_width) * ($days_moving_average_percentiles / 2) following)\r\n)\r\n\r\nselect\r\n    series_id::timestamptz,\r\n    (select percentile_cont(0.075) within group (order by s) from unnest(arr) trick(s)) as \"$days_moving_average_percentiles Day Moving 7,5% Percentile (${bucket_width:text} buckets)\",\r\n    battery_level_avg as \"$days_moving_average_percentiles Day Moving Average (${bucket_width:text} buckets)\",\r\n    (select percentile_cont(0.5) within group (order by s) from unnest(arr) trick(s)) as \"$days_moving_average_percentiles Day Moving Median (${bucket_width:text} buckets)\",\r\n    (select percentile_cont(0.925) within group (order by s) from unnest(arr) trick(s)) as \"$days_moving_average_percentiles Day Moving 92,5% Percentile (${bucket_width:text} buckets)\"\r\nfrom\r\n    positions_bucketed_gapfilled_locf_percentile_intermediate where $__timeFilter(series_id) and series_min_date >= min_date",
           "refId": "C",
           "sql": {
             "columns": [
@@ -507,6 +507,6 @@
   "timezone": "",
   "title": "Charge Level",
   "uid": "WopVO_mgz",
-  "version": 16,
+  "version": 22,
   "weekStart": ""
 }