How to group by time bucket in ClickHouse and fill missing data with nulls/0s

Question

Suppose I have a given time range. For explanation, let\'s consider something simple, like whole year 2018. I want to query data from ClickHouse as a sum aggregation for eac

Answer 1

Here is how I did it for hour buckets (needed to visualize this in Grafana) thanks to @filimonov and @mikhail

SELECT t, SUM(metric) as metric FROM (
    SELECT 
        arrayJoin(
          arrayMap( x -> toStartOfHour(addHours(toDateTime($from),x)),
              range(toUInt64(
                  dateDiff('hour', 
                      toDateTime($from), 
                      toDateTime($to)) + 1)))
        ) as t,
        0 as metric

    UNION ALL

    SELECT
        toStartOfHour(my_date) as t,
        COUNT(metric)
        FROM my_table
        WHERE t BETWEEN toDateTime($from) AND toDateTime($to)
        GROUP BY t
)
GROUP BY t ORDER BY t

So, for example for range from 2019-01-01 to 2019-01-02 it will give you:

SELECT t, SUM(metric) as metric FROM (
    SELECT 
        arrayJoin(
          arrayMap( x -> toStartOfHour(addHours(toDateTime('2019-01-01 00:00:00'),x)),
              range(toUInt64(
                  dateDiff('hour', 
                      toDateTime('2019-01-01 00:00:00'), 
                      toDateTime('2019-01-02 00:00:00')) + 1)))
        ) as t,
        0 as metric

    UNION ALL

    SELECT
        toStartOfHour(my_date) as t,
        COUNT(1) as metric
        FROM my_table
        WHERE t BETWEEN toDateTime('2019-01-01 00:00:00') AND toDateTime('2019-01-02 00:00:00')
        GROUP BY t
)
GROUP BY t ORDER BY t;

t                  |metric|
-------------------|------|
2019-01-01 00:00:00|     0|
2019-01-01 01:00:00|     0|
2019-01-01 02:00:00|     0|
2019-01-01 03:00:00|     0|
2019-01-01 04:00:00|     0|
2019-01-01 05:00:00|     0|
2019-01-01 06:00:00|     0|
2019-01-01 07:00:00|105702|
2019-01-01 08:00:00|113315|
2019-01-01 09:00:00|149837|
2019-01-01 10:00:00|185314|
2019-01-01 11:00:00|246106|
2019-01-01 12:00:00|323036|
2019-01-01 13:00:00|     0|
2019-01-01 14:00:00|409160|
2019-01-01 15:00:00|379113|
2019-01-01 16:00:00|256634|
2019-01-01 17:00:00|286601|
2019-01-01 18:00:00|280039|
2019-01-01 19:00:00|248504|
2019-01-01 20:00:00|218642|
2019-01-01 21:00:00|186152|
2019-01-01 22:00:00|148478|
2019-01-01 23:00:00|109721|
2019-01-02 00:00:00|     0|