问题
My goal is to perform a left join on intervals where the bike_id matches and the created_at timestamp in records is BETWEEN start and end in the intervals table
> class(records)
[1] "data.table" "data.frame"
> class(intervals)
[1] "data.table" "data.frame"
> records
bike_id created_at resolved_at
1 28780 2019-05-03 08:29:18 2019-05-03 08:35:37
2 28780 2019-05-03 21:05:28 2019-05-03 21:07:28
3 28780 2019-05-04 21:13:39 2019-05-04 21:15:40
4 28780 2019-05-07 17:24:20 2019-05-07 17:26:39
5 28780 2019-05-08 11:34:32 2019-05-08 12:16:44
6 28780 2019-05-08 23:38:39 2019-05-08 23:40:36
> intervals
bike_id start end id
1: 28780 2019-05-03 04:44:45 2019-05-03 16:58:56 1
2: 28780 2019-05-04 07:07:39 2019-05-04 14:48:29 2
3: 28780 2019-05-07 23:28:32 2019-05-08 12:56:24 3
4: 28780 2019-05-10 06:06:21 2019-05-10 13:12:08 4
5: 28780 2019-05-12 05:21:24 2019-05-12 11:35:52 5
6: 28780 2019-05-13 08:44:54 2019-05-13 12:28:31 6
In this case, the output would look like
> output
bike_id created_at resolved_at id
1 28780 2019-05-03 08:29:18 2019-05-03 08:35:37 1
2 28780 2019-05-03 21:05:28 2019-05-03 21:07:28 NULL
3 28780 2019-05-04 21:13:39 2019-05-04 21:15:40 NULL
4 28780 2019-05-07 17:24:20 2019-05-07 17:26:39 NULL
5 28780 2019-05-08 11:34:32 2019-05-08 12:16:44 NULL
6 28780 2019-05-08 23:38:39 2019-05-08 23:40:36 NULL
I have tried using the solution posted here using tidyverse but this causes R to run out of memory (although the amount of record in both tables are only about 100K)
fuzzy_left_join(
records, intervals,
by = c(
"bike_id" = "bike_id",
"created_at" = "start",
"created_at" = "end"
),
match_fun = list(`==`, `>=`, `<=`)
) %>%
select(id, bike_id = bike_id.x, created_at, start, end)
this throws the error: Error: vector memory exhausted (limit reached?)
Is there an alternative method with rolling join in data.table or even in base R using merge() ? What is a good method to join two dataframes by id and where a timestamp is between two others n the join table?
Here is the data
dput(intervals)
structure(list(bike_id = c(28780L, 28780L, 28780L, 28780L, 28780L,
28780L), start = structure(c(1556858685, 1556953659, 1557271712,
1557468381, 1557638484, 1557737094), class = c("POSIXct", "POSIXt"
), tzone = "UTC"), end = structure(c(1556902736, 1556981309,
1557320184, 1557493928, 1557660952, 1557750511), class = c("POSIXct",
"POSIXt"), tzone = "UTC"), id = c(1, 2, 3, 4, 5, 6)), row.names = c(NA,
-6L), class = c("data.table", "data.frame"), .internal.selfref = <pointer: 0x1030056e0>)
dput(records)
structure(list(bike_id = c(28780L, 28780L, 28780L, 28780L, 28780L,
28780L), created_at = structure(c(1556872158.796, 1556917528.845,
1557004419.928, 1557249860.939, 1557315272.396, 1557358719.333
), class = c("POSIXct", "POSIXt"), tzone = "UTC"), resolved_at = structure(c(1556872537.867,
1556917648.118, 1557004540.056, 1557249999.892, 1557317804.183,
1557358836.202), class = c("POSIXct", "POSIXt"), tzone = "UTC")), row.names = c(NA,
6L), class = "data.frame")
回答1:
I know OP asked for a tidyverse or data.table solution, but SQL seems to be the perfect tool for this:
library(sqldf)
sqldf("select a.*, b.id
from records as a
left join intervals as b
on a.bike_id = b.bike_id and
a.created_at >= b.start and
a.created_at <= b.end")
or use between for an alternate syntax:
sqldf("select a.*, b.id
from records as a
left join intervals as b
on a.bike_id = b.bike_id and
a.created_at between b.start and b.end")
Edit: As noted by @G. Grothendieck, we can set the timezone of the environment (with Sys.setenv) before reading in the data to match OP's timezone.
Output:
bike_id created_at resolved_at id
1 28780 2019-05-03 08:29:18 2019-05-03 08:35:37 1
2 28780 2019-05-03 21:05:28 2019-05-03 21:07:28 NA
3 28780 2019-05-04 21:13:39 2019-05-04 21:15:40 NA
4 28780 2019-05-07 17:24:20 2019-05-07 17:26:39 NA
5 28780 2019-05-08 11:34:32 2019-05-08 12:16:44 3
6 28780 2019-05-08 23:38:39 2019-05-08 23:40:36 NA
Data: (OP's dput does work because of the pointer created from data.table)
Sys.setenv(TZ = "GMT")
records <- structure(list(bike_id = c(28780L, 28780L, 28780L, 28780L, 28780L,
28780L), created_at = c("2019-05-03 08:29:18", "2019-05-03 21:05:28",
"2019-05-04 21:13:39", "2019-05-07 17:24:20", "2019-05-08 11:34:32",
"2019-05-08 23:38:39"), resolved_at = c("2019-05-03 08:35:37",
"2019-05-03 21:07:28", "2019-05-04 21:15:40", "2019-05-07 17:26:39",
"2019-05-08 12:16:44", "2019-05-08 23:40:36")), class = "data.frame", row.names = c(NA,
-6L))
intervals <- structure(list(bike_id = c(28780L, 28780L, 28780L, 28780L, 28780L,
28780L), start = c("2019-05-03 04:44:45", "2019-05-04 07:07:39",
"2019-05-07 23:28:32", "2019-05-10 06:06:21", "2019-05-12 05:21:24",
"2019-05-13 08:44:54"), end = c("2019-05-03 16:58:56", "2019-05-04 14:48:29",
"2019-05-08 12:56:24", "2019-05-10 13:12:08", "2019-05-12 11:35:52",
"2019-05-13 12:28:31"), id = c(1, 2, 3, 4, 5, 6)), class = "data.frame", row.names = c(NA,
-6L))
回答2:
We can use data.table nonequi join
library(data.table)
setDT(records)[intervals, on = .(bike_id, created_at >= start, created_at <= end)]
回答3:
An alternative would be to join on bike_idand the date part of created_at, and then to remove IDs where created_at isn't in the interval start-end. This might solve the memory issue by breaking things up into separate steps:
library(dplyr)
library(lubridate)
library(purrr)
intervals %>%
mutate(date = date(start)) %>%
right_join(mutate(records,
date = date(created_at)),
by = c("bike_id", "date")
) %>%
mutate(within = created_at %within% interval(start, end),
within = replace_na(within, F),
id = map2_dbl(id, within, ~ ifelse(.y, .x, NA))
) %>%
select(bike_id, id, created_at, resolved_at)
Which returns:
# A tibble: 6 x 4
bike_id id created_at resolved_at
<int> <dbl> <dttm> <dttm>
1 28780 1 2019-05-03 08:29:18 2019-05-03 08:35:37
2 28780 NA 2019-05-03 21:05:28 2019-05-03 21:07:28
3 28780 NA 2019-05-04 21:13:39 2019-05-04 21:15:40
4 28780 NA 2019-05-07 17:24:20 2019-05-07 17:26:39
5 28780 NA 2019-05-08 11:34:32 2019-05-08 12:16:44
6 28780 NA 2019-05-08 23:38:39 2019-05-08 23:40:36
来源：https://stackoverflow.com/questions/56688598/left-joining-in-r-between-two-timestamps