You change the table grobs.
## get the table grobs
g1 <- ggplot_gtable(ggplot_build(p))
library(gtable)
library(grid)
## here the main modification
## change one panel by a new rectangle.
pp <- gtable_add_grob(g1,rectGrob(gp=gpar(col=NA)),t=8,l=6,b=8,r=6)
grid.draw(pp)
