In this thread discussing this topic, there is a link to this page, which has an example of exactly what you're looking for.
The strategy seems to be to overlay the map with rounded corner images.
EDIT: Here is the sample code.
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>Rounded Map Corners - Google Maps Javascript API v3</title>
<style>
html, body {height: 100%; margin: 0;}
#Container {position:relative;width:400px;margin:20px;}
.TopLeft, .TopRight, .BottomLeft, .BottomRight {position:absolute;z-index:1000;background-image: url(corners.png);width:20px;height:20px;}
.TopLeft {left: 0; top: 0;}
.TopRight {right: 0; top: 0; background-position: top right;}
.BottomRight {right: 0; bottom: 0; background-position: bottom right;}
.BottomLeft {left: 0; bottom: 0; background-position: bottom left;}
#map_canvas {width: 400px; height: 400px;}
</style>
<script type="text/javascript" src="http://maps.google.com/maps/api/js?sensor=false"></script>
<script type="text/javascript">
var map;
function Initialize() {
var MapOptions = {
zoom: 15,
center: new google.maps.LatLng(37.20084, -93.28121),
mapTypeId: google.maps.MapTypeId.ROADMAP,
sensor: false
};
map = new google.maps.Map(document.getElementById("map_canvas"), MapOptions);
}
</script>
</head>
<body onload="Initialize()">
<div id="Container">
<div class="TopLeft"></div>
<div class="TopRight"></div>
<div id="map_canvas"></div>
<div class="BottomLeft"></div>
<div class="BottomRight"></div>
</div>
</body></html>
