use e.preventDefault(); to prevent the form from submitting, and then use this.submit() (isn't calling the jQuery .submit() trigger function, but rather the native function) to submit the form.
$("form").submit(function(e) {
e.preventDefault();
var tray = $('select[name=tray_id]').val();
var form = this;
$.ajax({
type: "POST",
url: "modules/reserve-check.php",
data: {tray_id: tray},
cache: false,
complete : function(result){callback(result, form)}
});
});
var callback = function(result, form){
if(!result)
form.submit();
};