Edit: Realized after I posted this was an old post, leaving anyway in event it's useful for anyone else...
Try something similar to following in form script as part of your onSubmit function (in our case the user input was the servers variable defined below)...
//onSubmit
onSubmit : function () {
var attachments = document.getElementsByName("attachments-1")[0].innerText.match(/\(([^)]+)\)/)[1];
var servers = ca_fdGetSelectedRadio(ca_fd.formId, 'radiog_servers');
if(servers == "Custom" && attachments == 0) {
alert("Attach a spreadsheet containing the custom server list using the Attachments link at the top or bottom of this form, then press Submit.");
return false;
} else {
return true;
}
},