Another (very flexible) option is to use a regular expression. This allows you to retain the + or any other characters you want to remain.
let numberFromAddressBook = "+1 818 323 323 323"
let cleanNumber = numberFromAddressBook.stringByReplacingOccurrencesOfString("[^0-9+]", withString: "", options: NSStringCompareOptions.RegularExpressionSearch, range:nil)
"+1818323323323"