lulatex doesn't seem to accept U+FFFD even when correctly encoded as UTF-8, which seems wrong but anyway you can remove it using the input buffer callback, here changing it to ?

\documentclass{article}
\begin{document}
\directlua{
function zzz(buff)
return string.gsub(buff,"^^^^fffd","?")
end
luatexbase.add_to_callback ( "process_input_buffer", zzz, "zzz" )
}
hmmm�
\end{document}
looking in the source it is clear why this happens, if the input stream can not be decoded the function returns FFFD and then if it is returning FFFD it issues the invalid sequence error. However this means that you get the error on a correctly encoded FFFD input character.
unsigned str2uni(const unsigned char *k)
{
register int ch;
int val = 0xFFFD;
const unsigned char *text = k;
if ((ch = *text++) < 0x80) {
val = (unsigned) ch;
} else if (ch <= 0xbf) { /* error */
} else if (ch <= 0xdf) {
if (*text >= 0x80 && *text < 0xc0)
val = (unsigned) (((ch & 0x1f) << 6) | (*text++ & 0x3f));
} else if (ch <= 0xef) {
if (*text >= 0x80 && *text < 0xc0 && text[1] >= 0x80 && text[1] < 0xc0) {
val = (unsigned)
(((ch & 0xf) << 12) | ((text[0] & 0x3f) << 6) |
(text[1] & 0x3f));
}
} else if (ch <= 0xf7) {
int w = (((ch & 0x7) << 2) | ((text[0] & 0x30) >> 4)) - 1, w2;
w = (w << 6) | ((text[0] & 0xf) << 2) | ((text[1] & 0x30) >> 4);
w2 = ((text[1] & 0xf) << 6) | (text[2] & 0x3f);
val = (unsigned) (w * 0x400 + w2 + 0x10000);
if (*text < 0x80 || text[1] < 0x80 || text[2] < 0x80 ||
*text >= 0xc0 || text[1] >= 0xc0 || text[2] >= 0xc0)
val = 0xFFFD;
} else {
/* the 5- and 6-byte UTF-8 sequences generate integers
that are outside of the valid UCS range, and therefore
unsupported
*/
}
if (val == 0xFFFD)
utf_error();
return (val);
}
\lstlisting{begin}
. This should be\begin{lstlisting}
rather