lstlisting gives error String contains an invalid utf-8 sequence

Question

So I have a code listing that contains 0xef 0xbf 0xbd. It's part of a test case:

\documentclass{article}
\usepackage{listings}

\begin{document}

\begin{lstlisting}

�\x04��

\end{lstlisting}

\end{document}

hexdump:

000000d0  67 7b 62 65 67 69 6e 7d  0a 0a ef bf bd 5c 78 30  |g{begin}.....\x0|
000000e0  34 ef bf bd ef bf bd 0a  0a 5c 6c 73 74 6c 69 73  |4........\lstlis|
000000f0  74 69 6e 67 7b 65 6e 64  7d 0a 0a 5c 65 6e 64 7b  |ting{end}..\end{|

lualatex says ! String contains an invalid utf-8 sequence

How can I get lualatex to print it as a question mark on black background or something like that?

Answer 1

lulatex doesn't seem to accept U+FFFD even when correctly encoded as UTF-8, which seems wrong but anyway you can remove it using the input buffer callback, here changing it to ?

\documentclass{article}



\begin{document}

\directlua{
function zzz(buff)
 return string.gsub(buff,"^^^^fffd","?")
end
luatexbase.add_to_callback ( "process_input_buffer", zzz, "zzz" )
}

hmmm�

\end{document}

---

looking in the source it is clear why this happens, if the input stream can not be decoded the function returns FFFD and then if it is returning FFFD it issues the invalid sequence error. However this means that you get the error on a correctly encoded FFFD input character.

unsigned str2uni(const unsigned char *k)
{
    register int ch;
    int val = 0xFFFD;
    const unsigned char *text = k;
    if ((ch = *text++) < 0x80) {
        val = (unsigned) ch;
    } else if (ch <= 0xbf) {    /* error */
    } else if (ch <= 0xdf) {
        if (*text >= 0x80 && *text < 0xc0)
            val = (unsigned) (((ch & 0x1f) << 6) | (*text++ & 0x3f));
    } else if (ch <= 0xef) {
        if (*text >= 0x80 && *text < 0xc0 && text[1] >= 0x80 && text[1] < 0xc0) {
            val = (unsigned)
                (((ch & 0xf) << 12) | ((text[0] & 0x3f) << 6) |
                 (text[1] & 0x3f));
        }
    } else if (ch <= 0xf7) {
        int w = (((ch & 0x7) << 2) | ((text[0] & 0x30) >> 4)) - 1, w2;
        w = (w << 6) | ((text[0] & 0xf) << 2) | ((text[1] & 0x30) >> 4);
        w2 = ((text[1] & 0xf) << 6) | (text[2] & 0x3f);
        val = (unsigned) (w * 0x400 + w2 + 0x10000);
        if (*text < 0x80 || text[1] < 0x80 || text[2] < 0x80 ||
            *text >= 0xc0 || text[1] >= 0xc0 || text[2] >= 0xc0)
            val = 0xFFFD;
    } else {
        /* the 5- and 6-byte UTF-8 sequences generate integers
           that are outside of the valid UCS range, and therefore
           unsupported
         */
    }

---

    if (val == 0xFFFD)
        utf_error();

---

    return (val);
}