a macro to print the catcodes of all tokens

Question

Mainly for debugging purposes I would like to have a macro which prints all tokens in a list of tokens and their corresponding catcodes.

Based on an answer by Joseph Wright I have written a macro which returns the catcode of one token.

Based on this question which shows three different loops I see two fundamentally different approaches two iterate over the tokens: dealing with explicit tokens by passing them as arguments or dealing with implicit tokens using \let. The latter seems more promissing therefore I am calling it approach 1.

I am having the following problems:

• using approach 1 (implicit tokens):
  • I don't know how to print the symbol of an implicit token if it does not have catcode 11 or 12 (letter or other).
  • Active characters are printed as catcode 16.
• using approach 2 (explicit tokens):
  • Spaces are gobbled.
  • { and } are not printed but used to make groups.

I don't think that the problems of the second approach can be solved. But I am hoping that there are improvements for the first approach or that there might be a third, better approach which I am missing.

\documentclass{article}
\usepackage{filecontents}

% ========== get catcode ==========
% I am not using something like \the\catcode`#1 because:
% (1) I want the catcode of the token, 
%     not the catcode which a token would get 
%     if it was created at this position
% (2) that would not work with implicit tokens

\makeatletter
\@firstofone{\let\implicitSpaceToken= }
\makeatother

% based on https://tex.stackexchange.com/a/7413/120953
\newcommand{\getCatcode}[1]{%
    %    0: escape character, no tokens of that catcode exist
    \ifcat \egroup\noexpand#1%
         1%
    \else\ifcat \bgroup\noexpand#1%
         2%
    \else\ifcat $\noexpand#1%$ (the commented out dollar sign is important for the syntax highlighting in TeXstudio)
         3%
    \else\ifcat &\noexpand#1%
         4%
    %    5: end of line, no tokens of that catcode exist
    \else\ifcat ##\noexpand#1%
         6%
    \else\ifcat ^\noexpand#1%
         7%
    \else\ifcat _\noexpand#1%
         8%
    %    9: ignored character, no tokens of that catcode exist
    \else\ifcat \implicitSpaceToken\noexpand#1%
        10%
    \else\ifcat a\noexpand#1%
        11%
    \else\ifcat 1\noexpand#1%
        12%
    \else\ifcat \noexpand~\noexpand#1%
        13%
    %   14: comment character, no tokens of that catcode exist
    %   15: invalid character, no tokens of that catcode exist
    \else\ifcat \relax\noexpand#1%
        16%
    \else
        error% this can not happen
    \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi
}


% ========== approach 1: loop based on implicit tokens ==========
% + does not ignore spaces
% + no problems with groups
% - wrong catcode for active characters (16 instead of 13)
% - I don't see a feasable way to print the characters (other than those with catcode 10 and 11)

\begin{filecontents}{loop-implicit-tokens.tex}
\def\printtokens#1{
%   \def\do##1{$\texttt{\string##1}_{\getCatcode##1}$}%
    \def\do##1{%
        \edef\i{\getCatcode{##1}}%
        (\i%
        \ifnum \i = 11\relax
            :\,\texttt{##1}%
        \else\ifnum \i = 12\relax
            :\,\texttt{##1}%
        \fi \fi
        )%
    }%
    \iterate#1\relax
}

% based on https://tex.stackexchange.com/q/359189/120953
\def\iterate{\afterassignment\loopbody\let\xchar= }
\def\loopbody{%
    \ifx\relax\xchar
        \let\next=\relax
    \else
        \do\xchar
        \let\next=\iterate
    \fi
    \next
}
\end{filecontents}


% ========== approach 2: loop based on explicit tokens ==========
% + prints correct catcode of active characters
% + possible to print the character
% - ignores *explicit* tokens with catcode 10 (space)
% - problems with *explicit* tokens of catcodes 1 and 2 (groups)

\begin{filecontents*}{loop-explicit-tokens.tex}
\def\printtokens#1{
    \def\do##1{$\texttt{\string##1}_{\getCatcode##1}$}%
    \iterate#1\relax
}

% based on https://tex.stackexchange.com/q/359189/120953
\def\iterate#1{%
    \ifx\relax#1%
    \else
        \do{#1}%
        \expandafter\iterate
    \fi
}
\end{filecontents*}


% ========== main document ==========

\input{loop-implicit-tokens}
%\input{loop-explicit-tokens}

\newcommand{\printtokensinmacro}[1]{\expandafter\printtokens\expandafter{#1}}

\makeatletter
\newcommand{\test}{@ $i_\text{di}^2$&##~}
\makeatother

\begin{document}
\printtokensinmacro\test

\end{document}

Answer 1

The solution which uses only TeX primitives is here. You need not expl3, LaTeX etc. The result of \showcatcodes is the same as in another solution presented here.

\def\showcatcodes#1{\showcA#1\showcA}
\def\showcA{\let\next=\showcC \futurelet\nextc\showcB}
\def\showcB{%
   \ifx\nextc\showcA \def\next##1{}\fi
   \expandafter\ifx\space\nextc \def\next{\showcD\ {10}}\fi
   \ifx\nextc{\def\next{\showcD\{{1}}\fi
   \ifx\nextc}\def\next{\showcD\}{2}}\fi
   \next
}
\def\showcC#1{{\tt\string#1}\expandafter
   \ifcat\noexpand#1\relax \showcE{16}\else \showcE{\the\catcode`#1}\fi
   \showcA
}
\def\showcD#1#2{{\tt\char`#1}\showcE{#2}\afterassignment\showcA \let\nextc= }
\def\showcE#1{${}_{#1}$\thinspace}

\showcatcodes{a~b{cd{e1}}2 3!$_ ^y\xxx} 

\end

Edit With regard to the comment (below) by DavidCarlisle I'v added second version of my code:

\def\showcatcodes#1{\showcA#1\showcA}
\def\showcA{\let\next=\showcC \futurelet\nextc\showcB}
\def\showcB{%
   \ifx\nextc\showcA \def\next##1{}\fi
   \ifcat\space\noexpand\nextc \def\next{\showcD\ {10}}\fi
   \ifcat\noexpand\nextc{\def\next{\showcD\{{1}}\fi
   \ifcat\noexpand\nextc}\def\next{\showcD\}{2}}\fi
   \next
}
\def\showcC#1{{\tt\string#1}\showcE{%
   \ifcat\noexpand#1$3\fi \ifcat\noexpand#1&4\fi \ifcat\noexpand#1##6\fi
   \ifcat\noexpand#1^7\fi \ifcat\noexpand#1_8\fi \ifcat\noexpand#1x11\fi 
   \ifcat\noexpand#1:12\fi \ifcat\noexpand#1\noexpand~13\fi 
   \ifcat\noexpand#1\hbox16\fi
   }\showcA
}
\def\showcD#1#2{{\tt\char`#1}\showcE{#2}\afterassignment\showcA \let\nextc= }
\def\showcE#1{${}_{#1}$\thinspace}

\showcatcodes{a~b{cd{e1}}2 3!$_ ^y\xxx} 

\end

Answer 2

You can use a variation on https://tex.stackexchange.com/a/358697/4427

\documentclass{article}
\usepackage{expl3,xparse}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\ExplSyntaxOn

\NewDocumentCommand\showcatcodes { m }
 {
  \group_begin:
  \ttfamily
  \tl_set:Nn \l_tmpa_tl { #1 }
  \jakun_remove_braces:
  \regex_extract_all:nVN { . } \l_tmpa_tl \l_tmpa_seq
  \seq_map_inline:Nn \l_tmpa_seq
   { \jakun_value_catcode:n { ##1 } }
  \group_end:
 }
\cs_new_protected:Nn \jakun_remove_braces:
 {
  \regex_match:nVT { \cB. } \l_tmpa_tl
   {
    \regex_replace_all:nnN { \cB. (.*?) \cE\} } { \cO\{ \1 \cO\} } \l_tmpa_tl
    \jakun_remove_braces:
   }
 }
\cs_generate_variant:Nn \regex_extract_all:nnN { nV }
\prg_generate_conditional_variant:Nnn \regex_match:nn { nV } { T }
\cs_new_protected:Nn \jakun_value_catcode:n
 {
  \bool_lazy_and:nnTF { \tl_if_single_p:n { #1 } } { \token_if_cs_p:N #1 }
   {
    \token_to_str:N #1 \textsubscript{16}
   }
   {
    \str_if_eq:nnTF { #1 } { ~ }
     { \textvisiblespace \textsubscript{10} }
     { \token_to_str:N #1 \textsubscript{\char_value_catcode:n { `#1 }} }
   }
 }
\ExplSyntaxOff


\begin{document}
\showcatcodes{a~b{cd{e1}}2 3!$_ ^y\xxx}
\end{document}

A version that uses the current category codes; I think that much more work would be needed to cope with implicit characters. You can play with it.

I think that \tl_show_analysis:n is much better for the purpose of debugging.

\documentclass{article}
\usepackage[T1]{fontenc}
\usepackage{xparse}

\ExplSyntaxOn
\NewDocumentCommand{\showcatcodes}{sm}
 {
  \group_begin:
  \ttfamily
  \IfBooleanTF{#1}
   {
    \exp_last_unbraced:NV \jakun_showcatcodes: #2 \q_stop
   }
   {
    \jakun_showcatcodes: #2 \q_stop
   }
  \group_end:
 }

\cs_new_protected:Nn \jakun_showcatcodes:
 {
  \peek_meaning_remove:NTF \q_stop
   {
    %\unskip
   }
   {
    \peek_catcode_remove:NTF \c_space_token
     {
      \jakun_print_catcode:nn { \textvisiblespace } { 10 }
      \jakun_showcatcodes:
     }
     {
      \peek_catcode_remove:NTF \c_group_begin_token
       {
        \jakun_print_catcode:nn { \{ } { 1 }
        \jakun_showcatcodes:
       }
       {
        \peek_catcode_remove:NTF \c_group_end_token
         {
          \jakun_print_catcode:nn { \} } { 2 }
          \jakun_showcatcodes:
         }
         {
          \jakun_other_catcode:N
         }
       }
     }
   }
 }

\cs_new_protected:Nn \jakun_other_catcode:N
 {
  \token_if_cs:NTF #1
   {
    \jakun_print_catcode:nn { \token_to_str:N #1 } { 16 }
   }
   {
    \token_if_eq_catcode:NNTF \c_math_toggle_token #1
     {
      \jakun_print_catcode:nn { \token_to_str:N #1 } { 3 }
     }
     {
      \token_if_eq_catcode:NNTF \c_alignment_token #1
       {
        \jakun_print_catcode:nn { \token_to_str:N #1 } { 4 }
       }
       {
        \token_if_eq_catcode:NNTF \c_parameter_token #1
         {
          \jakun_print_catcode:nn { \token_to_str:N #1 } { 6 }
         }
         {
          \token_if_eq_catcode:NNTF \c_math_superscript_token #1
           {
            \jakun_print_catcode:nn { \token_to_str:N #1 } { 7 }
           }
           {
            \token_if_eq_catcode:NNTF \c_math_subscript_token #1
             {
              \jakun_print_catcode:nn { \token_to_str:N #1 } { 8 }
             }
             {
              \token_if_eq_catcode:NNTF \c_catcode_letter_token #1
               {
                \jakun_print_catcode:nn { \token_to_str:N #1 } { 11 }
               }
               {
                \token_if_eq_catcode:NNTF \c_catcode_other_token #1
                 {
                  \jakun_print_catcode:nn { \token_to_str:N #1 } { 12 }
                 }
                 {
                  \jakun_print_catcode:nn { \token_to_str:N #1 } { 13 }
                 }
               }
             }
           }
         }
       }
     }
   }
   \jakun_showcatcodes:
 }

\cs_new_protected:Nn \jakun_print_catcode:nn
 {
  #1\textsubscript{#2}~
 }

\ExplSyntaxOff

\begin{document}

\showcatcodes{abc x{y{z}}~&#_\xyz}

{
\catcode`z=\active
\showcatcodes{abc x{y{z}}~&#_\xyz}
\gdef\test{abc x{y{z}}~&##_\xyz}
}

\showcatcodes*{\test}

\end{document}