Mainly for debugging purposes I would like to have a macro which prints all tokens in a list of tokens and their corresponding catcodes.

Based on an answer by Joseph Wright I have written a macro which returns the catcode of one token.

Based on this question which shows three different loops I see two fundamentally different approaches two iterate over the tokens: dealing with explicit tokens by passing them as arguments or dealing with implicit tokens using \let. The latter seems more promissing therefore I am calling it approach 1.

I am having the following problems:

  • using approach 1 (implicit tokens):
    • I don't know how to print the symbol of an implicit token if it does not have catcode 11 or 12 (letter or other).
    • Active characters are printed as catcode 16.
  • using approach 2 (explicit tokens):
    • Spaces are gobbled.
    • { and } are not printed but used to make groups.

I don't think that the problems of the second approach can be solved. But I am hoping that there are improvements for the first approach or that there might be a third, better approach which I am missing.

\documentclass{article}
\usepackage{filecontents}

% ========== get catcode ==========
% I am not using something like \the\catcode`#1 because:
% (1) I want the catcode of the token, 
%     not the catcode which a token would get 
%     if it was created at this position
% (2) that would not work with implicit tokens

\makeatletter
\@firstofone{\let\implicitSpaceToken= }
\makeatother

% based on https://tex.stackexchange.com/a/7413/120953
\newcommand{\getCatcode}[1]{%
    %    0: escape character, no tokens of that catcode exist
    \ifcat \egroup\noexpand#1%
         1%
    \else\ifcat \bgroup\noexpand#1%
         2%
    \else\ifcat $\noexpand#1%$ (the commented out dollar sign is important for the syntax highlighting in TeXstudio)
         3%
    \else\ifcat &\noexpand#1%
         4%
    %    5: end of line, no tokens of that catcode exist
    \else\ifcat ##\noexpand#1%
         6%
    \else\ifcat ^\noexpand#1%
         7%
    \else\ifcat _\noexpand#1%
         8%
    %    9: ignored character, no tokens of that catcode exist
    \else\ifcat \implicitSpaceToken\noexpand#1%
        10%
    \else\ifcat a\noexpand#1%
        11%
    \else\ifcat 1\noexpand#1%
        12%
    \else\ifcat \noexpand~\noexpand#1%
        13%
    %   14: comment character, no tokens of that catcode exist
    %   15: invalid character, no tokens of that catcode exist
    \else\ifcat \relax\noexpand#1%
        16%
    \else
        error% this can not happen
    \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi
}


% ========== approach 1: loop based on implicit tokens ==========
% + does not ignore spaces
% + no problems with groups
% - wrong catcode for active characters (16 instead of 13)
% - I don't see a feasable way to print the characters (other than those with catcode 10 and 11)

\begin{filecontents}{loop-implicit-tokens.tex}
\def\printtokens#1{
%   \def\do##1{$\texttt{\string##1}_{\getCatcode##1}$}%
    \def\do##1{%
        \edef\i{\getCatcode{##1}}%
        (\i%
        \ifnum \i = 11\relax
            :\,\texttt{##1}%
        \else\ifnum \i = 12\relax
            :\,\texttt{##1}%
        \fi \fi
        )%
    }%
    \iterate#1\relax
}

% based on https://tex.stackexchange.com/q/359189/120953
\def\iterate{\afterassignment\loopbody\let\xchar= }
\def\loopbody{%
    \ifx\relax\xchar
        \let\next=\relax
    \else
        \do\xchar
        \let\next=\iterate
    \fi
    \next
}
\end{filecontents}


% ========== approach 2: loop based on explicit tokens ==========
% + prints correct catcode of active characters
% + possible to print the character
% - ignores *explicit* tokens with catcode 10 (space)
% - problems with *explicit* tokens of catcodes 1 and 2 (groups)

\begin{filecontents*}{loop-explicit-tokens.tex}
\def\printtokens#1{
    \def\do##1{$\texttt{\string##1}_{\getCatcode##1}$}%
    \iterate#1\relax
}

% based on https://tex.stackexchange.com/q/359189/120953
\def\iterate#1{%
    \ifx\relax#1%
    \else
        \do{#1}%
        \expandafter\iterate
    \fi
}
\end{filecontents*}


% ========== main document ==========

\input{loop-implicit-tokens}
%\input{loop-explicit-tokens}

\newcommand{\printtokensinmacro}[1]{\expandafter\printtokens\expandafter{#1}}

\makeatletter
\newcommand{\test}{@ $i_\text{di}^2$&##~}
\makeatother

\begin{document}
\printtokensinmacro\test

\end{document}
  • You really only have to deal with spaces and braces, otherwise you can grab with a usual argument. With \peek_N_type:TF from expl3 you are almost done, check if N type, grab as an argument and process, if not, check if it's a space, or a brace and output the correct catcode for those three particular cases. – Manuel Dec 23 '17 at 11:38
up vote 4 down vote accepted

The solution which uses only TeX primitives is here. You need not expl3, LaTeX etc. The result of \showcatcodes is the same as in another solution presented here.

\def\showcatcodes#1{\showcA#1\showcA}
\def\showcA{\let\next=\showcC \futurelet\nextc\showcB}
\def\showcB{%
   \ifx\nextc\showcA \def\next##1{}\fi
   \expandafter\ifx\space\nextc \def\next{\showcD\ {10}}\fi
   \ifx\nextc{\def\next{\showcD\{{1}}\fi
   \ifx\nextc}\def\next{\showcD\}{2}}\fi
   \next
}
\def\showcC#1{{\tt\string#1}\expandafter
   \ifcat\noexpand#1\relax \showcE{16}\else \showcE{\the\catcode`#1}\fi
   \showcA
}
\def\showcD#1#2{{\tt\char`#1}\showcE{#2}\afterassignment\showcA \let\nextc= }
\def\showcE#1{${}_{#1}$\thinspace}

\showcatcodes{a~b{cd{e1}}2 3!$_ ^y\xxx} 

\end

Edit With regard to the comment (below) by DavidCarlisle I'v added second version of my code:

\def\showcatcodes#1{\showcA#1\showcA}
\def\showcA{\let\next=\showcC \futurelet\nextc\showcB}
\def\showcB{%
   \ifx\nextc\showcA \def\next##1{}\fi
   \ifcat\space\noexpand\nextc \def\next{\showcD\ {10}}\fi
   \ifcat\noexpand\nextc{\def\next{\showcD\{{1}}\fi
   \ifcat\noexpand\nextc}\def\next{\showcD\}{2}}\fi
   \next
}
\def\showcC#1{{\tt\string#1}\showcE{%
   \ifcat\noexpand#1$3\fi \ifcat\noexpand#1&4\fi \ifcat\noexpand#1##6\fi
   \ifcat\noexpand#1^7\fi \ifcat\noexpand#1_8\fi \ifcat\noexpand#1x11\fi 
   \ifcat\noexpand#1:12\fi \ifcat\noexpand#1\noexpand~13\fi 
   \ifcat\noexpand#1\hbox16\fi
   }\showcA
}
\def\showcD#1#2{{\tt\char`#1}\showcE{#2}\afterassignment\showcA \let\nextc= }
\def\showcE#1{${}_{#1}$\thinspace}

\showcatcodes{a~b{cd{e1}}2 3!$_ ^y\xxx} 

\end
  • but this shows \the\catcode`#1 ie the current value of \catcode not the catcode stored in the token, unless I misread it? – David Carlisle Dec 23 '17 at 19:33
  • @DavidCarlisle You are right. I misunderstood the question. Of course, we can replace \the\catcode by a \ifcats collection. – wipet Dec 23 '17 at 19:52
  • yes there is a \ifcat based test in the question, egreg has same comment under his answer:-) – David Carlisle Dec 23 '17 at 20:07
  • @wipet in the case of a { or a } it says \let\next=\showcE \def\next. Is there a hidden meaning behind the let? I would have thought that it should not have any effect because it is directly overwritten by the def. – jakun Dec 24 '17 at 8:36
  • @jakun You are right. This \let is only a leftover of my previous experiments. I corrected it. – wipet Dec 24 '17 at 11:00

You can use a variation on https://tex.stackexchange.com/a/358697/4427

\documentclass{article}
\usepackage{expl3,xparse}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\ExplSyntaxOn

\NewDocumentCommand\showcatcodes { m }
 {
  \group_begin:
  \ttfamily
  \tl_set:Nn \l_tmpa_tl { #1 }
  \jakun_remove_braces:
  \regex_extract_all:nVN { . } \l_tmpa_tl \l_tmpa_seq
  \seq_map_inline:Nn \l_tmpa_seq
   { \jakun_value_catcode:n { ##1 } }
  \group_end:
 }
\cs_new_protected:Nn \jakun_remove_braces:
 {
  \regex_match:nVT { \cB. } \l_tmpa_tl
   {
    \regex_replace_all:nnN { \cB. (.*?) \cE\} } { \cO\{ \1 \cO\} } \l_tmpa_tl
    \jakun_remove_braces:
   }
 }
\cs_generate_variant:Nn \regex_extract_all:nnN { nV }
\prg_generate_conditional_variant:Nnn \regex_match:nn { nV } { T }
\cs_new_protected:Nn \jakun_value_catcode:n
 {
  \bool_lazy_and:nnTF { \tl_if_single_p:n { #1 } } { \token_if_cs_p:N #1 }
   {
    \token_to_str:N #1 \textsubscript{16}
   }
   {
    \str_if_eq:nnTF { #1 } { ~ }
     { \textvisiblespace \textsubscript{10} }
     { \token_to_str:N #1 \textsubscript{\char_value_catcode:n { `#1 }} }
   }
 }
\ExplSyntaxOff


\begin{document}
\showcatcodes{a~b{cd{e1}}2 3!$_ ^y\xxx}
\end{document}

enter image description here

A version that uses the current category codes; I think that much more work would be needed to cope with implicit characters. You can play with it.

I think that \tl_show_analysis:n is much better for the purpose of debugging.

\documentclass{article}
\usepackage[T1]{fontenc}
\usepackage{xparse}

\ExplSyntaxOn
\NewDocumentCommand{\showcatcodes}{sm}
 {
  \group_begin:
  \ttfamily
  \IfBooleanTF{#1}
   {
    \exp_last_unbraced:NV \jakun_showcatcodes: #2 \q_stop
   }
   {
    \jakun_showcatcodes: #2 \q_stop
   }
  \group_end:
 }

\cs_new_protected:Nn \jakun_showcatcodes:
 {
  \peek_meaning_remove:NTF \q_stop
   {
    %\unskip
   }
   {
    \peek_catcode_remove:NTF \c_space_token
     {
      \jakun_print_catcode:nn { \textvisiblespace } { 10 }
      \jakun_showcatcodes:
     }
     {
      \peek_catcode_remove:NTF \c_group_begin_token
       {
        \jakun_print_catcode:nn { \{ } { 1 }
        \jakun_showcatcodes:
       }
       {
        \peek_catcode_remove:NTF \c_group_end_token
         {
          \jakun_print_catcode:nn { \} } { 2 }
          \jakun_showcatcodes:
         }
         {
          \jakun_other_catcode:N
         }
       }
     }
   }
 }

\cs_new_protected:Nn \jakun_other_catcode:N
 {
  \token_if_cs:NTF #1
   {
    \jakun_print_catcode:nn { \token_to_str:N #1 } { 16 }
   }
   {
    \token_if_eq_catcode:NNTF \c_math_toggle_token #1
     {
      \jakun_print_catcode:nn { \token_to_str:N #1 } { 3 }
     }
     {
      \token_if_eq_catcode:NNTF \c_alignment_token #1
       {
        \jakun_print_catcode:nn { \token_to_str:N #1 } { 4 }
       }
       {
        \token_if_eq_catcode:NNTF \c_parameter_token #1
         {
          \jakun_print_catcode:nn { \token_to_str:N #1 } { 6 }
         }
         {
          \token_if_eq_catcode:NNTF \c_math_superscript_token #1
           {
            \jakun_print_catcode:nn { \token_to_str:N #1 } { 7 }
           }
           {
            \token_if_eq_catcode:NNTF \c_math_subscript_token #1
             {
              \jakun_print_catcode:nn { \token_to_str:N #1 } { 8 }
             }
             {
              \token_if_eq_catcode:NNTF \c_catcode_letter_token #1
               {
                \jakun_print_catcode:nn { \token_to_str:N #1 } { 11 }
               }
               {
                \token_if_eq_catcode:NNTF \c_catcode_other_token #1
                 {
                  \jakun_print_catcode:nn { \token_to_str:N #1 } { 12 }
                 }
                 {
                  \jakun_print_catcode:nn { \token_to_str:N #1 } { 13 }
                 }
               }
             }
           }
         }
       }
     }
   }
   \jakun_showcatcodes:
 }

\cs_new_protected:Nn \jakun_print_catcode:nn
 {
  #1\textsubscript{#2}~
 }

\ExplSyntaxOff

\begin{document}

\showcatcodes{abc x{y{z}}~&#_\xyz}

{
\catcode`z=\active
\showcatcodes{abc x{y{z}}~&#_\xyz}
\gdef\test{abc x{y{z}}~&##_\xyz}
}

\showcatcodes*{\test}

\end{document}

enter image description here

  • 1
    but this shows \the\catcode`#1 ie the current value of \catcode not the catcode stored in the token, unless I misread it? – David Carlisle Dec 23 '17 at 19:39

Your Answer

 

By clicking "Post Your Answer", you acknowledge that you have read our updated terms of service, privacy policy and cookie policy, and that your continued use of the website is subject to these policies.

Not the answer you're looking for? Browse other questions tagged or ask your own question.