fully expand toks and save for later (appendix with exact copies of tables from the main text)

Question

I have a document with some complex tables, defined by a command called \Rule with named parameters (using the keyval package). The individual rows of the table are first stored into a token list defined via \newtoks because some of the named parameters correspond to multiple cells of the table and simultaneously must be enveloped inside an \ifcsname condition. I'd like to force \Rule to store a fully expanded copy of the resulting table (with the particular values for each named parameter) so that I can list all rules that appeared anywhere in the document in the appendix. But I was not able to force the token list to fully expand.

I compile with xetex and cannot change that.

---

EDIT: Each table corresponds to a single call of \Rule; the named parameters contain the information, and the definition of \Rule provides the formatting for each kind of named parameter (most are optional). I want exactly the same tables in main text and in the appendix: the output should be the same as if I manually copy-pasted all \Rule commands from the document into the appendix.

---

\documentclass{book}
\usepackage{xparse,keyval}

\newcounter{rule}
\renewcommand{\therule}[0]{G\arabic{rule}}

\makeatletter
\newtoks\@tabtoks
\newcommand\addtabtoks[1]{\@tabtoks\expandafter{\the\@tabtoks#1}}
\newcommand*\resettabtoks{\@tabtoks{}}
\newcommand*\printtabtoks{\the\@tabtoks}

\edef\allrules{}
\edef\oldallrules{\allrules}
\newcommand*\addtoallrules[1]{\g@addto@macro{\allrules}{#1}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%      PARAMETERS       %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\define@key{myKeys}{nonnumbered}{\def\mm@nonnumbered{#1}}
\define@key{myKeys}{label}{\def\mm@label{#1}}
\define@key{myKeys}{header}{\def\mm@header{#1}}
\define@key{myKeys}{FUNKTORY}{\def\mm@FUNKTORY{#1}}

%% one \ifcsname cannot directly generate more than a single table cell (problems with & and \\ characters)
%% so instead of spitting out the rows of the table immediately,
%% the content is added to @tabtoks via the command \addtabtoks
%% and printed when all parameters are read


\DeclareDocumentCommand{\Rule}{m}{
  \begingroup%
  \setkeys{myKeys}{#1}%reads the key=value pairs
  \medskip\noindent
  \begin{table}
  \addtabtoks{\begin{tabular}{|p{5cm}|l|}
                \hline}
  \ifcsname mm@nonnumbered\endcsname
               \addtabtoks{\multicolumn{2}{|l|}{{\bf\mm@header}}\\}
    \else      \refstepcounter{rule}
               \addtabtoks{\multicolumn{2}{|l|}{{\bf\mm@header}\hfill Rule \therule} \\}
    \fi
    \addtabtoks{\hline}
    \ifcsname mm@FUNKTORY\endcsname     \addtabtoks{\mm@FUNKTORY                                \\}\fi
    \ifcsname mm@OBLIG\endcsname        \addtabtoks{Obligatory          & \mm@OBLIG             \\}\fi
    \addtabtoks{\hline
                \end{tabular}}
    \printtabtoks

%         %%%%%%%% BEFORE RESETTING \@tabtoks, I'D LIKE TO STORE ITS FULLY EXPANDED VALUE
%           \expandafter\long\expandafter\edef\expandafter\tmp{\expandafter\write\expandafter\the\@tabtoks}
%           \show\tmp
%           \expandafter\addtoallrules\expandafter{\tmp}
%           \show\allrules

     \resettabtoks

  \ifcsname mm@label\endcsname \label{\mm@label}\fi
  \end{table}
  \smallskip
  \endgroup%
}
\makeatother

\begin{document}
\Rule{
  nonnumbered = {yes},
  header = {Rules in general},
  FUNKTORY = {VALUE 1           & VALUE 2},
}

\Rule{
  header = {Rule 1},
  FUNKTORY = {X           & Y},
  label = {rule:1},
}

\appendix
\allrules
\end{document}

Answer 1

It's not exactly clear what you want the final summary to look like, but this might get you started. As you see, the rules are all collected at the end.

What I do here is create a 2nd token register \globaltabtoks in parallel to the one employed by the OP, \@tabtoks. Then, I can add pieces-parts of the rules into the \globaltabtoks in the manner to construct a summary table. And I don't clear/reset the register between rules! So, for example, rather than setting a row between the title and the content, I merely employ a column separator.

The key is to expand the keys into their text before placing them into the token registers. Here I create \addglobaltabtoks{} to place literal text at the end of the \globaltabtoks register, and \xaddglobaltabtoks{} to expand-once the first token of the argument and then append it and the rest of the argument to \globaltabtoks.

In the case of \therule, I need more than a single expansion to get the desired number, so I used the new TeX primitive \expanded in the fashion of

\xaddglobaltabtoks{\expanded{\therule}&}

However, if your installation does not yet support \expanded, one could as an alternative use

\edef\tmp{\therule}
\xaddglobaltabtoks{\tmp&}

Here is the MWE:

\documentclass{book}
\usepackage[T1]{fontenc}
\usepackage{xparse,keyval}

\newcounter{rule}
\renewcommand{\therule}[0]{G\arabic{rule}}

\makeatletter
\newtoks\@tabtoks
\newtoks\globaltabtoks
\newcommand\addtabtoks[1]{\@tabtoks\expandafter{\the\@tabtoks#1}}
\newcommand\addglobaltabtoks[1]{%
  \global\globaltabtoks\expandafter{\the\globaltabtoks#1}%
}
\newcommand\xaddglobaltabtoks[1]{%
  \expandafter\addglobaltabtoks\expandafter{#1}%
}
\newcommand*\resettabtoks{\@tabtoks{}}
\newcommand*\resetglobaltabtoks{\globaltabtoks{}}
\newcommand*\printtabtoks{\the\@tabtoks}

\edef\allrules{}
\edef\oldallrules{\allrules}
\newcommand*\addtoallrules[1]{\g@addto@macro{\allrules}{#1}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%      PARAMETERS       %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\define@key{myKeys}{nonnumbered}{\def\mm@nonnumbered{#1}}
\define@key{myKeys}{label}{\def\mm@label{#1}}
\define@key{myKeys}{header}{\def\mm@header{#1}}
\define@key{myKeys}{FUNKTORY}{\def\mm@FUNKTORY{#1}}

%% one \ifcsname cannot directly generate more than a single table cell (problems with & and \\ characters)
%% so instead of spitting out the rows of the table immediately,
%% the content is added to @tabtoks via the command \addtabtoks
%% and printed when all parameters are read


\DeclareDocumentCommand{\Rule}{m}{
  \begingroup%
  \setkeys{myKeys}{#1}%reads the key=value pairs
  \medskip\noindent
  \begin{table}[ht]
  \addtabtoks{\begin{tabular}{|p{5cm}|l|}
                \hline}
  \ifcsname mm@nonnumbered\endcsname
               \addtabtoks{\multicolumn{2}{|l|}{{\bf\mm@header}}\\}
   \addglobaltabtoks{&}
    \else      \refstepcounter{rule}
               \addtabtoks{\multicolumn{2}{|l|}{{\bf\mm@header}\hfill Rule \therule} \\}
   \addglobaltabtoks{Rule }\xaddglobaltabtoks{\expanded{\therule}&}
    \fi
    \addtabtoks{\hline}
    \ifcsname mm@FUNKTORY\endcsname     \addtabtoks{\mm@FUNKTORY                                \\}
                                        \xaddglobaltabtoks{\mm@FUNKTORY                         \\}\fi
    \ifcsname mm@OBLIG\endcsname        \addtabtoks{Obligatory          & \mm@OBLIG             \\}
                                        \addglobaltabtoks{Obligatory    & }
                                        \xaddglobaltabtoks{\mm@OBLIG    \\}\fi
    \addtabtoks{\hline
                \end{tabular}}
    \addglobaltabtoks{\hline}
    \printtabtoks

     \resettabtoks

  \ifcsname mm@label\endcsname \label{\mm@label}\fi
  \end{table}
  \smallskip
  \endgroup%
}
\makeatother

\begin{document}
\Rule{
  nonnumbered = {yes},
  header = {Rules in general},
  FUNKTORY = {VALUE 1           & VALUE 2},
}

\Rule{
  header = {Rule 1},
  FUNKTORY = {X           & Y},
  label = {rule:1}
}

\Rule{
  header = {Rule 2},
  FUNKTORY = {P           & Q},
  label = {rule:2},
}

\appendix SUMMARY

\noindent
\begin{tabular}{|l|p{5cm}|l|}
                \hline
\the\globaltabtoks
\end{tabular}

\end{document}

Answer 2

I think what you want is something like what follows. Macros can record many things. Most things. Verbatim material is tricky, but you don't have any here, so you can just record the high-level commands with their arguments in a LaTeX3 sequence and use it in the appendix to replay everything (precisely, in \recapallrules). I only had to introduce an indirection level in two places:

• For the \label command: there must be only one label with a given name, so I use \mm@set@rule@label in \typesetRule. \mm@set@rule@label sets a label when used in the main text, but does nothing when used from \recapallrules, i.e., in the appendix (the behavior is trivial to change).

• For the counter incrementation: for the same reason, \mm@increment@rule@ctr does \refstepcounter{rule} when used in the main text and \stepcounter{rule} when used from \recapallrules (we don't want two different references—places recorded with a \label command—for the same value of the rule counter).

I also simplified your code by removing all the \addtabtoks-like things, which appear not to be necessary at all (unless proven otherwise!).

\documentclass{book}
\usepackage{keyval}
\usepackage{xparse}

\newcounter{rule}
\renewcommand{\therule}[0]{G\arabic{rule}}

\makeatletter

\define@key{myKeys}{nonnumbered}{\def\mm@nonnumbered{#1}}
\define@key{myKeys}{label}{\def\mm@label{#1}}
\define@key{myKeys}{header}{\def\mm@header{#1}}
\define@key{myKeys}{FUNKTORY}{\def\mm@FUNKTORY{#1}}

\newcommand*{\mm@normal@increment@rule@ctr}{\refstepcounter{rule}}
\newcommand*{\mm@appendix@increment@rule@ctr}{\stepcounter{rule}}
\let\mm@increment@rule@ctr=\mm@normal@increment@rule@ctr

\newcommand*{\mm@normal@rule@label@cmd}{%
  \ifcsname mm@label\endcsname
    \expandafter\label\expandafter{\mm@label}%
  \fi
}
\newcommand*{\mm@appendix@rule@label@cmd}{}
\let\mm@set@rule@label=\mm@normal@rule@label@cmd

\NewDocumentCommand{\typesetRule}{m}{% don't forget the percent sign here!
  \begingroup
  \setkeys{myKeys}{#1}%reads the key=value pairs
  \mm@set@rule@label
  \begin{tabular}{|p{5cm}|l|}
  \hline
  \ifcsname mm@nonnumbered\endcsname
     \multicolumn{2}{|l|}{{\bf\mm@header}}\\
  \else
     \multicolumn{2}{|l|}{{%
       \mm@increment@rule@ctr
       \bf\mm@header}\hfill Rule \therule} \\
  \fi
  \hline
  \ifcsname mm@FUNKTORY\endcsname
     \mm@FUNKTORY \\
  \fi
  \ifcsname mm@OBLIG\endcsname
     Obligatory & \mm@OBLIG \\
  \fi
  \hline
  \end{tabular}%
  \endgroup
}

\ExplSyntaxOn                   % Use LaTeX3-style input syntax (see expl3.pdf)

\seq_new:N \g_mm_rules_seq

\NewDocumentCommand \Rule { m }
  {
    \seq_gput_right:Nn \g_mm_rules_seq { \typesetRule {#1} }
    \medskip\noindent
    \typesetRule {#1}
  }

% Programming-level function
\cs_new_protected:Npn \mm_recap_all_rules:
  {
    \group_begin:               % limit effect of the \cs_set_eq:NN
    \setcounter {rule} {0}      % reset the rule counter
    % Special version of two commands for use here (we used different versions
    % in the main text). Note that \cs_set_eq:NN is like \let.
    \cs_set_eq:NN \mm@increment@rule@ctr \mm@appendix@increment@rule@ctr
    \cs_set_eq:NN \mm@set@rule@label \mm@appendix@rule@label@cmd
    % Use all elements we've recorded, separated using the provided separator
    \seq_use:Nn \g_mm_rules_seq { \par \medskip \noindent } % separator
    \group_end:
  }

% User-level function
\NewDocumentCommand \recapallrules { }
  {
    \mm_recap_all_rules:
  }

\ExplSyntaxOff                  % end of LaTeX3-style input syntax

\makeatother

\begin{document}

\Rule{
  nonnumbered = {yes},
  header = {Rules in general},
  FUNKTORY = {VALUE 1           & VALUE 2},
}

\Rule{
  header = {Rule 1},
  FUNKTORY = {X           & Y},
  label = {rule:1},
}

\Rule{
  header = {Rule 2},
  FUNKTORY = {Z           & T},
  label = {rule:2},
}

\appendix

\chapter{All rules}

\recapallrules

\end{document}

On page 1:

Appendix:

Answer 3

I wouldn't mix “legacy” programming with expl3. With careful usage of variants, it's easy to control what gets expanded and what doesn't.

\documentclass{book}
\usepackage{xparse}

\newcounter{rule}
\renewcommand{\therule}[0]{G\arabic{rule}}

\ExplSyntaxOn

\NewDocumentCommand{\Rule}{m}
 {
  \group_begin:
  \tl_clear:N \l__ansa_rule_body_tl
  \keys_set:nn { ansa/rule } { #1 }
  \__ansa_rule:
  \tl_gput_right:NV \g__ansa_rule_all_tl \l__ansa_rule_body_tl 
  \tl_gput_right:Nn \g__ansa_rule_all_tl { \par\bigskip }
  \group_end:
 }
\NewDocumentCommand{\allrules}{}
 {
  \begin{center}
  \cs_set_eq:NN \__ansa_rule_label:n \use_none:n
  \tl_use:N \g__ansa_rule_all_tl
  \end{center}
 }

\tl_new:N \l__ansa_rule_body_tl
\tl_new:N \g__ansa_rule_all_tl

\keys_define:nn { ansa/rule }
 {
  nonnumbered .bool_set:N = \l__ansa_rule_nonnumbered_bool,
  nonnumbered .default:n  = true,
  nonnumbered .initial:n  = false,
  label       .tl_set:N   = \l__ansa_rule_label_tl,
  header      .tl_set:N   = \l__ansa_rule_header_tl,
  FUNKTORY    .tl_set:N   = \l__ansa_rule_funktory_tl,
  OBLIG       .tl_set:N   = \l__ansa_rule_oblig_tl,
 }

\cs_new_protected:Nn \__ansa_rule:
 {
  \__ansa_rule_add:n { \begin{tabular}{|p{5cm}|l|}\hline }
  \bool_if:NTF \l__ansa_rule_nonnumbered_bool
   {
    \__ansa_rule_add_multi:V \l__ansa_rule_header_tl
   }
   {
    \refstepcounter{rule}
    \tl_if_empty:NF \l__ansa_rule_label_tl { \__ansa_rule_label:n {\l__ansa_rule_label_tl} }
    \__ansa_rule_add_multi_number:Vx \l__ansa_rule_header_tl { \therule }
   }
  \__ansa_rule_add:n { \hline }
  \tl_if_empty:NF \l__ansa_rule_funktory_tl
   {
    \__ansa_rule_add:V \l__ansa_rule_funktory_tl
    \__ansa_rule_add:n { \\ }
   }
  \tl_if_empty:NF \l__ansa_rule_oblig_tl
   {
    \__ansa_rule_add:V \l__ansa_rule_oblig_tl
    \__ansa_rule_add:n { \\ }
   }
  \__ansa_rule_add:n { \hline\end{tabular} }
  \tl_use:N \l__ansa_rule_body_tl
}

\cs_new:Nn \__ansa_rule_label:n { \label{#1} }

\cs_new_protected:Nn \__ansa_rule_add:n
 {
  \tl_put_right:Nn \l__ansa_rule_body_tl { #1 }
 }
\cs_generate_variant:Nn \__ansa_rule_add:n { V }

\cs_new_protected:Nn \__ansa_rule_add_multi_number:nn
 {
  \__ansa_rule_add:n { \multicolumn{2}{|l|}{\bfseries#1\hfill Rule~#2} \\ }
 }
\cs_generate_variant:Nn \__ansa_rule_add_multi_number:nn { Vx }

\cs_new_protected:Nn \__ansa_rule_add_multi:n
 {
  \__ansa_rule_add:n { \multicolumn{2}{|l|}{\bfseries#1} \\ }
 }
\cs_generate_variant:Nn \__ansa_rule_add_multi:n { V }

\ExplSyntaxOff

\begin{document}

\Rule{
  nonnumbered,
  header = {Rules in general},
  FUNKTORY = {VALUE 1           & VALUE 2},
}

\Rule{
  header = {Rule 1},
  FUNKTORY = {X           & Y},
  label = {rule:1},
}

\allrules

\end{document}