The size command is only applied to the first delimiter, not to the following ones; there's no error message, but what you get is a byproduct of the actual implementation.
You can use \DeclarePairedDelimiterX
:
\documentclass{article}
\usepackage{mathtools}
\usepackage{unicode-math}
\setmathfont{STIX Two Math}
\DeclarePairedDelimiterX{\double}[1]{(}{)}{\delimsize(#1\delimsize)}
\DeclarePairedDelimiterX{\triple}[1]{\{}{]}{%
\delimsize[\delimsize(#1\delimsize\rangle\delimsize)%
}
\DeclarePairedDelimiterX{\dif}[1]{(}{)}{#1\delimsize)}
\begin{document}
\begin{align}
\double{x}\\
\double*{\dfrac{n}{k}}\\
\triple{x}\\
\triple*{\dfrac{a}{b}}\\
\dif*{\dfrac{x}{y}}
\end{align}
\end{document}
A full reimplementation, but the numbers of delimiters must match, use .
for balancing.
\documentclass{article}
\usepackage{mathtools,mleftright}
\usepackage{unicode-math}
\setmathfont{STIX Two Math}
\ExplSyntaxOn
\NewDocumentCommand{\DeclareMultiPairedDelimiter}{mmm}
{
\NewDocumentCommand{#1}{sO{}m}
{
\IfBooleanTF{##1}
{ \fraiman_multi_open_lr:n { #2 } }
{ \fraiman_multi_open:nn { ##2 } { #2 } }
##3
\IfBooleanTF{##1}
{ \fraiman_multi_close_lr:n { #3 } }
{ \fraiman_multi_close:nn { ##2 }{ #3 } }
}
}
\cs_new_protected:Nn \fraiman_multi_open_lr:n
{
\tl_map_inline:nn { #1 } { \mleft ##1 }
}
\cs_new_protected:Nn \fraiman_multi_close_lr:n
{
\tl_map_inline:nn { #1 } { \mright ##1 }
}
\cs_new_protected:Nn \fraiman_multi_open:nn
{
\mathopen { \tl_map_inline:nn { #2 } {\tl_if_eq:nnF { ##1 } { . } { #1 ##1 } } }
}
\cs_new_protected:Nn \fraiman_multi_close:nn
{
\mathclose { \tl_map_inline:nn { #2 } {\tl_if_eq:nnF { ##1 } { . } { #1 ##1 } } }
}
\ExplSyntaxOff
\DeclareMultiPairedDelimiter{\double}{((}{))}
\DeclareMultiPairedDelimiter{\triple}{\{[(}{>)]}
\DeclareMultiPairedDelimiter{\dif}{(.}{))}
\begin{document}
\begin{align}
\double{x}\\
\double*{\dfrac{n}{k}}\\
\triple{x}\\
\triple*{\dfrac{a}{b}}\\
\dif*{\dfrac{x}{y}}
\end{align}
\end{document}