Nice curved brackets (in the shape of \prec and \succ)

Question

Does anyone know how I can use the following style of angled brackets?

Answer 1

With just pdflatex you can use the STIX fonts and build the big version with picture:

\documentclass{article}
\usepackage{amsmath,pict2e}

\DeclareFontEncoding{LS1}{}{}
\DeclareFontSubstitution{LS1}{stix}{m}{n}
\DeclareSymbolFont{symbols2}{LS1}{stixfrak}{m}{n}

\DeclareMathSymbol{\lcurvyangle}{\mathopen}{symbols2}{"E9}
\DeclareMathSymbol{\rcurvyangle}{\mathclose}{symbols2}{"EA}

\DeclareRobustCommand{\biglcurvyangle}{\mathopen{\makebiglcurvy}}
\DeclareRobustCommand{\bigrcurvyangle}{\mathclose{\makebigrcurvy}}

\newcommand{\makebiglcurvy}{%
  \vcenter{\hbox{%
    \sbox0{$\bigg|$}%
    \setlength{\unitlength}{.5\ht0}%
    \addtolength{\unitlength}{.5\dp0}%
    \begin{picture}(1,2)
    \linethickness{0.6pt}\roundcap
    \put(0,0){\arc[0,90]{1}}
    \put(0,2){\arc[270,360]{1}}
    \end{picture}%
  }}%
}

\newcommand{\makebigrcurvy}{%
  \vcenter{\hbox{%
    \sbox0{$\bigg|$}%
    \setlength{\unitlength}{.5\ht0}%
    \addtolength{\unitlength}{.5\dp0}%
    \begin{picture}(1,2)
    \linethickness{0.6pt}\roundcap
    \put(1,0){\arc[90,180]{1}}
    \put(1,2){\arc[180,270]{1}}
    \end{picture}%
  }}%
}

\begin{document}
$dF^1_{\lcurvyangle\alpha,\beta\rcurvyangle}(\xi)$

if $\lVert\lcurvyangle\zeta,\eta\rcurvyangle\rVert<1$
\[
\biglcurvyangle
  q_1,\dots,q_n,
  \frac{\partial V}{\partial q_1},\dots,\frac{\partial V}{\partial q_n},
  t
\bigrcurvyangle
\]
\end{document}

An implementation with only curves drawn in picture mode that also adds larger variants.

\documentclass{article}
\usepackage{amsmath,pict2e}

\DeclareRobustCommand{\lcurvyangle}{%
  \mathopen{%
    \nonscript\mskip2mu
    \text{\makelcurvy}%
    \nonscript\mskip2mu
  }%
}
\DeclareRobustCommand{\rcurvyangle}{%
  \mathclose{%
    \nonscript\mskip2mu
    \text{\makercurvy}%
    \nonscript\mskip2mu
  }%
}
\DeclareRobustCommand{\biglcurvyangle}{\mathopen{\makebiglcurvy{0}}}
\DeclareRobustCommand{\bigrcurvyangle}{\mathclose{\makebigrcurvy{0}}}
\DeclareRobustCommand{\Biglcurvyangle}{\mathopen{\makebiglcurvy{1}}}
\DeclareRobustCommand{\Bigrcurvyangle}{\mathclose{\makebigrcurvy{1}}}
\DeclareRobustCommand{\bigglcurvyangle}{\mathopen{\makebiglcurvy{2}}}
\DeclareRobustCommand{\biggrcurvyangle}{\mathclose{\makebigrcurvy{2}}}


\newcommand{\makelcurvy}{%
  \sbox0{$\langle$}%
  \setlength{\unitlength}{.4\ht0}%
  \begin{picture}(2,2)
  \linethickness{0.5pt}\roundcap
  \put(0,-1){\arc[26.5,90]{2}}
  \put(0,3){\arc[270,333.5]{2}}
  \end{picture}%
}

\newcommand{\makercurvy}{%
  \sbox0{$\langle$}%
  \setlength{\unitlength}{.4\ht0}%
  \begin{picture}(2,2)
  \linethickness{0.5pt}\roundcap
  \put(2,-1){\arc[90,153,5]{2}}
  \put(2,3){\arc[206.5,270]{2}}
  \end{picture}%
}

\newcommand{\makebiglcurvy}[1]{%
  \vcenter{\hbox{%
    \sbox0{$\bigg|$}%
    \setlength{\unitlength}{.25\ht0}%
    \addtolength{\unitlength}{.25\dp0}%
    \begin{picture}(2,\numexpr4+2*#1\relax)
    \linethickness{0.6pt}\roundcap
    \put(0,\numexpr#1\relax){\arc[0,90]{2}}
    \put(0,\numexpr4+#1\relax){\arc[270,360]{2}}
    \put(2,0){\line(0,1){#1}}
    \put(2,\numexpr4+#1\relax){\line(0,1){#1}}
    \end{picture}%
  }}%
}

\newcommand{\makebigrcurvy}[1]{%
  \vcenter{\hbox{%
    \sbox0{$\bigg|$}%
    \setlength{\unitlength}{.25\ht0}%
    \addtolength{\unitlength}{.25\dp0}%
    \begin{picture}(2,\numexpr4+2*#1\relax)
    \linethickness{0.6pt}\roundcap
    \put(2,\numexpr#1\relax){\arc[90,180]{2}}
    \put(2,\numexpr4+#1\relax){\arc[180,270]{2}}
    \put(0,0){\line(0,1){#1}}
    \put(0,\numexpr4+#1\relax){\line(0,1){#1}}
    \end{picture}%
  }}%
}

\begin{document}
$dF^1_{\lcurvyangle\alpha,\beta\rcurvyangle}(\xi)$

if $\lVert\lcurvyangle\zeta,\eta\rcurvyangle\rVert<1$
\[
\biglcurvyangle
  q_1,\dots,q_n,
  \frac{\partial V}{\partial q_1},\dots,\frac{\partial V}{\partial q_n},
  t
\bigrcurvyangle
\]

\[
\Biglcurvyangle
\begin{bmatrix}1\\2\\3\end{bmatrix}
\Bigrcurvyangle
\qquad
\bigglcurvyangle
\begin{bmatrix}1\\2\\3\\4\end{bmatrix}
\biggrcurvyangle
\]

\end{document}

Answer 2

Sorry, the code is far away from beautiful but it shows which unicodes you may use for this and how do define your brackets with those. In the end, you see 4 different fonts, which you may choose. You can find more of them here.

% arara: lualatex

\documentclass{article}
\usepackage{fontspec}
\newcommand*{\myL}{\symbol{"29FC}}
\newcommand*{\myR}{\symbol{"29FD}}
\newcommand*{\mySmallL}{\symbol{"227A}}
\newcommand*{\mySmallR}{\symbol{"227B}}

\usepackage{mathtools}
\DeclarePairedDelimiter\myBrak{\text{{\fontspec{stix-regular.otf}\symbol{"227A}}}}{{\text{\fontspec{stix-regular.otf}\symbol{"227B}}}}
\DeclarePairedDelimiter\myBigBrak{\text{{\fontspec{stix-regular.otf}\symbol{"29FC}}}}{{\text{\fontspec{stix-regular.otf}\symbol{"29FD}}}}   
% note: you wont be able to use the starred version of these commands. 

\begin{document}
        $-dF^1_{\myBrak{\alpha,\beta}}(\zeta)$,

        $\in\qquad\text{if}\quad\left\|\myBrak{\zeta,\eta}\right\|$

        $=\myBigBrak{q_1, \dots, q_n, \frac{\partial V}{\partial q_1}, \dots, \frac{\partial V}{\partial q_n}, t}$

    \bigskip
    {\fontspec{quivira.otf}\myL{\fontspec{code2000.ttf}\myL{\fontspec{stix-regular.otf}\myL{\fontspec{Cambria Math}\myL\myR}\myR}\myR}\myR}

    {\fontspec{quivira.otf}\mySmallL{\fontspec{code2000.ttf}\mySmallL{\fontspec{stix-regular.otf}\mySmallL{\fontspec{Cambria Math}\mySmallL\mySmallR}\mySmallR}\mySmallR}\mySmallR} 
\end{document}