To get original amsmath-brackets, I worked with some tricks due to length-measurement of the calc-library:

\documentclass[border=5mm, varwidth]{standalone}
\usepackage{amsmath}
\usepackage{tikz}
\usetikzlibrary{matrix}
\usetikzlibrary{calc}
\pgfkeys{/tikz/savevalue/.code 2 args={\global\edef#1{#2}}}
%\pgfmathsetlengthmacro{\InnerSep}{1pt} % optional 1/2
\begin{document}
\section{amsmath-Brackets inside a TikZ-matrix}
\begin{tikzpicture}[]
\matrix[matrix of math nodes, nodes in empty cells,
row sep=5mm, column sep=3mm,
nodes={draw=none, % Test
%inner sep=\InnerSep % optional 2/2
},
% Maybe useful:
%every odd row/.style={anchor=base west},% <--- don't know, how
%every even row/.style={anchor=base west},% to make this shorter
](m){
a11 a2& a12 & a13 & a14 & a15 \\
a21 & a22 & a23 & a24 & a25 \\
a31 & a32 & a33 & a34 & a35 \\
a41 & a42 & a43 & a44 & a45 \\
a51 & a52 & a53 & a54 & a55 \\
a61 & a62 & a63 & a64 & a65 \\
a71 & a72 & a73 & a74 & a75 \\
};
% Brackets around A
\coordinate[](Aul) at (m-5-1.north west);
\coordinate[](All) at (m-7-1.south west);
\coordinate[](Aur) at (m-5-3.north east);
\coordinate[](Alr) at (m-7-3.south east);
\path let \p1=($(Aul)-(All)$) in
\pgfextra{ \pgfmathsetlengthmacro{\templength}{veclen(\x1,\y1)} }
[savevalue={\u}{\templength}];
\pgfmathsetlengthmacro{\uHalf}{0.5*\u}
\node[] at ($(Aul)!0.5!(All)$) {$
\left( \rule[-\uHalf]{0mm}{\u} \right.$};
\node[] at ($(Aur)!0.5!(Alr)$) {$
\left. \rule[-\uHalf]{0mm}{\u} \right)$};
% Brackets around B
\coordinate[](Bul) at (m-1-4.north west);
\coordinate[](Bll) at (m-2-4.south west);
\coordinate[](Bur) at (m-1-5.north east);
\coordinate[](Blr) at (m-2-5.south east);
\path let \p1=($(Bul)-(Bll)$) in
\pgfextra{ \pgfmathsetlengthmacro{\templength}{veclen(\x1,\y1)} }
[savevalue={\u}{\templength}];
\pgfmathsetlengthmacro{\uHalf}{0.5*\u*1}
\node[blue] at ($(Bul)!0.5!(Bll)$) {$
\left( \rule[-\uHalf]{0mm}{\u} \right.$};
\node[blue] at ($(Bur)!0.5!(Blr)$) {$
\left. \rule[-\uHalf]{0mm}{\u} \right)$};
\end{tikzpicture}
\section{For Comparison: Brackets at amsmath-Matrix}
$\begin{pmatrix}
a51 & a52 & a53 \\[5mm]
a61 & a62 & a63 \\[5mm]
a71 & a72 & a73
\end{pmatrix}$
\end{document}