In order to avoid using explicit distances, I'd use the following code.
\documentclass[12pt,a4paper]{article}
% CODE BELOW FROM https://tex.stackexchange.com/a/1070/128068
% Load TikZ
\usepackage{tikz}
\usetikzlibrary{matrix,decorations.pathreplacing,calc}
% Set various styles for the matrices and braces. It might pay off to fiddle around with the values a little bit
\pgfkeys{tikz/mymatrixenv/.style={decoration=brace,every left delimiter/.style={xshift=3pt},every right delimiter/.style={xshift=-3pt}}}
\pgfkeys{tikz/mymatrix/.style={matrix of math nodes,left delimiter=[,right delimiter={]},inner sep=2pt,column sep=1em,row sep=0.5em,nodes={inner sep=0pt}}}
\pgfkeys{tikz/mymatrixbrace/.style={decorate,thick}}
\newcommand\mymatrixbraceoffseth{0.5em}
\newcommand\mymatrixbraceoffsetv{0.2em}
\newcommand*\mymatrixbraceright[4][m]{
\draw[mymatrixbrace] ($(#1.north west)!(#1-#3-1.south west)!(#1.south west)-(\mymatrixbraceoffseth,0)$)
-- node[left=2pt] {#4}
($(#1.north west)!(#1-#2-1.north west)!(#1.south west)-(\mymatrixbraceoffseth,0)$);
}
\newcommand*\mymatrixbraceleft[4][m]{
\draw[mymatrixbrace] ($(#1.north east)!(#1-#2-1.north east)!(#1.south east)+(\mymatrixbraceoffseth,0)$)
-- node[right=2pt] {#4}
($(#1.north east)!(#1-#3-1.south east)!(#1.south east)+(\mymatrixbraceoffseth,0)$);
}
\newcommand*\mymatrixbracetop[4][m]{
\draw[mymatrixbrace] ($(#1.north west)!(#1-1-#2.north west)!(#1.north east)+(0,\mymatrixbraceoffsetv)$)
-- node[above=2pt] {#4}
($(#1.north west)!(#1-1-#3.north east)!(#1.north east)+(0,\mymatrixbraceoffsetv)$);
}
\newcommand*\mymatrixbracebottom[4][m]{
\draw[mymatrixbrace] ($(#1.south west)!(#1-1-#3.south east)!(#1.south east)-(0,\mymatrixbraceoffsetv)$)
-- node[below=2pt] {#4}
($(#1.south west)!(#1-1-#2.south west)!(#1.south east)-(0,\mymatrixbraceoffsetv)$);
}
% CHEATING DASH FROM https://tex.stackexchange.com/a/133357/128068
\tikzset{
cheating dash/.code args={on #1 off #2}{
% Use csname so catcode of @ doesn't have do be changed.
\csname tikz@addoption\endcsname{%
\pgfgetpath\currentpath%
\pgfprocessround{\currentpath}{\currentpath}%
\csname pgf@decorate@parsesoftpath\endcsname{\currentpath}{\currentpath}%
\pgfmathparse{\csname pgf@decorate@totalpathlength\endcsname-#1}\let\rest=\pgfmathresult%
\pgfmathparse{#1+#2}\let\onoff=\pgfmathresult%
\pgfmathparse{max(floor(\rest/\onoff), 1)}\let\nfullonoff=\pgfmathresult%
\pgfmathparse{max((\rest-\onoff*\nfullonoff)/\nfullonoff+#2, #2)}\let\offexpand=\pgfmathresult%
\pgfsetdash{{#1}{\offexpand}}{0pt}}%
}
}
\begin{document}
\begin{equation}
\mathbf{X} =
\begin{tikzpicture}[baseline=0cm,mymatrixenv]
\matrix [mymatrix,inner sep=4pt,row sep=1em] (m)
{
\frac{x}{y} + C & m\omega^2 + U \\
0 & 1 \\
1 & \frac{x}{y} + Dx^2 \\
};
\path (m-1-1.east) -- (m-1-2.west) coordinate[midway] (X)
(m-1-1.south) -- (m-2-1.north) coordinate[midway] (Y);
% Dashed lines
\draw [cheating dash=on 2pt off 2pt,red]
(X |- m.north) -- (X |- m.south);
\draw [cheating dash=on 2pt off 2pt,red]
(Y -| m.west) -- (Y -| m.east);
% Braces
\mymatrixbraceright{1}{3}{$A$}
\mymatrixbracetop{2}{2}{$B$}
\end{tikzpicture}
\end{equation}
\end{document}
EDIT: Using your own suggestion (with some minor modifications), the code can be simplified to:
\documentclass[12pt,a4paper]{article}
% CODE BELOW FROM https://tex.stackexchange.com/a/1070/128068
% Load TikZ
\usepackage{tikz}
\usetikzlibrary{matrix,decorations.pathreplacing,calc}
% Set various styles for the matrices and braces. It might pay off to fiddle around with the values a little bit
\pgfkeys{tikz/mymatrixenv/.style={decoration=brace,every left delimiter/.style={xshift=3pt},every right delimiter/.style={xshift=-3pt}}}
\pgfkeys{tikz/mymatrix/.style={matrix of math nodes,left delimiter=[,right delimiter={]},inner sep=2pt,column sep=1em,row sep=0.5em,nodes={inner sep=0pt}}}
\pgfkeys{tikz/mymatrixbrace/.style={decorate,thick}}
\newcommand\mymatrixbraceoffseth{0.5em}
\newcommand\mymatrixbraceoffsetv{0.2em}
\newcommand*\mymatrixbraceright[4][m]{
\draw[mymatrixbrace] ($(#1.north west)!(#1-#3-1.south west)!(#1.south west)-(\mymatrixbraceoffseth,0)$)
-- node[left=2pt] {#4}
($(#1.north west)!(#1-#2-1.north west)!(#1.south west)-(\mymatrixbraceoffseth,0)$);
}
\newcommand*\mymatrixbraceleft[4][m]{
\draw[mymatrixbrace] ($(#1.north east)!(#1-#2-1.north east)!(#1.south east)+(\mymatrixbraceoffseth,0)$)
-- node[right=2pt] {#4}
($(#1.north east)!(#1-#3-1.south east)!(#1.south east)+(\mymatrixbraceoffseth,0)$);
}
\newcommand*\mymatrixbracetop[4][m]{
\draw[mymatrixbrace] ($(#1.north west)!(#1-1-#2.north west)!(#1.north east)+(0,\mymatrixbraceoffsetv)$)
-- node[above=2pt] {#4}
($(#1.north west)!(#1-1-#3.north east)!(#1.north east)+(0,\mymatrixbraceoffsetv)$);
}
\newcommand*\mymatrixbracebottom[4][m]{
\draw[mymatrixbrace] ($(#1.south west)!(#1-1-#3.south east)!(#1.south east)-(0,\mymatrixbraceoffsetv)$)
-- node[below=2pt] {#4}
($(#1.south west)!(#1-1-#2.south west)!(#1.south east)-(0,\mymatrixbraceoffsetv)$);
}
% CHEATING DASH FROM https://tex.stackexchange.com/a/133357/128068
\tikzset{
cheating dash/.code args={on #1 off #2}{
% Use csname so catcode of @ doesn't have do be changed.
\csname tikz@addoption\endcsname{%
\pgfgetpath\currentpath%
\pgfprocessround{\currentpath}{\currentpath}%
\csname pgf@decorate@parsesoftpath\endcsname{\currentpath}{\currentpath}%
\pgfmathparse{\csname pgf@decorate@totalpathlength\endcsname-#1}\let\rest=\pgfmathresult%
\pgfmathparse{#1+#2}\let\onoff=\pgfmathresult%
\pgfmathparse{max(floor(\rest/\onoff), 1)}\let\nfullonoff=\pgfmathresult%
\pgfmathparse{max((\rest-\onoff*\nfullonoff)/\nfullonoff+#2, #2)}\let\offexpand=\pgfmathresult%
\pgfsetdash{{#1}{\offexpand}}{0pt}}%
}
}
\newcommand*\mymatrixdashedlineH[4][]{
\path (#2-#3-1.south) -- (#2-#4-1.north)
coordinate [midway] (Y); \draw [#1] (Y -| #2.west) -- (Y -| #2.east);}
\newcommand*\mymatrixdashedlineV[4][]{ \path (#2-1-#3.east) -- (#2-1-#4.west)
coordinate [midway] (X); \draw [#1] (X |- #2.north) -- (X |- #2.south);}
\begin{document}
\begin{equation}
\mathbf{X} =
\begin{tikzpicture}[baseline=0cm,mymatrixenv]
\matrix [mymatrix,inner sep=4pt,row sep=1em] (m)
{
\frac{x}{y} + C & m\omega^2 + U \\
0 & 1 \\
1 & \frac{x}{y} + Dx^2 \\
};
\path (m-1-1.east) -- (m-1-2.west) coordinate[midway] (X)
(m-1-1.south) -- (m-2-1.north) coordinate[midway] (Y);
% Dashed lines
\mymatrixdashedlineV[red,dashed]{m}{1}{2}
\mymatrixdashedlineH[red,dashed]{m}{1}{2}
% Braces
\mymatrixbraceright{1}{3}{$A$}
\mymatrixbracetop{2}{2}{$B$}
\end{tikzpicture}
\end{equation}
\end{document}
EDIT: Just for fun: some TikZy versions of your nice proposals. (NOTE: I am not saying that this is a safe path since I am using \pgfextra
. It is really just for fun.)
\documentclass[12pt,a4paper]{article}
% CODE BELOW FROM https://tex.stackexchange.com/a/1070/128068
% Load TikZ
\usepackage{tikz}
\usetikzlibrary{matrix,decorations.pathreplacing,calc}
% Set various styles for the matrices and braces. It might pay off to fiddle around with the values a little bit
\pgfkeys{tikz/mymatrixenv/.style={decoration=brace,every left delimiter/.style={xshift=3pt},every right delimiter/.style={xshift=-3pt}}}
\pgfkeys{tikz/mymatrix/.style={matrix of math nodes,left delimiter=[,right
delimiter={]},inner sep=2pt,column sep=1em,row sep=0.5em,nodes={inner
sep=0pt}}}
\pgfkeys{tikz/mymatrixbrace/.style={decorate,thick}}
\newcommand\mymatrixbraceoffseth{0.5em}
\newcommand\mymatrixbraceoffsetv{0.2em}
\newcommand*\mymatrixbraceright[4][m]{
\draw[mymatrixbrace] ($(#1.north west)!(#1-#3-1.south west)!(#1.south west)-(\mymatrixbraceoffseth,0)$)
-- node[left=2pt] {#4}
($(#1.north west)!(#1-#2-1.north west)!(#1.south west)-(\mymatrixbraceoffseth,0)$);
}
\newcommand*\mymatrixbraceleft[4][m]{
\draw[mymatrixbrace] ($(#1.north east)!(#1-#2-1.north east)!(#1.south east)+(\mymatrixbraceoffseth,0)$)
-- node[right=2pt] {#4}
($(#1.north east)!(#1-#3-1.south east)!(#1.south east)+(\mymatrixbraceoffseth,0)$);
}
\newcommand*\mymatrixbracetop[4][m]{
\draw[mymatrixbrace] ($(#1.north west)!(#1-1-#2.north west)!(#1.north east)+(0,\mymatrixbraceoffsetv)$)
-- node[above=2pt] {#4}
($(#1.north west)!(#1-1-#3.north east)!(#1.north east)+(0,\mymatrixbraceoffsetv)$);
}
\newcommand*\mymatrixbracebottom[4][m]{
\draw[mymatrixbrace] ($(#1.south west)!(#1-1-#3.south east)!(#1.south east)-(0,\mymatrixbraceoffsetv)$)
-- node[below=2pt] {#4}
($(#1.south west)!(#1-1-#2.south west)!(#1.south east)-(0,\mymatrixbraceoffsetv)$);
}
% CHEATING DASH FROM https://tex.stackexchange.com/a/133357/128068
\tikzset{
cheating dash/.code args={on #1 off #2}{
% Use csname so catcode of @ doesn't have do be changed.
\csname tikz@addoption\endcsname{%
\pgfgetpath\currentpath%
\pgfprocessround{\currentpath}{\currentpath}%
\csname pgf@decorate@parsesoftpath\endcsname{\currentpath}{\currentpath}%
\pgfmathparse{\csname pgf@decorate@totalpathlength\endcsname-#1}\let\rest=\pgfmathresult%
\pgfmathparse{#1+#2}\let\onoff=\pgfmathresult%
\pgfmathparse{max(floor(\rest/\onoff), 1)}\let\nfullonoff=\pgfmathresult%
\pgfmathparse{max((\rest-\onoff*\nfullonoff)/\nfullonoff+#2, #2)}\let\offexpand=\pgfmathresult%
\pgfsetdash{{#1}{\offexpand}}{0pt}}%
}
}
\tikzset{
vertical line/.style args={with style #1 right of column #2 in
row #3}{
append after command={
\pgfextra{ \pgfmathparse{int(#2+1)}
\path (\tikzlastnode-#3-#2.east) -- (\tikzlastnode-#3-\pgfmathresult.west) coordinate [midway] (MiloX);
\draw [#1] (MiloX |- \tikzlastnode.north) -- (MiloX |- \tikzlastnode.south);
}
}
},
horizontal line/.style args={with style #1 below row #2 in
column #3}{append after command={\pgfextra{ \pgfmathparse{int(#2+1)}
\path (\tikzlastnode-#2-#3.south) -- (\tikzlastnode-\pgfmathresult-#3.north) coordinate [midway] (MiloY);
\draw [#1] (MiloY -| \tikzlastnode.west) -- (MiloY -| \tikzlastnode.east);
}}
}
}
\begin{document}
\begin{equation}
\mathbf{X} =
\begin{tikzpicture}[baseline=0cm,mymatrixenv]
\matrix [mymatrix,inner sep=4pt,row sep=1em,
vertical line={with style {dashed,red} right of column 1 in row 1},
horizontal line={with style {dashed,red} below row 2 in column 1}] (m)
{
\frac{x}{y} + C & m\omega^2 + U \\
0 & 1 \\
1 & \frac{x}{y} + Dx^2 \\
};
\path (m-1-1.east) -- (m-1-2.west) coordinate[midway] (X)
(m-1-1.south) -- (m-2-1.north) coordinate[midway] (Y);
% Braces
\mymatrixbraceright{1}{3}{$A$}
\mymatrixbracetop{2}{2}{$B$}
\end{tikzpicture}
\end{equation}
\end{document}