Matrix using tikz package

Question

I am working on my dissertation and I have no idea how I should do this in LaTeX.

And the shading definitely is challenging.

I have only been able to do this so far, I am still an amateur at it.

\usepackage{color}
\newcommand\x{\times}
\newcommand\y{\cellcolor{green}}

\begin{document}
\begin{equation*}
   \left[
  \begin{array}{ccc}
\y & \y    & \y   \\
   & \y   & \y R  \\
   &       & \y   \\
\end{array}
 \right]\left[
     \begin{array}{c}
        \\
       a_j \\
        \\
     \end{array}
   \right]=
\end{equation*}
\end{document}

Answer 1

With TikZ matrix:

\documentclass{article}
\usepackage{amsmath}
\usepackage{tikz}
\usetikzlibrary{matrix,  positioning, patterns, backgrounds}
\tikzset{
    every matrix/.append style={
        matrix of math nodes,
        nodes in empty cells,
        inner sep=0pt,
        outer sep=0pt,
        column sep=-\pgflinewidth,
        row sep=-\pgflinewidth,
        },
    mylargenode/.style={
        text centered,
        text width=30pt,
        text height=20pt,
        text depth=10pt,
    },
    mylongnode/.style={
        text centered,
        text width=20pt,
        text height=40pt,
        text depth=30pt,
    },
    mysmallnode/.style={
        text centered,
        text width=20pt,
        text height=20pt,
        text depth=10pt,
    },
    mylonglargenode/.style={
        text centered,
        text width=30pt,
        text height=40pt,
        text depth=30pt,
    },
}

\begin{document}
With delimiters like in your image:
\[
\begin{tikzpicture}
\matrix[
    nodes={mylargenode},
    column 6/.style={nodes={mysmallnode}},
    row 6/.style={nodes={mylonglargenode}}
    ] (A) {%
    &&&&&|[draw=blue, fill=blue!10!white]| \\
    &&&&&|[draw=blue, fill=blue!30!white]|\\
    &&&&&|[draw=blue, fill=blue!50!white]|\\
    &&&&&|[draw=blue,dotted]| \\
    &&&&&|[draw=blue, fill=blue!70!white]|\\
    &&&&& |[mylongnode,pattern=crosshatch,pattern color=blue!40!white, draw=blue]|\Dot{F}\\
    };
\begin{scope}[on background layer]
\foreach \ind in {1,2,3,5}
    {
    \draw[green!70!black,fill=green!40!lightgray] (A-\ind-\ind.north west) -- (A-\ind-\ind.north east) -- (A-\ind-\ind.south east) -- cycle;
    \node[anchor=north east] at (A-\ind-\ind.north east) {R};
    }
\end{scope}
\draw[dotted, green!70!black] (A-4-4.north west) -- (A-4-4.north east) -- (A-4-4.south east) -- cycle;
\node[font=\Huge,shift={(-1,-2)}] at (A) {$0$}; 
\node[font=\Huge,shift={(1,3)}] at (A) {$0$};
\matrix[
    right =of A.north east,
    anchor=north west,
    nodes={mysmallnode}
    ] (B) {%
    \underline{a}_1  \\
    \underline{a}_2 \\
    \underline{a}_3 \\
    \\
    \underline{a}_M\\
    \underline{b}\\
    };
\foreach \ind in {1,2,...,5}{
    \draw (B-\ind-1.south west) -- (B-\ind-1.south east);
}
\draw[dotted] (B-4-1.north) -- (B-4-1.south);
\matrix[
    right =4em of B.north east,
    anchor=north west,
    nodes={draw,mysmallnode}
    ] (C) {%
    |[fill=yellow!10!white]|\\
    |[fill=yellow!30!white]|\\
    |[fill=yellow!50!white]|\\
    \\
    |[fill=yellow!70!white]|\\
    |[mylongnode, pattern=horizontal lines, pattern color=yellow!90!white]|\underline{\Dot{f}}\\
    };
\path (B-4-1) -- node[midway] {$=$} (C-4-1);
\draw[dotted] (C-4-1.north) -- (C-4-1.south);
% delimiters:
\foreach \mymatr in {A,B,C}
{
\draw[very thick] ([xshift=4pt]\mymatr.north west) -- (\mymatr.north west) -- (\mymatr.south west) -- ([xshift=4pt]\mymatr.south west);
\draw[very thick] ([xshift=-4pt]\mymatr.north east) -- (\mymatr.north east) -- (\mymatr.south east) -- ([xshift=-4pt]\mymatr.south east);
}
\end{tikzpicture}
\]

Or with ordinary delimiter:
\[
\begin{tikzpicture}[   
    every matrix/.append style={
        left delimiter={[},
        right delimiter={]},
        }
 ]
\matrix[
    nodes={mylargenode},
    column 6/.style={nodes={mysmallnode}},
    row 6/.style={nodes={mylonglargenode}}
    ] (A) {%
    &&&&&|[draw=blue, fill=blue!10!white]| \\
    &&&&&|[draw=blue, fill=blue!30!white]|\\
    &&&&&|[draw=blue, fill=blue!50!white]|\\
    &&&&&|[draw=blue,dotted]| \\
    &&&&&|[draw=blue, fill=blue!70!white]|\\
    &&&&& |[mylongnode,pattern=crosshatch,pattern color=blue!40!white, draw=blue]|\Dot{F}\\
    };
\begin{scope}[on background layer]
\foreach \ind in {1,2,3,5}
    {
    \draw[green!70!black,fill=green!40!lightgray] (A-\ind-\ind.north west) -- (A-\ind-\ind.north east) -- (A-\ind-\ind.south east) -- cycle;
    \node[anchor=north east] at (A-\ind-\ind.north east) {R};
    }
\end{scope}
\draw[dotted, green!70!black] (A-4-4.north west) -- (A-4-4.north east) -- (A-4-4.south east) -- cycle;
\node[font=\Huge,shift={(-1,-2)}] at (A) {$0$}; 
\node[font=\Huge,shift={(1,3)}] at (A) {$0$};
\matrix[
    right =of A.north east,
    anchor=north west,
    nodes={mysmallnode}
    ] (B) {%
    \underline{a}_1  \\
    \underline{a}_2 \\
    \underline{a}_3 \\
    \\
    \underline{a}_M\\
    \underline{b}\\
    };
\foreach \ind in {1,2,...,5}{
    \draw (B-\ind-1.south west) -- (B-\ind-1.south east);
}
\draw[dotted] (B-4-1.north) -- (B-4-1.south);
\matrix[
    right =4em of B.north east,
    anchor=north west,
    nodes={draw,mysmallnode}
    ] (C) {%
    |[fill=yellow!10!white]|\\
    |[fill=yellow!30!white]|\\
    |[fill=yellow!50!white]|\\
    \\
    |[fill=yellow!70!white]|\\
    |[mylongnode, pattern=horizontal lines, pattern color=yellow!90!white]|\underline{\Dot{f}}\\
    };
\path (B-4-1) -- node[midway] {$=$} (C-4-1);
\draw[dotted] (C-4-1.north) -- (C-4-1.south);
\end{tikzpicture}
\]
\end{document}

Answer 2

It is indeed a bit tedious. (EDIT: Fixed the first R to become math mode, mille grazie to @Sebastiano!)

\documentclass[tikz,border=3.14mm]{standalone}
\usetikzlibrary{positioning,backgrounds,patterns}
\begin{document}
\begin{tikzpicture}
 \begin{scope}[local bounding box=mat1]
  \path node (R0) {$R$} foreach \X [remember=\X as \lastX (initially 0)] in
  {1,...,4}
  {\ifnum\X=3
    node[below right=1.5em of R\lastX] (R\X) {$\phantom{R}$}
   \else
    node[below right=1.5em of R\lastX] (R\X)  {$R$}
   \fi
   };
  \begin{scope}[on background layer]
   \draw[fill=green!70!black] 
   foreach \X in {0,1,2,4} {
   ([xshift=-1em]R\X.north west) -- ([yshift=-1em]R\X.south east)
   coordinate(b\X)
   |- cycle};
   \draw[densely dotted] ([xshift=-1em]R3.north west) -- ([yshift=-1em]R3.south east)
   coordinate(b3) |- cycle;
  \end{scope}
  \node[pattern=crosshatch,pattern color=purple,anchor=north west,
  minimum height=6em] (F) at ([yshift=-1em]R4.south east) {$F$}; 
  \draw[densely dotted] ([yshift=-1em]R3.south east-|R4.south east)
   rectangle  (F.east|-R3.north);
  \foreach \X [evaluate=\X as \Y using {int(40+10*\X)}] in {0,1,2,4} 
  {\draw[fill=purple!\Y] ([yshift=-1em]R\X.south east-|R4.south east)
   rectangle  (F.east|-R\X.north);}
 \end{scope} 
 \path (mat1.south west) -- (mat1.north east)
  node[pos=0.3,font=\Large] {$0$} node[pos=0.7,font=\Large] {$0$};
 \draw[thick] ([xshift=0.7em]mat1.north west) -| (mat1.south west) -- ++ (0.7em,0)
 ([xshift=-0.7em]mat1.north east) -| (mat1.south east) -- ++ (-0.7em,0);
 %
 \begin{scope}[local bounding box=mat2]
  \foreach \X [remember=\X as \lastX (initially 0)] in {1,2,3}
  {\path ([xshift=1em]mat1.east|-R\lastX.north) --
  ([xshift=2.5em]mat1.east|-b\lastX) node[midway] (a\X) {$a_{\X}$};}
  \path ([xshift=1em]mat1.east|-R4.north) --
  ([xshift=2.5em]mat1.east|-b4) node[midway] (aM) {$a_{M}$};
  \node[below=0.5em of aM] (b) {$b$}; 
 \end{scope}
 \draw[thick] ([xshift=0.5em]mat2.north west) -| (mat2.south west) -- ++ 
 (0.5em,0)
 ([xshift=-0.5em]mat2.north east) -| (mat2.south east) -- ++ (-0.5em,0);
  \foreach \X in {0,...,4}
  {\draw (mat2.west|-b\X) -- (mat2.east|-b\X);}
 %
 \node[anchor=west] (eq) at (mat2.east|-mat1.center) {$=$};
 %
 \begin{scope}[local bounding box=mat3]
  \node[pattern=dots,anchor=north west,pattern color=yellow!60!orange,
  minimum height=6em] (f) at ([xshift=0.1em]eq.east|-F.north) {$f$}; 
  \foreach \X [remember=\X as \lastX (initially 0),
  evaluate=\X as \Y using {int(30*\X+10)}] in {1,2,3}
  {\fill[yellow!\Y] ([xshift=0.1em]eq.east|-R\lastX.north) rectangle
  (f.east|-b\lastX); }
  \fill[yellow] ([xshift=0.1em]eq.east|-R4.north) rectangle
  (f.east|-b4); 
  \draw[densely dotted] (f.center|-b2) -- (f.center|-b3);
 \end{scope}
 \foreach \X in {0,...,4}
  {\draw (mat3.west|-b\X) -- (mat3.east|-b\X);}
 \draw[thick] ([xshift=0.3em]mat3.north west) -| (mat3.south west) -- ++ 
 (0.3em,0)
 ([xshift=-0.3em]mat3.north east) -| (mat3.south east) -- ++ (-0.3em,0);
\end{tikzpicture}
\end{document}