You can get round caps using picture mode:
\documentclass{article}
\usepackage{amsmath,stmaryrd,pict2e,picture}
\newcommand{\varbslash}{%
\mathbin{\mathpalette\pictvarbslash\relax}%
}
\newcommand{\pictvarbslash}[2]{%
\vcenter{\hbox{%
\sbox0{$#1\varobslash$}\dimen0=.55\wd0
\begin{picture}(\dimen0,\dimen0)
\roundcap
\put(0,\dimen0){\line(1,-1){\dimen0}}
\end{picture}%
}}%
}
\begin{document}
$\varobslash\varbslash_{\varobslash\varbslash}$
\end{document}

Here's an expanded version with \varslash
\documentclass{article}
\usepackage{amsmath,stmaryrd,pict2e,picture}
\newcommand{\varslash}{%
\mathbin{\mathpalette\pictslash{{0}{1}}}%
}
\newcommand{\varbslash}{%
\mathbin{\mathpalette\pictslash{{1}{-1}}}%
}
\makeatletter
\newcommand{\pictslash}[2]{%
\vcenter{\hbox{%
\sbox0{$\m@th#1\varobslash$}\dimen0=.55\wd0
\pictslash@aux#2%
}}%
}
\newcommand{\pictslash@aux}[2]{%
\begin{picture}(\dimen0,\dimen0)
\roundcap
\put(0,#1\dimen0){\line(1,#2){\dimen0}}
\end{picture}%
}
\makeatother
\begin{document}
$\varobslash\varbslash_{\varobslash\varbslash}$
$\varoslash\varslash_{\varoslash\varslash}$
\end{document}

Another variant where the new symbols occupy the same space as the corresponding circled ones:
\documentclass{article}
\usepackage{amsmath,stmaryrd,pict2e,picture}
\newcommand{\varslash}{%
\mathbin{\mathpalette\pictslash{{0}{1}}}%
}
\newcommand{\varbslash}{%
\mathbin{\mathpalette\pictslash{{1}{-1}}}%
}
\makeatletter
\newcommand{\pictslash}[2]{%
\vcenter{%
\sbox0{$\m@th#1\varobslash$}\dimen0=.55\wd0
\hbox to\wd 0{%
\hfil\pictslash@aux#2\hfil
}%
}%
}
\newcommand{\pictslash@aux}[2]{%
\begin{picture}(\dimen0,\dimen0)
\roundcap
\put(0,#1\dimen0){\line(1,#2){\dimen0}}
\end{picture}%
}
\makeatother
\begin{document}
$\varobslash\varobslash_{\varobslash\varobslash}$
$\varobslash\varbslash_{\varobslash\varbslash}$
$\varoslash\varslash_{\varoslash\varslash}$
\end{document}
