Yes, it's a bug in the definition of \mdtheorem
. Below I show the fix:
\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{mdframed}
\makeatletter
\DeclareDocumentCommand{\mdtheorem}{ O{} m o m o }%
{\ifcsdef{#2}%
{\mdf@PackageWarning{Environment #2 already exits\MessageBreak}}%
{%
\IfNoValueTF {#3}%
{%#3 not given -- number relationship
\IfNoValueTF {#5}%
{%#3+#5 not given
\@definecounter{#2}%
\expandafter\xdef\csname the#2\endcsname{\@thmcounter{#2}}%
\newenvironment{#2}[1][]{%
\refstepcounter{#2}%
\ifstrempty{##1}%
{\let\@temptitle\relax}%
{%
\def\@temptitle{\mdf@theoremseparator%
\mdf@theoremspace%
\mdf@theoremtitlefont%
##1}%
\mdf@thm@caption{#2}{{#4}{\csname the#2\endcsname}{##1}}%
}%
\begin{mdframed}[#1,frametitle={\strut#4\ \csname the#2\endcsname%
\@temptitle}]}%
{\end{mdframed}}%
\newenvironment{#2*}[1][]{%
\ifstrempty{##1}{\let\@temptitle\relax}{\def\@temptitle{\mdf@theoremseparator\ ##1}}% <- the problem was here
\begin{mdframed}[#1,frametitle={\strut#4\@temptitle}]}%
{\end{mdframed}}%
}%
{%#5 given -- reset counter
\@definecounter{#2}\@newctr{#2}[#5]%
\expandafter\xdef\csname the#2\endcsname{\@thmcounter{#2}}%
\expandafter\xdef\csname the#2\endcsname{%
\expandafter\noexpand\csname the#5\endcsname \@thmcountersep%
\@thmcounter{#2}}%
\newenvironment{#2}[1][]{%
\refstepcounter{#2}%
\ifstrempty{##1}%
{\let\@temptitle\relax}%
{%
\def\@temptitle{\mdf@theoremseparator%
\mdf@theoremspace%
\mdf@theoremtitlefont%
##1}%
\mdf@thm@caption{#2}{{#4}{\csname the#2\endcsname}{##1}}%
}
\begin{mdframed}[#1,frametitle={\strut#4\ \csname the#2\endcsname%
\@temptitle}]}%
{\end{mdframed}}%
\newenvironment{#2*}[1][]{%
\ifstrempty{##1}%
{\let\@temptitle\relax}%
{%
\def\@temptitle{\mdf@theoremseparator%
\mdf@theoremspace%
\mdf@theoremtitlefont%
##1}%
\mdf@thm@caption{#2}{{#4}{\csname the#2\endcsname}{##1}}%
}%
\begin{mdframed}[#1,frametitle={\strut#4\@temptitle}]}%
{\end{mdframed}}%
}%
}%
{%#3 given -- number relationship
\global\@namedef{the#2}{\@nameuse{the#3}}%
\newenvironment{#2}[1][]{%
\refstepcounter{#3}%
\ifstrempty{##1}%
{\let\@temptitle\relax}%
{%
\def\@temptitle{\mdf@theoremseparator%
\mdf@theoremspace%
\mdf@theoremtitlefont%
##1}%
\mdf@thm@caption{#2}{{#4}{\csname the#2\endcsname}{##1}}%
}
\begin{mdframed}[#1,frametitle={\strut#4\ \csname the#2\endcsname%
\@temptitle}]}%
{\end{mdframed}}%
\newenvironment{#2*}[1][]{%
\ifstrempty{##1}{\let\@temptitle\relax}{\def\@temptitle{:\ ##1}}%
\begin{mdframed}[#1,frametitle={\strut#4\@temptitle}]}%
{\end{mdframed}}%
}%
}%
}
\makeatother
\mdfdefinestyle{myStyle}
{
theoremseparator={ --},
theoremspace=\space
}
\mdtheorem[style=myStyle]{theo}{Theorem}
\begin{document}
\begin{theo}[test]
Hi
\end{theo}
\begin{theo*}[test]
Hi
\end{theo*}
\end{document}

The problem is that in the lines
\newenvironment{#2*}[1][]{%
\ifstrempty{##1}{\let\@temptitle\relax}{\def\@temptitle{:\ ##1}}% <- the problem is here
\begin{mdframed}[#1,frametitle={\strut#4\@temptitle}]}%
{\end{mdframed}}%
where instead of the hard-coded colon, there should be \mdf@theoremseparator