deneir/sds/detectors_evaluation/main.tex

\documentclass[aspectratio=169,10pt]{beamer}

\usetheme[progressbar=head,numbering=fraction,sectionpage=none]{metropolis}
\usepackage{graphicx}
\usepackage{ulem}
\usepackage{xcolor}
\usepackage[scale=2]{ccicons}
\usepackage{pgfplots}
\usepgfplotslibrary{dateplot}
\usepackage{hyperref}
\usepackage{xspace}
\newcommand{\themename}{\textbf{\textsc{metropolis}}\xspace}

\title{State Detection and Segmentation (SDS)}
\subtitle{May the best detector win.}

\date{}
\author{Arthur Grisel-Davy}
\institute{University of Waterloo - EET2 Project}

\begin{document}


\maketitle

\begin{frame}{Goal}
    \begin{center}
        Detect the state of the machine at any point in time.
        \includegraphics[width=0.9\textwidth]{images/trace.pdf}
    \end{center}
\end{frame}

\begin{frame}{Training Data}
    \begin{center}
        Proto Examples:
    \includegraphics[width=\textwidth]{images/protos.pdf}
    \end{center}

    \begin{itemize}
        \item Not exacts examples.
        \item Various lengths.
        \item Tile any trace completely.
    \end{itemize}
\end{frame}

\begin{frame}{Evaluation Ground}
    \begin{itemize}
        \item Synthetically generated input trace \& protos.
        \item Trace start at randomly selected sample.
        \item Score: global accuracy $\left(\dfrac{TP+TN}{TP+TN+FP+FN}\right)$
    \end{itemize}
\end{frame}

\begin{frame}{Simplest Detector}
    Label each sample with the closest proto example.
    \begin{center}
    \includegraphics[width=0.7\textwidth]{images/simplest.pdf}
    \end{center}
\end{frame}

\begin{frame}{Scoreboard}
    \begin{table}
        \begin{tabular}{l|c}
            \textbf{Detector} & Simplest\\
            \textbf{Score} & 0.32\\
        \end{tabular}
    \end{table}
\end{frame}

\begin{frame}{Confident Detector}
    The complete length of the closest proto example is assigned the label.
    \begin{center}
    \includegraphics[width=0.7\textwidth]{images/confident.pdf}
    \end{center}
\end{frame}

\begin{frame}{Scoreboard}
    \begin{table}
        \begin{tabular}{l|c|c}
            \textbf{Detector} & Simplest & Confident\\
            \textbf{Score} & 0.32 & 0.69 \\
        \end{tabular}
    \end{table}
\end{frame}


\begin{frame}{Cautious Detector}
    The complete length of the closest proto example is assigned the label, but still moves by 1 sample.
    \begin{center}
    \includegraphics[width=0.7\textwidth]{images/cautious.pdf}
    \end{center}
\end{frame}


\begin{frame}{Scoreboard}
    \begin{table}
        \begin{tabular}{l|c|c|c}
            \textbf{Detector} & Simplest & Confident & Cautious\\
            \textbf{Score} & 0.32 & 0.69 & 0.39\\
        \end{tabular}
    \end{table}
\end{frame}

\begin{frame}{Let's be smart}
    \begin{itemize}
        \item \sout{Respect Causality} $\rightarrow$ Use all data available.
        \item Proto example represent the whole trace.
        \item Try to place each example as best as possible independently.
    \end{itemize}
\end{frame}

\begin{frame}{Generous label Activation}
    \begin{center}
    \includegraphics[width=\textwidth]{images/overlap.pdf}
    \end{center}
\end{frame}


\begin{frame}{Generous label Activation}
    \begin{center}
    \includegraphics[width=\textwidth]{images/overlap_explained.pdf}
    \end{center}
\end{frame}

\begin{frame}{Inter-protos Distances}
    \begin{itemize}
        \item Minimize overlaps.
        \item Resolve overlap conflicts.
    \end{itemize}
    $\rightarrow$ Compute inter-protos distances.
\end{frame}

\begin{frame}{Inter-protos Distances}
    \begin{center}
    \includegraphics[width=\textwidth]{images/inter_distances.pdf}
    \end{center}

! Hyper-parameter: threshold placement coefficient (default=1) 
\end{frame}


\begin{frame}{Detection}
    \begin{center}
    \includegraphics[height=0.9\textheight]{images/detection.pdf}
    \end{center}
\end{frame}

\begin{frame}{Detection}
    \begin{center}
    \includegraphics[height=0.9\textheight]{images/detection_0.7.pdf}
    \end{center}
\end{frame}

\begin{frame}{Detection}
    \begin{center}
    \includegraphics[height=0.9\textheight]{images/detection_0.5.pdf}
    \end{center}
\end{frame}


\begin{frame}{Scoreboard}
    \begin{table}
        \begin{tabular}{l|c|c|c|c}
            \textbf{Detector} & Simplest & Confident & Cautious & Inter-distances\\
            \textbf{Score} & 0.32 & 0.69 & 0.39 & 0.91 - 0.99\\
        \end{tabular}
    \end{table}
\end{frame}

\begin{frame}{Future Work}
    \begin{itemize}
        \item Generate real data.
        \item Evaluate impact of imperfect protos.
        \item Evaluate possibility of incomplete proto dictionary.
        \item Evaluate possibility of automatic threshold optimization.
    \end{itemize}
\end{frame}

\begin{frame}{Real Data}
    \begin{center}
        \includegraphics[width=\textwidth]{images/real_labeled_data_1.pdf}
    \end{center}
\end{frame}

\begin{frame}{Real Data (ASUS)}
    \begin{center}
        \includegraphics[width=\textwidth]{images/detection_real_asus_1_1.pdf}
    \end{center}
\end{frame}

\begin{frame}{Real Data (LINKSYS)}
    \begin{center}
        \includegraphics[width=\textwidth]{images/detection_real_linksys_1_1.pdf}
    \end{center}
\end{frame}


\begin{frame}{Increasing Complexity}
    \begin{center}
        \includegraphics[width=\textwidth]{images/complexity.png}
    \end{center}
\end{frame}


\begin{frame}{NUC Windows Machine}
    \begin{center}
        \includegraphics[width=\textwidth]{images/detection_real_NUC1_1_1.pdf}
    \end{center}
\end{frame}

\begin{frame}{NUC Windows Machine}
    \begin{enumerate}
        \item Remove Shutdown Patterns.
    \end{enumerate}
\end{frame}

\begin{frame}{NUC Windows Machine}
    \begin{center}
        \includegraphics[width=\textwidth]{images/detection_real_NUC1_2_1.pdf}
    \end{center}
\end{frame}

\begin{frame}{NUC Windows Machine}
    \begin{enumerate}
        \item Remove Shutdown Patterns.
        \item Separate BIOS and OS states.
    \end{enumerate}
\end{frame}

\begin{frame}{NUC Windows Machine}
    \begin{center}
        \includegraphics[width=\textwidth]{images/detection_real_NUC1_3_0.9.pdf}
    \end{center}
\end{frame}

\begin{frame}{Current state}
    \begin{itemize}
        \item State detection not satisfactory on Windows machines.
        \item Confusion between Activity and Reboot.
        \item Likely very performant on PLCs data.
    \end{itemize}
\end{frame}
\end{document}