\documentstyle[eclepsf]{article}

\textheight 260mm
\topmargin -10mm
\textwidth 180mm
\oddsidemargin -7mm

\pagestyle{myheadings}
\markright{(Neural Networks: Advanced)}

\begin{document}
\begin{large}

\begin{center}
\begin{Huge}
\noindent
{\bf Neural Networks: Advanced}\\
\end{Huge}

\vspace{1in}
Dokuz Eyl\"{u}l University \\
Computer Engineering Department \\
\hspace*{1in}\\
\begin{large}
24 October 2001 -- \\
\end{large}
\end{center}

%---------------------------------------------------------------------
\clearpage
\noindent
$\Box$ {\bf What is Neural Network?}\footnote{For more in details,
see ``Neural Network - Basics'' in this home-page.}\\
\begin{itemize}
\item Neurons and Synapses.
\item Weights, Transfer-function and Threshold.
\item Layered and Fully-connected.\\
\end{itemize}
\hspace*{1in}\\
\hspace*{1in}\\

\noindent
$\Box$ {\bf What for?}\\
\hspace*{1in}\\
\hspace*{1in}\\

\noindent
$\Box$ {\bf How can we make them work?}\\
\hspace*{1in}--- Learning: specification of its weights.
\begin{itemize}
\item Perceptron Learning.
\item Backpropagation Algorithm.
\item Evolutionary Computation.
\end{itemize}


\clearpage
\noindent
$\Box$ {\bf Fully-connected Neural Networks.}\\
\begin{itemize}
\item Toy Model of Three Neurons:

\begin{figure}[h]
 \begin{center}
   \epsfile{file=fig-nn.eps,width=10cm,height=5cm}
 \end{center}
\end{figure}

\item State Transition

\begin{displaymath}
  s_{i}\hspace{.02in} (t+1) 
    = sgn \hspace{.02in} \left( \hspace{.04in} 
      \sum_{j \ne i}^N { w_{ij} \cdot s_{j}(t)} \hspace{.04in} \right).
\end{displaymath}

\item Example 1

\begin{figure}[h]
 \begin{center}
    \epsfile{file=three-neuron-1-portrait.eps,width=10cm,height=7cm}
\end{center}
\end{figure}

\noindent
An example of state transition in the above toy example:
\begin{eqnarray}
\nonumber
s_{1}\hspace{.02in} (1) &=& sgn \hspace{.02in} \left( \hspace{.04in} 
                            w_{11} \cdot s_1(0) + w_{12} \cdot s_2(0)+ 
                            w_{13} \cdot s_3(0)
                            \hspace{.04in} \right) \\
\nonumber         
                        &=& sgn \hspace{.02in} \left( \hspace{.04in} 
                            0.14 \cdot 1 + (-0.34) \cdot 1 + (-0.52) \cdot 1
                            \hspace{.04in} \right) \\
\nonumber
                        &=& -1
\end{eqnarray}

\clearpage
\item Example 2

\begin{figure}[h]
 \begin{center}
    \epsfile{file=three-neuron-2-portrait.eps,width=5cm,height=5cm}
\end{center}
\end{figure}

\item Example 3

\begin{figure}[h]
 \begin{center}
    \epsfile{file=three-neuron-3-portrait.eps,width=4cm,height=6cm}
\end{center}
\end{figure}

%-----------------------------------------------------------------------
\clearpage
\noindent
$\Box$ {\bf Associative Memory as a Dynamical System}\\
\begin{figure}[h]
 \begin{center}
    \epsfile{file=what-is-am.eps,width=14cm,height=6cm}
\end{center}
\end{figure}

\hspace*{1in}\\
\hspace*{1in}\\
\noindent
$\Box$ {\bf A category of trajectories}\\
\begin{figure}[h]
 \begin{center}
    \epsfile{file=trajectory.eps,width=12cm,height=8cm}
\end{center}
\end{figure}
\end{itemize}
%-----------------------------------------------------------------------
\clearpage
\noindent
$\Box$ {\bf What is Associative Memory}\\

\begin{itemize}
\item $N$ neurons that are fully connected store $N$-bit of  
      $p$ patterns simultanously 
      as {\it fixed point attractors.}\footnote{Here we restrict patterns 
                                                to bipolar patterns.}
\item Convergence to the initial state of the noisy stored pattern
      implys an {\it error correcting capability.}
\item How can we determine weights.
   \begin{itemize}
   \item Hebbian Weights

\begin{displaymath}
w_{ij} = \frac{1}{N} \sum _{ \nu = 1}^{p} 
         {{ \xi }_{i}^{ \nu }{ \xi }_{j}^{ \nu }}
         \hspace{.1in}(i \ne j), \hspace{.1in} w_{ii} = 0.
\end{displaymath}

   \item[] In the above toy example:

\begin{displaymath}
\left(
   \begin{array}{ccc}
   0    & 0.11 & 0.11 \\
   0.11 & 0    & 0.11 \\
   0.11 & 0.11 & 0 
   \end{array}
\right)
\end{displaymath}

\hspace*{1in}\\

   \item[] An interpretation of Hebb's rule:
      \begin{itemize}
      \item A strong weight affects neurons at both ends 
            to be same states.
      \end{itemize}
\item[]
   \item Many other solutions of a {\it weight configuration} 
         to store a same set of patterns exist.
     \begin{itemize}
     \item Each can {\it recall} from
            \begin{itemize}
            \item partial information;
            \item incorrect information
            \end{itemize}
       \item but has a different
         \begin{itemize}
         \item {\it basin of attraction};
         \item {\it storage capacity}.
         \end{itemize}
      \end{itemize}

   \end{itemize}
\end{itemize}




\end{large}
\end{document}

