\documentstyle[12pt,eclepsf]{article}
\textheight 250mm
\topmargin -10mm
\textwidth 180mm
\oddsidemargin -7mm

\markright{(Kohonen's Self Organizing Map)}

\begin{document}
\title{Kohonen's Self Organizing Map: \\
Reduction of  Dimension}
\author{Akira Imada}
\date{}
\maketitle

\begin{Large}
\section{Map from 2D to 2D}
We now assume a Neural Network with two input neurons and 
$N_x \times N_y$ output neurons, no hidden layers.\\

\noindent
The location of each of the output neurons is denoted with \\
\begin{displaymath}
\mbox {\bf n} = (n_x, n_y) \hspace{.3in} n_x = 1, \cdots N_x, 
                           \hspace{.15in} \mbox{and} \hspace{.15in} 
                                         n_y = 1, \cdots N_y
\end{displaymath}
Each of the two input neurons connect to each of the output neurons via a synapse.
The weight value of the synapse from the $d$-th input neuron $(d=1,2)$
to the output neuron {\bf n} is denoted as $w_d^{\mbox{\bf n}}$ and we use a notation
\begin{displaymath}
\mbox{\bf w}^{\mbox{{\large {\bf n}}}} = (w_1^{\mbox{\large {\bf n}}}, w_2^{\mbox{\large {\bf n}}})
\end{displaymath}

\begin{figure}[h]
 \begin{center}
   \epsfile{file=kohonen.eps,width=18cm,height=7.5cm}
 \end{center}
\end{figure}

\clearpage
\noindent
Then the $\mbox{\bf w}^{\mbox{\large {\bf n}}}$ can be self-organized in the following way:
\begin{itemize}
\item[1.] Determin $\mbox{\bf w}^{\mbox{\large {\bf n}}}$ at random
\item[2.] The input $\mbox {\bf x} = (x_1, x_2)$
is given by drawing the point $(x_1, x_2)$ at random from a given domain, like

\begin{figure}[h]
 \begin{center}
   \epsfile{file=kohonen-patterns.eps,width=12cm,height=4cm}
 \end{center}
\end{figure}

\item[3.] Select one output neuron $\mbox{\bf n}_{0}$ such that
$\vert\vert \mbox{\bf x} - \mbox{\bf n}_{0} \vert\vert$ is minimum of all 
$\mbox{\bf x} - \mbox{\bf n}$.
\item[4.] Renew weight values as follows:
\begin{equation}\label{eq:01}
\mbox{\bf w}^{\mbox{\large {\bf n}}} = \mbox{\bf w}^{\mbox{\large {\bf n}}}
 + \epsilon \cdot g(\mbox{\bf n} - 
        \mbox{\bf n}_{0})(\mbox{\bf x} - \mbox{\bf w}^{\mbox{\large {\bf n}}}),
\end{equation}
where $\eta$ decreases gradually as the numer of repetition (see Step 5. bellow)
increases,
\begin{displaymath}
g(\mbox{\bf n} - \mbox{\bf n}_{0}) 
   = \exp(-\frac{\vert\vert \mbox{\bf x} - \mbox{\bf n}_{0} \vert\vert^2}{2\delta}),
\end{displaymath}
and $\delta$ is a constant to adjust to obtain an appropriate result.\\
\item[]
Or,
\begin{displaymath}
g(\mbox{\bf n} - \mbox{\bf n}_{0}) \equiv 1.
\end{displaymath}
\item[5.] Repeat 2.~- 4.~above.
\end{itemize}

\clearpage
\newtheorem{excersise}{Excersise}
\begin{excersise}
As an example of mapping from 2D to 1D,
construct a SOM network with two input and 100 output with $N_x =100$ and $N_y = 1$. 
\end{excersise}

\begin{figure}[h]
 \begin{center}
   \epsfile{file=som-from-1-to-2.eps,width=12cm,height=4cm}
 \end{center}
\end{figure}

\noindent
Then try the following algorithm.
\begin{itemize}
\item[1.] Choose an input point by sampling it at randome from 2D triangle like (b)
           in the previous page,
\item[2.] Feed the cordinate of the point $(x_1, x_2)$ to the input of the Kohonen SOM network,
           so that weight values  are self-organized according to Eq.(\ref{eq:01}), that is,
\begin{displaymath}
w_d^{i} = w_d^{i} + \epsilon \cdot \exp \{ -(x_d - w_d^{\mbox{\small max}})^2\}
                                   \cdot (x_d - w_d^{i})
\end{displaymath}

where
$w_d^{\mbox{\small max}}$ is the weight value from $x_d$ to the 
output neuron of the winner, 
$d=1,2$ and $i = 1,2, \cdots, 100$.
The value of $\epsilon$ is decreased gradually like, for example,
   \begin{itemize}
   \item[$\cdot$] Starting with $\epsilon = 0.05$ for the 1st repetion 1 -- 5000;
                  $\epsilon = 0.04$ for the 2nd repetion 5001 -- 10000;
                  $\epsilon = 0.03$ for the 3rd repetion 10001 -- 15000; 
                  $\epsilon = 0.03$ for the 4th repetion 15001 -- 20000; 
                  and $\epsilon = 0.01$ for the last repetion 20001 -- 25000
   \end{itemize}
\item[] Or, simply by
\begin{displaymath}
w_d^{i} = w_d^{i} + \epsilon (x_d - w_d^{i})
\end{displaymath}
\item[3.] Repeat 1.~and 2.~above, say, 25000 times.
\item[4.] Plot 100 pairs of weigthts $(w_1^i, w_2^i)$ \hspace{0.05in} $i = 1, 2, \cdots, 100$
\end{itemize}

\noindent
The bellow is an example of the result of plotting
\begin{displaymath}
(w_1^1, w_2^1), (w_1^2, w_2^2), \cdots, (w_1^{100}, w_2^{100})
\end{displaymath}

\begin{figure}[h]
 \begin{center}
   \epsfile{file=triangle.eps,width=8cm,height=6cm}
 \end{center}
\end{figure}




\subsection*{A more general discription of the Network}
In the more general form, we also assume a two layered Neural Network. 
It has also $N_x \times N_y$ output neurons but the number of  input neurons
is more generally $d$. 
It maps a point in $d$-dimensional space
to a point in two-dimensional space. Hence we might use it for the purpose of 
dimension reduction.




\end{Large}
\end{document}
