\documentstyle[12pt,eclepsf]{article}
\textheight 250mm
\topmargin -10mm
\textwidth 180mm
\oddsidemargin -7mm

\begin{document}
\markright{(Practice at BSTU in fall 2006 -- Kohonen's Self Organizing Map)}

\title{Kohonen's Self Organizing Map: \\
Reduction of  Dimension}
\author{Akira Imada\\
\hspace*{1in}\\
Most recently renewed on}
\maketitle

\newtheorem{excersise}{Excersise} 

\section{Dimension reduction from 2-D  to 1-D}
We can use the Kohonen's SOM for dimension reduction.
It's important because, we, human being couldnt never imagine the space 
whose dimension is more than three. In this section we study, as a toy example, 
how to map 2-D world into 1-D World.
Also we are going to study how we can enjoy this trivial-looking mapping.

\subsection{Let's construct a framework.}
\noindent
First of all,  try the following as the very 1st excersise.
\begin{excersise}[Winner takes all] \label{ex:01}
As an example of mapping from 2D to 1D,
construct a SOM with two input and 100 output as follows:
\begin{itemize}
\setlength{\itemsep}{0pt}
\item[(1)] All of the two input neurons connect to all of 100 
output neurons. (So we have 200 syanapses)
\item[(2)] Give each of all these 200 syanapses a random weight
value between 0.00 to 1.00.
\item[(3)] Design a {\it man-machine interface} to show on the display screen
how {\it Winner-Take-All} works.
   \begin{itemize}
   \item[(i)]   Display 100 output neurons in a 10 $\times$ 10
                rectanglar array on the display screen.
   \item[(ii)]  Disign so that the algorithm can accept 2 inputs $x_1$ and $x_2$ 
                manually from key-board. Assume $x_1$ and $x_2$ 
                will be taken from 0.00 to 1.00. 
   \item[(iii)] Make only one neuron who is the winner highlited. 
                The winner is the neuron which has the synaptic weights
                $(w_1, w_2)$ closest to the input $(x_1, x_2)$.
   \end{itemize}
\end{itemize}
\end{excersise}

\noindent
%%See Figure 1 to image our Kohonen SOM in this subsection.
\begin{figure}[h] \label{fig:01}
 \begin{center}
   \epsfile{file=som-from-1-to-2.eps,width=8cm,height=2.0cm}
 \end{center}
 \caption{A framework of Kohonen's SOM for a mapping from 2-D to 1-D space.}
\end{figure}



\subsection{How pairs of weights are distributed?}

\begin{excersise}[Distribution of a pair of weights] \label{ex:02}
Under the framework created in the Excersise~1 try the following experiment.
\begin{itemize}
\setlength{\itemsep}{0pt}
\item[(1)] Plot all the weight pairs those determined at random in the Excersise 1
           in 2D space. Show the result in the display screen.
\item[(2)] Design your code so that waight pair of the winner 
           for a input from key-board is highlited.
           Demonstrate it on the screen.
\end{itemize}
\end{excersise}


\subsection{Let's renew the weights of the winner neuron.}

\begin{excersise}[Renewal of weights] \label{ex:03}
Let's renew the weights of the winner neuron in the following way.

\begin{itemize}
\setlength{\itemsep}{0pt}
\item[(1)] Give an input (x1, x2) randomly from keyboard
\item[(2)] Renew weights only of the winner neuron, the two neurons next to the winners, and
           and the next of the next (two neurons apart from the winner neuron. 
\begin{figure}[h]
 \begin{center}
   \epsfile{file=display-koh-5.eps,width=10cm,height=2.5cm}
\caption{The winner neuron and its four neighbours.}
 \end{center}
\end{figure}
\hspace*{1in}\\           
Hence, a total of  $2\times 5=10$ weights.
           with the equation
              \begin{equation}
              %%w^{\mbox{\tiny new}}_i = w^{\mbox{\tiny old}}_i + (x_i - w^{\mbox{\tiny old}}_i)
              {\mbox {\bf w}}_{new} = {\mbox {\bf w}}_{old} + 
                                            g * K * ({\mbox {\bf x}} - {\mbox {\bf w}}_{old})
              \end{equation}

where $g$ is a strength of infuluence of renew. 
Strength of the modification of the weights is the strongest at the winner,
then gradually become weak and if the neuron is more than two neurons far from the winner,
weights will be no more modified. \footnote{The first neuron doesn't have next neuron and
the next of the next neuron of its left hand side. Also the second neuron only has the next 
neuron to its left hand side. The same holds the second last neuron and the last neurons.}
That is, (1) the winner should be strongly influenced while
(2) the next two neighbors should be not so strong,
and (3) the next of the next neighbors should have a weaker influence than the next ones.
So, try $g = 1$ for the winner,  $g=0.5$ for the next two, and $g=0.25$ for the next next.
\item[(3)] Then show on your display 5 weight pair with winner being center
the two next neighbors both sides and two next next neighbors also both sides, like the below:
\begin{figure}[h]
 \begin{center}
   \epsfile{file=renewed-w.eps,width=16cm,height=2cm}
\caption{The winner neuron and its four neighbours.}
 \end{center}
\end{figure}
\end{itemize}
\end{excersise}

\noindent



\noindent

\subsection{Self Organization of Weights}
We now experiment a self-organization of weights values. Try the below.
\begin{excersise}[A Self Organization of Weights]
Instead of inputs from the keyboard we give our Kohonen network a set of 
many inputs, for example, from inside a triangle. 
\begin{itemize}
\setlength{\itemsep}{0pt}
\item[(1)] Pick up one point from inside of the triangle whose three tops
are (0,0), (1,0) and (0.5, 1)\footnote{If $x_1<0.5$ and $x_2<2x_1$ then point 
$(x_1,x_2)$ is inside the triangle, else if $x_1>0.5$ and $x_2< -2x_1+2$ then point 
$(x_1,x_2)$ is inside the triangle.}
\hspace*{1in}\\
\begin{figure}[h]
 \begin{center}
   \epsfile{file=kohonen-patterns-triangle.eps,width=6cm,height=3cm}
 \end{center}
\end{figure}

\item[(2)] Renew weights.
\item[(3)] Repeat (1) and (2) 30,000 times with $K=0.4$ for the 1st 10,000 iteration,
$K=0.2$ from the next 10,000 iteration, and $K=0.1$ for the last 10,000 iteration
\end{itemize}
\end{excersise}

\bigskip
\noindent
Or, more in general, try inputs from the followings

\begin{figure}[h]
 \begin{center}
   \epsfile{file=kohonen-patterns.eps,width=9cm,height=3cm}
 \end{center}
\end{figure}

\clearpage
\subsection{A Result and its Interpretation}
\noindent
The bellow is an example of the result of plotting
\begin{displaymath}
(w_1^1, w_2^1), (w_1^2, w_2^2), \cdots, (w_1^{100}, w_2^{100})
\end{displaymath}

\begin{figure}[h]
 \begin{center}
   \epsfile{file=triangle.eps,width=6cm,height=4.5cm}
\caption{100 pairs of weight linked with a line from point No.1 to point No.100. Note that
this look like a {\it Peano curve}.}
 \end{center}
\end{figure}

\noindent
However, note that this is not a result of {\it dimension reduction}
but a result of {\it self-organization of weight values.}
It would be easy to understand if you think that dimensionality of weight space
of $n$-D to $n$-D mapping is also $n$-D.

\subsection{Let's see an actual mapping from 2D to 1D}
Then what on earth is the result of {\it demension reduction?}
\begin{figure}[h]
 \begin{center}
   \epsfile{file=kohonen-2to1.eps,width=6cm,height=5cm}
 \end{center}
\end{figure}

\clearpage
\subsection{A More Practical Application from 2D to 1D \\
\hspace*{2in} -- A Thought Experiment}
If you map a coloured map made up of pikcels defined by RGB color element.
That is, each point of source space needs 5 parameters -- 2 coordinates for location,
abd 3 RGB values. Then mapping them implies to 1D but each point mapped also have an
information of RGB colors. \\

\noindent
This might be better to say mapping from 5D to 4D, though visually it is mapping 
to 1D. Of course similar colors should be mapped closer than 
others.\\

\noindent
Try a {\it Thought Experiment} how the mapping of points including RGB information below
on 2D space on to a 1D space.


\begin{figure}[h]
 \begin{center}
   \epsfile{file=rgb-kohonen.eps,width=6cm,height=5.6cm}
   \caption{A source of RGB color map visually in 2-D but actually implying 6-D.} 
 \end{center}
\end{figure}
 
 

\section{Demension Reduction from 2-D to 2-D}
This section title sounds strange in the sense that from 2-D to 2-D would not be a reduction.
Actually, it will be a tribial example but, nevertheless, it will be interesting
to observe what will be happen to plotted pair of weights which map from the input 2-D space
to lattice grid in the output 2-D space.


%%\begin{excersise}[Lattice Grid in the output 2-D space]
%%\end{excersise}

\section{From more than 3D into 2D}
\begin{footnotesize}
\begin{table}[h]
\begin{center}
\begin{tabular}{c c c c c c c c c c c c} \hline \hline
\multicolumn{4}{c}{Setosa}    &\multicolumn{4}{c}{Versicolor}  &\multicolumn{4}{c}{Virginica}\\
\hline
 $x_1$& $x_2$& $x_3$& $x_4$& $x_1$& $x_2$& $x_3$& $x_4$& $x_1$& $x_2$& $x_3$& $x_4$\\
\hline
 0.65 & 0.80 & 0.20 & 0.08 & 0.89 & 0.73 & 0.68 & 0.56 & 0.80 & 0.75 & 0.87 & 1.00 \\
 0.62 & 0.68 & 0.20 & 0.08 & 0.81 & 0.73 & 0.65 & 0.60 & 0.73 & 0.61 & 0.74 & 0.76 \\
 0.59 & 0.73 & 0.19 & 0.08 & 0.87 & 0.70 & 0.71 & 0.60 & 0.90 & 0.68 & 0.86 & 0.84 \\
 0.58 & 0.70 & 0.22 & 0.08 & 0.70 & 0.52 & 0.58 & 0.52 & 0.80 & 0.66 & 0.81 & 0.72 \\
 0.63 & 0.82 & 0.20 & 0.08 & 0.82 & 0.64 & 0.67 & 0.60 & 0.82 & 0.68 & 0.84 & 0.88 \\
 0.68 & 0.89 & 0.25 & 0.16 & 0.72 & 0.64 & 0.65 & 0.52 & 0.96 & 0.68 & 0.96 & 0.84 \\
 0.58 & 0.77 & 0.20 & 0.12 & 0.80 & 0.75 & 0.68 & 0.64 & 0.62 & 0.57 & 0.65 & 0.68 \\
 0.63 & 0.77 & 0.22 & 0.08 & 0.62 & 0.55 & 0.48 & 0.40 & 0.92 & 0.66 & 0.91 & 0.72 \\
 0.56 & 0.66 & 0.20 & 0.08 & 0.84 & 0.66 & 0.67 & 0.52 & 0.85 & 0.57 & 0.84 & 0.72 \\
 0.62 & 0.70 & 0.22 & 0.04 & 0.66 & 0.61 & 0.57 & 0.56 & 0.91 & 0.82 & 0.88 & 1.00 \\
 0.68 & 0.84 & 0.22 & 0.08 & 0.63 & 0.45 & 0.51 & 0.40 & 0.82 & 0.73 & 0.74 & 0.80 \\
 0.61 & 0.77 & 0.23 & 0.08 & 0.75 & 0.68 & 0.61 & 0.60 & 0.81 & 0.61 & 0.77 & 0.76 \\
 0.61 & 0.68 & 0.20 & 0.04 & 0.76 & 0.50 & 0.58 & 0.40 & 0.86 & 0.68 & 0.80 & 0.84 \\
 0.54 & 0.68 & 0.16 & 0.04 & 0.77 & 0.66 & 0.68 & 0.56 & 0.72 & 0.57 & 0.72 & 0.80 \\
 0.73 & 0.91 & 0.17 & 0.08 & 0.71 & 0.66 & 0.52 & 0.52 & 0.73 & 0.64 & 0.74 & 0.96 \\
 0.72 & 1.00 & 0.22 & 0.16 & 0.85 & 0.70 & 0.64 & 0.56 & 0.81 & 0.73 & 0.77 & 0.92 \\
 0.68 & 0.89 & 0.19 & 0.16 & 0.71 & 0.68 & 0.65 & 0.60 & 0.82 & 0.68 & 0.80 & 0.72 \\
 0.65 & 0.80 & 0.20 & 0.12 & 0.73 & 0.61 & 0.59 & 0.40 & 0.97 & 0.86 & 0.97 & 0.88 \\
 0.72 & 0.86 & 0.25 & 0.12 & 0.78 & 0.50 & 0.65 & 0.60 & 0.97 & 0.59 & 1.00 & 0.92 \\
 0.65 & 0.86 & 0.22 & 0.12 & 0.71 & 0.57 & 0.57 & 0.44 & 0.76 & 0.50 & 0.72 & 0.60 \\
 0.68 & 0.77 & 0.25 & 0.08 & 0.75 & 0.73 & 0.70 & 0.72 & 0.87 & 0.73 & 0.83 & 0.92 \\
 0.65 & 0.84 & 0.22 & 0.16 & 0.77 & 0.64 & 0.58 & 0.52 & 0.71 & 0.64 & 0.71 & 0.80 \\
 0.58 & 0.82 & 0.14 & 0.08 & 0.80 & 0.57 & 0.71 & 0.60 & 0.97 & 0.64 & 0.97 & 0.80 \\
 0.65 & 0.75 & 0.25 & 0.20 & 0.77 & 0.64 & 0.68 & 0.48 & 0.80 & 0.61 & 0.71 & 0.72 \\
 0.61 & 0.77 & 0.28 & 0.08 & 0.81 & 0.66 & 0.62 & 0.52 & 0.85 & 0.75 & 0.83 & 0.84 \\
 0.63 & 0.68 & 0.23 & 0.08 & 0.84 & 0.68 & 0.64 & 0.56 & 0.91 & 0.73 & 0.87 & 0.72 \\
 0.63 & 0.77 & 0.23 & 0.16 & 0.86 & 0.64 & 0.70 & 0.56 & 0.78 & 0.64 & 0.70 & 0.72 \\
 0.66 & 0.80 & 0.22 & 0.08 & 0.85 & 0.68 & 0.72 & 0.68 & 0.77 & 0.68 & 0.71 & 0.72 \\
 0.66 & 0.77 & 0.20 & 0.08 & 0.76 & 0.66 & 0.65 & 0.60 & 0.81 & 0.64 & 0.81 & 0.84 \\
 0.59 & 0.73 & 0.23 & 0.08 & 0.72 & 0.59 & 0.51 & 0.40 & 0.91 & 0.68 & 0.84 & 0.64 \\
 0.61 & 0.70 & 0.23 & 0.08 & 0.70 & 0.55 & 0.55 & 0.44 & 0.94 & 0.64 & 0.88 & 0.76 \\
 0.68 & 0.77 & 0.22 & 0.16 & 0.70 & 0.55 & 0.54 & 0.40 & 1.00 & 0.86 & 0.93 & 0.80 \\
 0.66 & 0.93 & 0.22 & 0.04 & 0.73 & 0.61 & 0.57 & 0.48 & 0.81 & 0.64 & 0.81 & 0.88 \\
 0.70 & 0.95 & 0.20 & 0.08 & 0.76 & 0.61 & 0.74 & 0.64 & 0.80 & 0.64 & 0.74 & 0.60 \\
 0.62 & 0.70 & 0.22 & 0.04 & 0.68 & 0.68 & 0.65 & 0.60 & 0.77 & 0.59 & 0.81 & 0.56 \\
 0.63 & 0.73 & 0.17 & 0.08 & 0.76 & 0.77 & 0.65 & 0.64 & 0.97 & 0.68 & 0.88 & 0.92 \\
 0.70 & 0.80 & 0.19 & 0.08 & 0.85 & 0.70 & 0.68 & 0.60 & 0.80 & 0.77 & 0.81 & 0.96 \\
 0.62 & 0.70 & 0.22 & 0.04 & 0.80 & 0.52 & 0.64 & 0.52 & 0.81 & 0.70 & 0.80 & 0.72 \\
 0.56 & 0.68 & 0.19 & 0.08 & 0.71 & 0.68 & 0.59 & 0.52 & 0.76 & 0.68 & 0.70 & 0.72 \\
 0.65 & 0.77 & 0.22 & 0.08 & 0.70 & 0.57 & 0.58 & 0.52 & 0.87 & 0.70 & 0.78 & 0.84 \\
 0.63 & 0.80 & 0.19 & 0.12 & 0.70 & 0.59 & 0.64 & 0.48 & 0.85 & 0.70 & 0.81 & 0.96 \\
 0.57 & 0.52 & 0.19 & 0.12 & 0.77 & 0.68 & 0.67 & 0.56 & 0.87 & 0.70 & 0.74 & 0.92 \\
 0.56 & 0.73 & 0.19 & 0.08 & 0.73 & 0.59 & 0.58 & 0.48 & 0.73 & 0.61 & 0.74 & 0.76 \\
 0.63 & 0.80 & 0.23 & 0.24 & 0.63 & 0.52 & 0.48 & 0.40 & 0.86 & 0.73 & 0.86 & 0.92 \\
 0.65 & 0.86 & 0.28 & 0.16 & 0.71 & 0.61 & 0.61 & 0.52 & 0.85 & 0.75 & 0.83 & 1.00 \\
 0.61 & 0.68 & 0.20 & 0.12 & 0.72 & 0.68 & 0.61 & 0.48 & 0.85 & 0.68 & 0.75 & 0.92 \\
 0.65 & 0.86 & 0.23 & 0.08 & 0.72 & 0.66 & 0.61 & 0.52 & 0.80 & 0.57 & 0.72 & 0.76 \\
 0.58 & 0.73 & 0.20 & 0.08 & 0.78 & 0.66 & 0.62 & 0.52 & 0.82 & 0.68 & 0.75 & 0.80 \\
 0.67 & 0.84 & 0.22 & 0.08 & 0.65 & 0.57 & 0.43 & 0.44 & 0.78 & 0.77 & 0.78 & 0.92 \\
 0.63 & 0.75 & 0.20 & 0.08 & 0.72 & 0.64 & 0.59 & 0.52 & 0.75 & 0.68 & 0.74 & 0.72 \\
\hline \hline
\end{tabular} 
\end{center}
\end{table}
\end{footnotesize}

\end{document}


\begin{excersise}[Renew weights of the winner]\label{ex:02}
With our  SOM frame work, let's show on the display screen 
\begin{itemize}
\setlength{\itemsep}{0pt}
\item[(1)] Display ll 100 neurons (in a 10 $\times$ 10 array) with the winner being hiligted
            as in Excersise \ref{ex:01}.
\item[(2)]  Describe weights of randomly determined at the beginning 
            $(w^{\mbox{\tiny old}}_1, w^{\mbox{\tiny old}}_1)$
\item[(3)]  Renew those weight by
\begin{equation}
w^{\mbox{\tiny new}}_i = w^{\mbox{\tiny old}}_i + 0.25 \cdot (x_i - w^{\mbox{\tiny old}}_i)
\end{equation}
where i = 1, 2.
\item[(4)]  Describe also the renewed weiths ($w^{\mbox{\tiny new}}_1, w^{\mbox{\tiny new}}_2)$
\end{itemize}
\end{excersise}


\clearpage
\section{Let's make it self-organized.}
The same as Excersize \ref{ex:01}, our SOM maps from 2D to 1D.
Using the following Algorithm, let's observe how weight values
will be {\it SELF ORGANIZED}.

\newtheorem{algorithms}{Algorithm}
\begin{algorithms}[A winner takes all algorihthm] \label{al:01}








\section{Map from 2D to 2D}
We now assume a Neural Network with two input neurons and 
$N_x \times N_y$ output neurons, no hidden layers.\\

\noindent
The location of each of the output neurons is denoted with \\
\begin{displaymath}
\mbox {\bf n} = (n_x, n_y) \hspace{.3in} n_x = 1, \cdots N_x, 
                           \hspace{.15in} \mbox{and} \hspace{.15in} 
                                         n_y = 1, \cdots N_y
\end{displaymath}
Each of the two input neurons connect to each of the output neurons via a synapse.
The weight value of the synapse from the $d$-th input neuron $(d=1,2)$
to the output neuron {\bf n} is denoted as $w_d^{\mbox{\bf n}}$ and we use a notation
\begin{displaymath}
\mbox{\bf w}^{\mbox{{\large {\bf n}}}} = (w_1^{\mbox{\large {\bf n}}}, w_2^{\mbox{\large {\bf n}}})
\end{displaymath}

\begin{figure}[h]
 \begin{center}
   \epsfile{file=kohonen.eps,width=18cm,height=7.5cm}
 \end{center}
\end{figure}


\noindent
Then the $\mbox{\bf w}^{\mbox{\large {\bf n}}}$ can be self-organized in the following way:
\begin{itemize}
\item[1.] Determin $\mbox{\bf w}^{\mbox{\large {\bf n}}}$ at random
\item[2.] The input $\mbox {\bf x} = (x_1, x_2)$
is given by drawing the point $(x_1, x_2)$ at random from a given domain, like

\begin{figure}[h]
 \begin{center}
   \epsfile{file=kohonen-patterns-triangle.eps,width=12cm,height=4cm}
 \end{center}
\end{figure}

\item[3.] Select one output neuron $\mbox{\bf n}_{0}$ such that
$\vert\vert \mbox{\bf x} - \mbox{\bf n}_{0} \vert\vert$ is minimum of all 
$\mbox{\bf x} - \mbox{\bf n}$.
\item[4.] Renew weight values as follows:
\begin{equation}\label{eq:01}
\mbox{\bf w}^{\mbox{\large {\bf n}}} = \mbox{\bf w}^{\mbox{\large {\bf n}}}
 + \epsilon \cdot g(\mbox{\bf n} - 
        \mbox{\bf n}_{0})(\mbox{\bf x} - \mbox{\bf w}^{\mbox{\large {\bf n}}}),
\end{equation}
where $\eta$ decreases gradually as the numer of repetition (see Step 5. bellow)
increases,
\begin{displaymath}
g(\mbox{\bf n} - \mbox{\bf n}_{0}) 
   = \exp(-\frac{\vert\vert \mbox{\bf x} - \mbox{\bf n}_{0} \vert\vert^2}{2\delta}),
\end{displaymath}
and $\delta$ is a constant to adjust to obtain an appropriate result.\\
\item[]
Or,
\begin{displaymath}
g(\mbox{\bf n} - \mbox{\bf n}_{0}) \equiv 1.
\end{displaymath}
\item[5.] Repeat 2.~- 4.~above.
\end{itemize}


\begin{figure}[h]
 \begin{center}
   \epsfile{file=kohonen-patterns.eps,width=12cm,height=4cm}
 \end{center}
\end{figure}


\clearpage
\newtheorem{excersise}{Excersise}
\begin{excersise}
As an example of mapping from 2D to 1D,
construct a SOM network with two input and 100 output with $N_x =100$ and $N_y = 1$. 
\end{excersise}

\begin{figure}[h]
 \begin{center}
   \epsfile{file=som-from-1-to-2.eps,width=12cm,height=4cm}
 \end{center}
\end{figure}

\noindent
Then try the following algorithm.
\begin{itemize}
\item[1.] Choose an input point by sampling it at randome from 2D triangle like (b)
           in the previous page,
\item[2.] Feed the cordinate of the point $(x_1, x_2)$ to the input of the Kohonen SOM network,
           so that weight values  are self-organized according to Eq.(\ref{eq:01}), that is,
\begin{displaymath}
w_d^{i} = w_d^{i} + \epsilon \cdot \exp \{ -(x_d - w_d^{\mbox{\tiny max}})^2\}
                                   \cdot (x_d - w_d^{i})
\end{displaymath}

where
$w_d^{\mbox{\tiny max}}$ is the weight value from $x_d$ to the 
output neuron of the winner, 
$d=1,2$ and $i = 1,2, \cdots, 100$.
The value of $\epsilon$ is decreased gradually like, for example,
   \begin{itemize}
   \item[$\cdot$] Starting with $\epsilon = 0.05$ for the 1st repetion 1 -- 5000;
                  $\epsilon = 0.04$ for the 2nd repetion 5001 -- 10000;
                  $\epsilon = 0.03$ for the 3rd repetion 10001 -- 15000; 
                  $\epsilon = 0.03$ for the 4th repetion 15001 -- 20000; 
                  and $\epsilon = 0.01$ for the last repetion 20001 -- 25000
   \end{itemize}
\item[] Or, simply by
\begin{displaymath}
w_d^{i} = w_d^{i} + \epsilon (x_d - w_d^{i})
\end{displaymath}
\item[3.] Repeat 1.~and 2.~above, say, 25000 times.
\item[4.] Plot 100 pairs of weigthts $(w_1^i, w_2^i)$ \hspace{0.05in} $i = 1, 2, \cdots, 100$
\end{itemize}

\noindent
The bellow is an example of the result of plotting
\begin{displaymath}
(w_1^1, w_2^1), (w_1^2, w_2^2), \cdots, (w_1^{100}, w_2^{100})
\end{displaymath}

\begin{figure}[h]
 \begin{center}
   \epsfile{file=triangle.eps,width=8cm,height=6cm}
 \end{center}
\end{figure}




\subsection*{A more general discription of the Network}
In the more general form, we also assume a two layered Neural Network. 
It has also $N_x \times N_y$ output neurons but the number of  input neurons
is more generally $d$. 
It maps a point in $d$-dimensional space
to a point in two-dimensional space. Hence we might use it for the purpose of 
dimension reduction.






\begin{table}[hbtp]
\begin{center}
\begin{tabular}{c c c c c} \hline \hline
\multicolumn{2}{c}{}          &\multicolumn{3}{c}{complexity $w_{ij}$}  \\
                              \cline{3-5}                                   
$i$           & $x_{ij}$      & low         & average     & high        \\
\hspace{.1in} & \hspace{.1in} &\large{$j=1$}&\large{$j=2$}&\large{$j=3$}\\
\hline
1 & external input            & 3           & --          & 6           \\
2 & external output           & 4           & --          & 7           \\
3 & logical internal file     & 7           & --          & 15          \\
  & {\large (files as the user might conceive them)}               &&&  \\
4 & external interface file   & 5           & --          & 10          \\
  & {\large (files accessed but not maintained by the application)}&&&  \\
5 & external inquiry          & 3           & --          & 6           \\
  & {\large (files that are supported on-line)}                    &&&  \\
\hline \hline
\end{tabular} 
\end{center}
\end{table}