\documentstyle[landscape,fleqn,a4j,12pt,epsf]{jarticle}
\thispagestyle{empty}
\pagestyle{empty}

\begin{document}
\begin{LARGE}

%% (1)title 
\clearpage
\begin{center}

\begin{Huge}
\noindent
{\bf Evolved Asymmetry and Dilution\\ of Random Synaptic Weights in Hopfield Network\\Turn a Spin-glass Phase into Associative Memory}\\

\end{Huge}

\vspace{1in}
{\bf
Akira Imada\\
Nara Institute of Science and Technology\\
(akira-i@is.aist-nara.ac.jp)\\
\hspace*{1in} \\
Keijiro Araki\\
Kyusyu University\\
(araki@is.aist-nara.ac.jp)\\
}
\end{center}

%% ---------------------------- same as fea

%% (2)introduction
\clearpage
{\bf
\noindent
Introduction:\\

\begin{itemize}
\item We apply 
   \begin{itemize}
   \item[]ECs to neural network model of Associative Memory.
   \end{itemize}
\item We study
   \begin{itemize}
   \item[] basic behaviors of Associative Memory under simple\\evolutionary processes.
   \end{itemize}
\end{itemize}
}

%% (3)asociative memory as a dynamical system
\clearpage
\begin{figure}[hbtp]
 \begin{center}
   \epsfile{file=what-is-am.eps,width=21cm,height=14cm}
 \end{center}
\end{figure}

%% its realization
\clearpage
{\bf
\noindent
It can be realized with
\begin{itemize}
\item neural network model (leyered/recurrent/self-organized)
\item immune system model
\item etc
\end{itemize}
}


%% (4)associative memory with Hopfield model
\clearpage
{\bf
\noindent
The Hopfield Model of Associative Memory:\\
\begin{itemize}
\item stores information by distributing it among neurons;
\item has a capability to tolerate noisy and/or partial input\\in recalling the information;
\item has a limited storage capacity.
\item The storage capacity is traded-off against basin of attraction.\\
\end{itemize}
}


%% (5)schematic diagram of hopfield model for N=3
\clearpage
\begin{figure}[hbtp]
 \begin{center}
   \epsfile{file=fig-nn.eps,width=21cm,height=10cm}
 \end{center}
\end{figure}

%% (5)schematic diagram of hopfield model for N=3
\clearpage
\begin{figure}[hbtp]
 \begin{center}
   \epsfile{file=trajectory.eps,width=21cm,height=14cm}
 \end{center}
\end{figure}

%% specification of synaptic weight
\clearpage
{\bf
\noindent
How to Specify Synaptic Weights ?
\begin{itemize}
\item Hopfield used the ``Hebb's rule''.
\item Kohonen \& Ruohonen's ``pseud inverse''.
\item Panch et al.'s ``specral algorithm''.
\item Gardner's algorithm.
\item[]
\item We explore it with ``evolutionary algorithms''.
\end{itemize}
}


%% (2)introduction
\clearpage
{\bf
\noindent
Modification of Synaptic Weights in the Classical Analysis:\\

\begin{displaymath}
\mbox{J}_{\mbox{ij}} \mbox{ + } \mbox{$\eta$}_{\mbox{ij}} \hspace{.4in}  \mbox{(Sompolinsky)}
\end{displaymath}

\begin{displaymath}
\mbox{J}_{\mbox{ij}} \mbox{ $B!&(B } \mbox{c}_{\mbox{ij}} \hspace{.4in}  \mbox{(Derrida, Gardner \& Zeppelius)}\\
\end{displaymath}

\vspace{.6in}

\noindent
We also modify them by ECs based on these 2 methods.


\vspace{.6in}

\noindent
They selected synapses ``randomly'' to show robustness,
\begin{center} 
while 
\end{center}
\noindent
we select synapses ``adaptively'' to enlarge/create memories.
}

%% 
\clearpage
\begin{figure}[hbtp]
 \begin{center}
   \epsfile{file=derrida.eps,width=21cm,height=14cm}
 \end{center}
\end{figure}

%% 
\clearpage
\begin{figure}[hbtp]
 \begin{center}
   \epsfile{file=sompolinsky.eps,width=24cm,height=17cm}
 \end{center}
\end{figure}

%%
\clearpage
{\bf 
\noindent
Our Implementations:\\

\begin{itemize}
\item[-] with Ternary Choromosome $( w_{ij} \cdot c_{ij} )$ 
   \begin{itemize}
   \item[$\bullet$] prune and balance synapses
   \end{itemize}
\item[]
\item[-] with Real Encoded Chromosome  $( w_{ij} + \eta_{ij} )$  $\Rightarrow$ FEA'97
   \begin{itemize}
   \item[$\bullet$] purterbation by chromosome
   \item[$\bullet$] purterbation by mutation
   \end{itemize}
\end{itemize}
}


%% (8) mutaion
%%\clearpage
%%\begin{figure}[hbtp]
%% \begin{center}
%%   \epsfile{file=mutation.eps,width=21cm,height=9cm}
%% \end{center}
%%\end{figure}


%% (7)chromosome
%%\clearpage
%%\begin{figure}[hbtp]
%% \begin{center}
%%   \epsfile{file=chromosom.eps,width=21cm,height=14cm}
%% \end{center}
%%\end{figure}

%% fitness evaluation
\clearpage
{\bf
\noindent
Fitness Evaluation\\

\indent
- ``overlap''of 2 states:
\begin{displaymath}
\frac{1}{N} \sum_{i=1}^{N} { {S}_{i}^{1} S^{2}_{i}}
\end{displaymath}

- overlap  of instantaneous state with input:
\begin{displaymath}
m^{\nu}(t) = \frac{1}{N} \sum_{i=1}^{N} { {\xi}_{i}^{\nu} s^{\nu}_{i}(t) },
\end{displaymath}

- temporal average:
\begin{displaymath}
\left \langle m^{\nu} \right \rangle
\end{displaymath}

- average over all patterns:
\begin{displaymath}
\left \langle\langle m \right \rangle \rangle
\end{displaymath}

- fitness:
\begin{displaymath}
f = \left \langle\langle m \right \rangle \rangle
\end{displaymath}

}

%% fitness evaluation for noisy input
\clearpage
{\bf
\noindent
Fitness for Noisy Input\\

- overlap  of instantaneous state with ``noisy'' input:
\begin{displaymath}
m'(t) = \frac{1}{N} \sum_{i=1}^{N} { {\xi}'_{i} s_{i}(t) },
\end{displaymath}

- temporal average:
\begin{displaymath}
\left \langle {(m')}^{\nu} \right \rangle
\end{displaymath}

- average over all patterns:
\begin{displaymath}
\left \langle\langle m' \right \rangle \rangle
\end{displaymath}

- fitness:
\begin{displaymath}
g = \left \langle\langle m' \right \rangle \rangle
\end{displaymath}

}

%% ----------------------------

%% conceptual algorithm
\clearpage
{\bf The Genetic Algorithm Implementation:}\\
\begin{itemize}
\setlength{\itemsep}{0pt}
\item[]{\bf begin}
   \begin{itemize}
   \item[]{\bf initialize} 256 chromosomes
   \item[] (each cromosome produces a copy of the original one)
   \item[]{\bf while} termination criterion not fulfilled {\bf do}
      \begin{itemize}
      \item[]{\bf evaluate} fitness of each individual
      \item[](by counting how many patterns can be memorized as fixed points)
%%      \item[]{\bf select} the best $T$\% individuals
      \item[]{\bf repeat} until the worst 60\% are replaced {\bf do} 
         \begin{itemize}
         \item[]{\bf select} two individuals {\it randomly} from the best 40\%
         \item[](neither ``roullet-wheel'' nor ``tournament'' )
         \item[]{\bf recombine} them with uniform crossover to produce one offspring
         \item[]{\bf mutate} the offspring
         \end{itemize}
%%    \item[]{\bf od}
      \end{itemize}
%% \item[]{\bf od}
   \end{itemize}
\item[]{\bf end}
\end{itemize}


%% (9)GA implementation
\clearpage
\begin{figure}[hbtp]
 \begin{center}
   \epsfile{file=overview.eps,width=21cm,height=14cm}
 \end{center}
\end{figure}




%% (8) initial trajectory
\clearpage
\begin{figure}[hbtp]
 \begin{center}
   \epsfile{file=hamming.eps,width=21cm,height=14cm}
 \end{center}
\end{figure}




%% (11)evolution
\clearpage
\begin{figure}[hbtp]
 \begin{center}
   \epsfile{file=fitness.eps,width=21cm,height=14cm}
 \end{center}
\end{figure}

%% (12)final result --- basin of attraction
\clearpage
\begin{figure}[hbtp]
 \begin{center}
   \epsfile{file=symm-zero.eps,width=21cm,height=14cm}
 \end{center}
\end{figure}

%% (12)final result --- basin of attraction
\clearpage
\begin{figure}[hbtp]
 \begin{center}
   \epsfile{file=basin.eps,width=21cm,height=14cm}
 \end{center}
\end{figure}

%% (13) future work
\clearpage
{\bf
\noindent
Conjectures:\\
\begin{itemize}
\item The GA turns chaotic trajectories to spin-glass attractors\\by pruning some connections
\item The asymmetry destabilizes the spin-glass attractors
\item But the GA cannot remove spurious attractors around created memories

\end{itemize}
}

%% (13) future work
\clearpage
{\bf
\noindent
Conclusion:\\
\begin{itemize}
\item The GA can create memory attractors at the location\\ of given patterns.
\item It was made by selecting syapses to prune or \\change excitatory/inhibitory synapses.
\item The basin was extremely small compared to Hebb'rule
\item[]

\item These study sheds new light on the analysis of the Hopfield\\model of associative memory.

\end{itemize}

}

\end{LARGE}
\end{document}





