\documentclass[12pt]{article}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{rsfs}

\usepackage{color}

\usepackage{graphicx}
%%\usepackage[dviout]{graphicx}
%%\DeclareGraphicsRule{.jpg}{eps}{.bb}{bmeps -c -tjpg #1}
%%\DeclareGraphicsRule{.jpg}{eps}{.bb}{`bmeps -c -tjpg #1}

\textheight 260mm
\topmargin 5mm
\textwidth 180mm
\oddsidemargin -7mm

\pagestyle{myheadings}
\markright{(Seminar in the Institute of Mathematics and Informatics, Vilnius)}
\thispagestyle{empty}
\begin{document}
\begin{huge}

\clearpage  % (title) =========================================
\color{blue}
\begin{center}
{\bf Spike timing dependent plasticity (STDP) \\
\bigskip
to make robot navigation intelligent}
\hspace*{,5in}\\
\color{black}
\bigskip
\hspace*{1in}\\
\begin{Large}
Akira Imada  \\
\bigskip
Brest State Technical University (Belarus)
\end{Large}
\end{center}

\clearpage  % =========================================
\vspace*{.02in}
\begin{center} \color{blue}
This is not a success report \\
\bigskip
but \\
\bigskip
a guideline of our ongoing project \\
\bigskip
or \\
\bigskip
a report of what we are  currently planning.

\end{center}


\clearpage  % =========================================
\vspace*{.02in}
\begin{center} \color{blue}
To start with \\ 
\bigskip
a benchmark to know \\ 
\bigskip
if what we are planning will work or not.
\end{center}


\clearpage  % =========================================
%%\vspace*{.005in}
\bigskip
\begin{center}
\color{black}
{\Large Benchmark will be} \\
\bigskip
\color{blue} {\bf Path Planning}\\
\color{black}
{\Large or}\\
\color{blue}
{\bf Robot Navigation}\\
\color{black}
$\Downarrow$ {\Large where}\\
{\Large the aim is usually to \color{red} minimize \color{black} the route from start to goal, but...}
\end{center}

%% Here I want to see it from yet another aspect.


\clearpage  % =========================================
\vspace*{.01in}
\begin{center}\color{blue}
{\bf Task of minimizing navigation \\ on its own \\ is not so difficult}\\
\bigskip
\color{black}
$\Downarrow$\\
\bigskip
{\Large A-star,
Genetic Algorithm,
Reinforcement Learning,
Neural Network, \\
Ant Colony Optimization, etc...}
\end{center}

%% A star algorithm is a kind of standard approach, or GA, RL, NN are also very popular for this problem, and AC might be a good choice...



\clearpage  % =========================================
\vspace*{.01in}
\begin{center} 
\color{blue}
{\bf Genetic Algorithm (GA)}\\
\bigskip
\color{black}
{\Large
$\Downarrow$\\
\bigskip
Agent decides actions following its \color{red} chromosome\\
\color{black}
\bigskip
e.g. \\
with  chromosomes like}\\
\color{black}
{\Large  (4 1 1 2 1 3 2 3 3 4 2 1 3 4 3 2 3 3 2 1 1 3 2 1 3 ...)} 
\end{center}

%% For example we can use GA with chromosomes with four different alleles each represent to north to south to east and to west



\clearpage  % =========================================
\begin{center}\color{blue}
{\bf Random walk evolved to be minimized}\\
\end{center}
\bigskip
\begin{figure}[bhtp]
\begin{center}
   \includegraphics[width=5cm,height=6cm]{trace-96-start.eps}
   \includegraphics[width=5cm,height=6cm]{trace-96-end-new.eps}
\end{center}
\end{figure}


\clearpage  % =========================================
\begin{center}\color{blue}
{\bf Task was easy}\\
%%{\bf Evolution successful}\\
%%Though number of steps grows linearly}\\
\end{center}
\begin{figure}[h]
\begin{center}
\includegraphics[width=10cm,height=6cm]{evolution-96-new.eps}
%%\includegraphics[width=6cm,height=6cm]{navi-1-new.eps}
\end{center}
\end{figure}



\clearpage  % =========================================
\begin{center} 
{\Large Let's extend it}\\
\color{blue}
{\bf From Minimization to Maximization}\\
\bigskip
\color{black}
{\Large 
$\Downarrow$ i.e. \\
\bigskip
We don't care goal points but to maximize an exploration}
\end{center}



\clearpage  % =========================================
\begin{center}\color{blue}
{\bf A Camel in a Desert}\\
\bigskip
\color{black}
{\LARGE
The 52nd problem \\
{\Large in the} \\
{\it ``Propositiones ad acuendos inventes''} (in Latin) \\
{\Large by} \\
Alcuin of York (732--804)}\\
\end{center}

\begin{figure}[h]
\begin{center}
%%\includegraphics[width=3cm,height=3cm]{alcuin-york.jpg}
\includegraphics[width=3cm,height=3cm]{alcuin-york.eps}
\end{center}
\end{figure}



\clearpage  % =========================================
\begin{center} \color{blue}
Can a camel maximize its exploration in a desert surviving with grains on his back?\\
\bigskip
\color{black}
\begin{figure}[bhtp]
\begin{center}
\rotatebox[origin=c]{90}{
\includegraphics*[clip,width=5cm,height=10cm]{camel-3.eps}}
%%\includegraphics*[clip,width=6cm,height=4cm]{camel-4.bmp}
%%includegraphics*[clip,width=6cm,height=4cm]{camel-4plus.eps}
\end{center}
\end{figure}
\end{center}


\clearpage  % =========================================
\vspace*{.02in}
\begin{center} \color{blue}
{\bf Modern version: \\From Camel to Jeep in a Desert}\\
\bigskip
\color{black}
\end{center}

\clearpage  % =========================================
\vspace*{.01in}
\begin{center} \color{blue}
Can a jeep maximize its exploration \\ 
\bigskip
in a desert with a tank of fuel?
\begin{figure}[bhtp]
\begin{center}
\rotatebox[origin=c]{270}{
\includegraphics[width=4.5cm,height=6cm]{jeep-3.eps}}
\end{center}
\end{figure}

\end{center}


\clearpage  % =========================================
\begin{center}\color{blue}
{\bf Still it's not so difficult \\by GA for example}
\end{center}
\begin{figure}[h]
\begin{center}
\includegraphics[width=8cm,height=6cm]{initial.eps}
\includegraphics[width=8cm,height=6cm]{final.eps}
\end{center}
\end{figure}


\clearpage  % =========================================
\begin{center}\color{blue}
{\bf Task was rather easier}\\
\begin{figure}[h]
\begin{center}
%%\includegraphics[width=6cm,height=6cm]{random-search-needle-2.eps}
\includegraphics[width=12cm,height=5cm]{complexity.eps}
\end{center}
\end{figure}
\color{black}
{\Large The complexity is \color{red}$O(N)$}
\end{center}



\clearpage  % =========================================
\begin{center} 
{\Large Why not extend it further}\\
\color{blue}
{\bf Let's make it return to the point \\ it started}\\
\color{black}
$\Downarrow$\\
{\Large while also maximizing an exploration}
\end{center}


\clearpage  % =========================================
\vspace*{.01in}
\begin{center} 
\color{blue}
{\bf Planet Land-rover Problem}\\
\color{black}
\hspace*{1in}\\
{\Large From A (start) to B (goal) minimizing its route}\\
$\Downarrow$\\
{\Large From A to A  maximizing its route\\

(This is our proposal here as a benchmark)}


\end{center}


\clearpage  % =========================================
\begin{center} \color{blue}
Can the rover return to the base \\ with its exploration being maximum \\
spending a limited energy given when it started?

\bigskip
\color{black}
\begin{figure}[bhtp]
\begin{center}
\rotatebox[origin=c]{90}{
\includegraphics[width=3cm,height=4cm]{rover-2.eps}}
\end{center}
\end{figure}
\end{center}


\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf Much more demanding}\\
\color{black}
\bigskip
$\Downarrow$\\
\bigskip
{\Large Taking GA as an example, what might be a fitness?\\
e.g.\\
{\it \{total length\} \& \{how often it has crossed previous route?\}} \\
with \\
Multi Objective GA (MOGA)}
\end{center}

%% I have tried such algorithms but so far I've not succeeded

%% So we need another criteria like...

\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf A Heuristic create such a route}\\

\bigskip
\begin{figure}[h]
\begin{center}
\includegraphics[width=10cm,height=5cm]{max-loop-5.eps}
\end{center}
\end{figure}
\color{black}
{\Large What heuristic do you guess?}
\end{center}


\clearpage  % =========================================
\vspace*{.02in}
\begin{center} 
{\Large Topic of today's talk:}\\
\bigskip
\color{blue}
{\bf What is intelligence?}\\
\color{black}
{\Large 
$\Downarrow$ \\
Intelligence should be 
\color{red}
spontaneous, flexible, or unpredictable 
\color{black}
more or less}
\end{center}


\clearpage  % =========================================
%% \vspace*{.01in}
\begin{center} 
\color{blue}
{\bf Stolle et al. (2002)}\\
\hspace*{1in}\\
\color{black}
{\Large {\it ``... every day we might be cooking a \color {red} different 
\color{black} breakfast, 
but the kitchen layout is the \color{red} same \color{black} from day to day.''}}
\end{center}


\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf ``I beg your pardon?''}\\
\hspace*{1in}\\
\color{black}
{\Large 
Intelligent people try a different explanation for an easier understanding\\
\bigskip
while \\
others repeat the expression, maybe with a bigger voice}
\end{center}


\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf What if your canary stop singing? \\  
{\Large Legendary three strategies in Japan.}}\\
\hspace*{1in}\\
\color{black}
{\Large 
(1) Just wait until she sings again. \\
\bigskip
(2) Try something so that she sings again. \\
\bigskip
(3) Kill her if she doesn't sing any more?}
\end{center}
\begin{figure}[h]
\begin{center}
\rotatebox[origin=c]{90}{
\includegraphics[width=2.5cm,height=2.5cm]{canary-2.eps}}
\end{center}
\end{figure}



\clearpage  % =========================================
\vspace*{.02in}
\begin{center} 
\color{blue}
{\bf To be intelligent, \\ 
\bigskip
action should be different more or less \\ 
\bigskip
even in an exactly identical situation}
\end{center}



\clearpage  % =========================================
\vspace*{.02in}
\begin{center} 
\color{blue}
{\bf Performance is not intelligent}\\
\bigskip
\color{black}
{\Large if we use \\
\color{red}
deterministic
\color{black}
A-star, GA, NN with \color{red} fixed \color{black} weights?} \\
\end{center}



\clearpage  % =========================================
\vspace*{.2in}
\begin{center} 
\color{blue}
%%{\bf Preliminary Result from this Aspect}\\
{\bf Let's make a learning occur \\ during behaviors}\\
\bigskip
\color{black}
$\Downarrow$\\
{\Large Evolution of Learning}
\end{center}

%% when we say evolution of learning it means ...

\clearpage  % =========================================
\begin{center} 
\color{blue}
{\Large A notation}\\
{\bf Hebbian Learning of NN}\\

\begin{figure}[h]
\begin{center}
\includegraphics[width=8cm,height=4cm]{plasticity.eps}
\end{center}
\end{figure}
{\Large
\color{red}
\begin{displaymath}
w_{ij}(t+1) = w_{ij}(t) + \eta x_j(t)y_i(t) \hspace{.2in} \color{black}(x_j, y_i \in [0,1])
\end{displaymath}
}
\end{center}

%% let's make a notation clear

\clearpage  % =========================================
\vspace*{.02in}
\begin{center} 
\color{blue}
{\bf Floreano's approach (2000)}\\
\bigskip
\color{black}
$\Downarrow$\\
\bigskip
{\Large Modification of $w_{ij}$ during exploration\\
with either one of four Hebbian and Hebbian-like rules}
\end{center}


\clearpage  % =========================================
\vspace*{.02in}
\begin{center}
\color{blue}
{\bf Hebbian learning}\\   %% explain each of them  and before this page a figure
\color{black}
{\Large
\begin{displaymath}
(1) \hspace{.5in} \Delta w = (1-w)xy
\end{displaymath}
}
\end{center}


\clearpage  % =========================================
\vspace*{.02in}
\begin{center}
\color{blue}
{\bf Hebbian type learning}\\   %% explain each of them  and before this page a figure
\color{black}
\hspace*{1in}\\
{\Large
Weaken if the post-synaptic is active, while the pre-synaptic is not
\begin{displaymath}
(2) \hspace{.5in} \Delta w = w(-1+x)y + (1-w)xy
\end{displaymath}
\hspace*{1in}\\
Weaken if the pre-synaptic is active, while the post-synaptic is not
\begin{displaymath}
(3) \hspace{.5in} \Delta w = wx(-1+y) + (1-w)xy
\end{displaymath}
}
\end{center}

\clearpage  % =========================================
\begin{center}
\color{blue}
{\bf And}\\
\color{black}
\hspace*{1in}\\
{\Large Strengthen if the two have similar activity and weaken otherwise.
\begin{eqnarray*}
(4) \hspace{.5in} \Delta w = \left\{
\begin{array}{ll}\nonumber
         (1-w) F(x,y)
                 & \hspace{.2in}\mbox{if}\hspace{.2in} F(x,y)>0 \\
         w F(x,y)
                 & \hspace{.2in}\mbox{otherwise}\\
         \end{array}
         \right.\\
      \end{eqnarray*}
where \\
$F(x,y) = \tanh (4(1- \vert x-y \vert) - 2)$
}
\end{center}


\clearpage  % =========================================
\vspace*{.02in}
\begin{center} 
\color{blue}
{\bf Evolution of leaning}\\

\hspace*{1in}\\
\color{black}
{\Large 
\color{red}Which rule \color{black}and  
\color{red} what parameter's value \color{black} should be assigned \\ to each of the synapses?\\
\bigskip
$\Downarrow$\\
Evolution leads the combination to the optimum.}
\end{center}




\clearpage  % =========================================
\vspace*{.02in}
\begin{center} 
\color{blue}
{\bf Stanley's implementation (2003)}\\
\color{black}
{\Large
\begin{eqnarray*}
\Delta w = \left\{
\begin{array}{ll}\nonumber
         \eta_1 (1-w) xy + \eta_2 wx(y-1)
                 & \hspace{.2in}\mbox{for}\hspace{.1in} {\mbox {\large excitatory neuron}} \\
         - \eta_1 (1-w) xy + \eta_2 (1-w) x(1-y)
                 & \hspace{.2in}\mbox{for}\hspace{.1in} {\mbox {\large inhibitory neuron}} \\
         \end{array}
         \right.\\
      \end{eqnarray*}
$\Downarrow$ \\
Evolution is on $(\eta_1, \eta_2)$ of each synaptic weight
}
\end{center}


\clearpage  % =========================================
\vspace*{.02in}
\begin{center} 
\color{blue}
{\bf Their result}\\

\hspace*{1in}\\
\color{black}
{\Large Starting with random weights every time anew\\

the weights are modified step by step \color{red} by the rule it learned \\
\color{black}
\bigskip
$\Downarrow$ \\

Every run is different depending on the initial random weights}
\end{center}


\clearpage  % =========================================
\vspace*{.02in}
\begin{center} 
\color{blue}
{\bf More general learning (Durr et al. 2008)}\\
\color{black}
{\Large
\begin{displaymath}
\Delta w = \eta(Axy+Bx+Cy+D)
\end{displaymath}
}
\end{center}



\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf An example of exploration \\ by a recurrent NN with random weights}\\


\begin{figure}[h]
\begin{center}
\includegraphics[width=10cm,height=4cm]{path-by-hopfield-no-learning.eps}
\end{center}
\end{figure}
\color{black}
{\Large Can it learn so that path will finish at the start point?}
\end{center}


\clearpage  % =========================================
\vspace*{.01in}
\begin{center} 
\color{blue}
{\bf Our Aim}\\
\bigskip
\color{black}
$\Downarrow$\\
\color{red}
{\Large Learning should occur during a random exploration\\
\color{black}
\bigskip
with a \color{red} spiking neural network \color{black} to seek its biological plausibility}

%% expection to bridge between humanm & computer
%% weight modified during action
\end{center}


\clearpage  % =========================================
\vspace*{.01in}
\begin{center} 
\color{blue}
{\bf Learning by STDP \\
\bigskip
{\Large (Spiking neuron's version of Hebbian Learning)}}\\
\bigskip
\color{black}
$\Downarrow$\\
{\Large The topic is still open!}
\end{center}


%% expection to bridge between humanm & computer
%% weight modified during action



\clearpage  % =========================================
\vspace*{.01in}
\begin{center} 
\color{blue}
{\bf Meunier et al. (2005)}\\
\bigskip
\color{black}
$\Downarrow$\\
\color{black}
{\Large \it ``Up to now, nobody has been able to show how it is possible to learn with STDP...''}
\end{center}


\clearpage  % =========================================
\vspace*{.01in}
\begin{center} 
\color{blue}
{\bf Farries et al. (2007)}\\
\bigskip
\color{black}
$\Downarrow$\\
\color{black}
{\Large \it ``Although synaptic plasticity is widely believed to be a major component of learning, it is unclear how STDP itself could serve as a mechanism for general purpose learning.''}
\end{center}


\clearpage  % =========================================
\vspace*{.01in}
\begin{center} 
\color{blue}
{\bf Still the method is not so fruitful}\\
\color{black}
\bigskip
{\Large but} \\
\color{blue}
\bigskip
{\bf Why not challenge?}
\end{center}



\clearpage  % =========================================
\begin{center} \color{blue}
{\bf Architecture we are planning}
%% A network of spike-response-model neurons with a recurrent architecture
\bigskip
\color{black}
\begin{figure}[bhtp]
\begin{center}
\includegraphics[width=8cm,height=5cm]{rnn.eps}
\end{center}
\end{figure}

$\Downarrow$ 
{\large a.k.a.\\
Echo State Network, Liquid State Network or Reservoir Computing}
\end{center}



\clearpage  % =========================================
\begin{center} \color{blue}
{\bf Integrate-and-Fire Model}
\bigskip
\color{black}
{\Large
\color{red}
\begin{displaymath}
u_i(t) = u_r +(u_i(t - \delta t) - u_r) \exp (- \delta t/ \tau) + \sum_j w_{ij}f_j (t - \delta t)
\end{displaymath}
}
\end{center}
\begin{figure}[bhtp]
\begin{center}
   \includegraphics[width=14cm,height=5cm]{spike-iaf.eps}
\end{center}
\end{figure}




\clearpage  % =========================================
\begin{center} \color{blue}
{\bf What is STDP?}
\bigskip
\color{red}
{\Large
      \begin{eqnarray*}
         W(\Delta t) = \left\{
         \begin{array}{ll}\nonumber
         \hspace{.18in}A_{+} \exp (-\Delta t/\tau_{+})
                          & \hspace{.2in}\mbox{if}\hspace{.2in}\Delta t \ge 0\\
         - A_{-} \exp (-\Delta t/\tau_{-})
                          & \hspace{.2in}\mbox{if}\hspace{.2in}\Delta t < 0\\
         \end{array}
         \right.\\
      \end{eqnarray*}
\color{black} where \hspace{.2in} \color{red} $\Delta t = t_{post} - t_{pre}$\\
\bigskip
\color{black}{\large $W(\Delta t)$}}
\begin{figure}[bhtp]
\begin{center}
   \includegraphics[width=12cm,height=3.5cm]{stdp.eps}
\end{center}
\end{figure}

\end{center}


\clearpage  % =========================================
\begin{center} \color{blue}
{\bf In short}\\
\bigskip
\color{black}
{\Large
\color{red} potentiation
\color{black} occurs when a pre-synaptic neuron fires \hspace{2.5in}\\
\hspace*{2.5in}shortly before a post-synaptic neuron\\
\bigskip
and \\
\bigskip
\color{red}depression
\color{black} occurs when the post-synaptic neuron fires shortly after.}
\end{center}

\clearpage  % =========================================
\vspace*{.2in}
\begin{center} \color{blue}
{\bf It's too basic to be applied \\ 
\bigskip
to a real simulation}\\
\end{center}


\clearpage  % =========================================
\begin{center} \color{blue}
{\bf Reward-modulated STDP Learning}\\
(Florian 2007)
\bigskip
\color{red}
{\Large
\begin{displaymath}
w_{ij} (t + \delta t) = w_{ij} (t) + \gamma r(t + \delta t) \zeta_{ij}(t)
\end{displaymath}
\color{black}
where
\begin{displaymath}
\zeta_{ij} (t) = P^{+}_{ij} (t)f_i(t) + P^{-}_{ij} (t)f_i(t)
\end{displaymath}
\begin{displaymath}
P^{+}_{ij} (t) = P^{+}_{ij} (t-\delta t) \exp (- \delta t/\tau_{+}) + A_{+}f_j(t)
\end{displaymath}
\begin{displaymath}
P^{-}_{ij} (t) = P^{-}_{ij} (t-\delta t) \exp (- \delta t/\tau_{-}) + A_{-}f_j(t)
\end{displaymath}
}
\end{center}

%% As you noticed reinforcement learning is used 


\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf Reinforcement Learning (RL)}\\
\hspace*{1in}\\
\color{black}
{\Large
What an agent learns is a \color{red} {\it policy}\\
\color{black}
\bigskip
and \\
\bigskip
policy is how to select an \color{red} {\it action} \color{black} in a given
\color{red} {\it situation (state)} \color{black}\\
\bigskip
maximizing \\
total \color{red} {\it rewards} \color{black} the agent occasionary will receive from the environment
}
\end{center}


\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf Policy indicates agent\\
which action should be chosen \\ in any possible situation}\\
{\Large 
\color{black}
\bigskip
$\Downarrow$\\
\bigskip
While in GA \\
Agent decides actions following its \color{red} chromosome\\
\color{black}
\bigskip
in RL \\
Agent decides actions following its \color{red} policy \color{black} it has already learned}
\end{center}

%% modify situation-actions prob. table by reward
%% state in our case => location + how-much-gas-remained



\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf However} \\
\bigskip
\color{black}
{\Large 
\bigskip
\color{black}
our world is with 
\color{red}
no obstacle, no wall, no corridor,
\color{black}
only one goal \\
\bigskip
$\Downarrow$ \\
Everywhere no reward except for one point (goal)\\
\bigskip
$\Downarrow$ \\
A needle in a Haystack \\
(Rewards are not likely to be encountered by a random exploration)}
\end{center}

%% this makes the search a needlo in a haystack


\clearpage  % =========================================

\begin{center}\color{blue}
{\bf We have not succeeded yet \\ 
%%\color{black} $\Downarrow$ \color{blue}\\ 
\bigskip
\color{black}
{\Large Nevertheless} \\
\bigskip
\color{blue}
we think of\\  a further extention of the benchmark \\ as a bigger challenge}\\
\bigskip
\color{black}
$\Downarrow$\\
\color{red}
{\Large What if the rover carries containers for fuel?}\\
\end{center}


\clearpage  % =========================================
\begin{center}\color{blue}
{\bf Jeep Problem}\\
\color{black}
\bigskip
{\Large Where jeep explore is
\color{red} 1-D \color{black} desert \\
\bigskip
Jeep can unload its fuels anywhere in the desert \\
\bigskip
Fuels can be filled only at the base \\
\bigskip
Jeep can go back to the base \color{red} $n$ \color{black} times to re-fill its tank \\
\bigskip
$\Downarrow$ thus \\
The jeep should maximize its penetration
}
\end{center}




\clearpage  % =========================================
\begin{center}\color{blue}
{\bf An example of a success}
\end{center}
\begin{figure}[h]
\begin{center}
\includegraphics[width=18cm,height=8cm]{jeep-1d.eps}
\end{center}
\end{figure}


\clearpage  % =========================================
\vspace*{.01in}
\begin{center}\color{blue}
{\bf Let's give the rover the same condition} \\
\bigskip
\color{black}
{\Large 
$\Downarrow$ \\
\bigskip
A very tough benchmark, and\\
 an infinitely large number of solutions (when it's in 2-D)\\
\bigskip
$\Downarrow$ \\
Good as a benchmark to check \color{red} our \color{black} intelligence of \\
\color{red}{\it a differenct action in a same environment}
}

\end{center}



\clearpage  % =========================================
%% \vspace*{.02in}
\begin{center} 
\color{blue}
{\bf Let me conclude}\\
\color{black}
%% \hspace*{1in}\\
\bigskip
{\Large To claim artificial NN to be intelligent \\
\bigskip
a different action more or less should be made under identical situation. \\
\bigskip
$\Downarrow$ \\
\bigskip
We want to, or we are going to realize it \\
by\\
Spiking NN with learning by STDP \\
\bigskip
also \\
to be more biologically plausible.}
\end{center}



\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf SONY's AIBO}

\bigskip
\begin{figure}[h]
\begin{center}
\rotatebox[origin=c]{270}{
\includegraphics[width=3cm,height=6cm]{aibo.eps}}
%%\includegraphics[width=3cm,height=6cm]{sony-2.eps}
\end{center}
\end{figure}
\color{black}
{\Large can excellently learn but still repeats \color{red} same action in the same situation}
\end{center}

%% my mother ... very old ... such a toy robot ... at the beggining fascinated with a tricky or funny actions but sooner or later loose her interest due to this same repetition


\clearpage  % =========================================
\vspace*{.02in}
\begin{center} 
\color{blue}
{\bf Hoping  collaborations \\ 
\bigskip
to design \\
\bigskip
an agent like real human intelligence\\
\hspace*{1in}\\
\color{red}
Thank You!}
\end{center}

\end{huge}
\end{document}



\clearpage  % =========================================
\begin{center}\color{blue}
{\bf An analytical  solution} \\
\bigskip
\color{black}{\Large 
by N. J. Fine (1947)
\begin{displaymath}
%% d_0 = 0, \hspace{.2in} d_n = d_{n-1}+\frac{1}{2n-1}
d_0 = 0, \hspace{.2in} d_n = d_{n-1}+1/(2n-1).
\end{displaymath}
}
\end{center}
\begin{figure}[h]
\begin{center}
\includegraphics[width=16cm,height=4cm]{jeep-1d-solution.eps}
\end{center}
\end{figure}



\clearpage  % =========================================
\begin{center}\color{blue}
{\bf The optimum solution for n=3}
\end{center}
\begin{figure}[h]
\begin{center}
\includegraphics[width=18cm,height=8cm]{jeep-1d-optimum.eps}
\end{center}
\end{figure}



\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf Reinforcement Learning \\ under greedy strategy would be the one}\\
\bigskip
\color{black}
\bigskip
{\Large 
Agent follows its policy with a probability $\epsilon$ \\
and \\
choose an action at random with probability $(1-\epsilon)$ \\
\bigskip
$\Downarrow$ \\
Even after learning, actions are somehow different from run to run}
\end{center}



\clearpage  % =========================================
\begin{center} 
\color{blue}
{\bf Candidate}\\
\color{black}
\hspace*{1in}\\
$\Downarrow$\\
\hspace*{1in}\\
%%\color{red}
{\Large Spiking Response Model under STDP}

\end{center}

\clearpage  % =========================================
\begin{center} \color{blue}
{\bf A spiking Neuron \\--- Spiking Recponce Model (SRM)}
\bigskip
\color{black}
{\Large
\begin{displaymath}
u_i(t) = \eta_i (s_i) + \sum_j w_j \sum_f \epsilon(t - t_{j}^{f})
\end{displaymath}
}
\end{center}

%% Hodgkin-Huxley, Leaky Integrate and Fire,


\clearpage  % =========================================
\hfill {\large (Cont'd)}
\begin{center} \color{blue}
{\bf Two Kernels}
\bigskip
\color{black}
\begin{displaymath}
\epsilon(s) = \exp(-(s-d)/\tau_m)(1-exp(-s-d)/\tau_s)   
\end{displaymath}
(for $d < x < 20$ otherwise $\epsilon(s) = 0$)
\hspace*{1in}\\
\begin{displaymath}
\eta(s) = - \exp (-s/\tau_m)
\end{displaymath}
\end{center}

koko








\clearpage  % =========================================
\begin{center}\color{blue}
{\bf Number of steps grows linearly}
\end{center}
\begin{figure}[bhtp]
\begin{center}
\includegraphics[width=14cm,height=8.5cm]{navi-1-new.eps}
\end{center}
\end{figure}



\clearpage  % =========================================
%%\hfill{\Large (cont'd)}
\begin{center}\color{blue}
{\bf The task in the modern version}\\
\color{black}
\bigskip
{\LARGE Maximize the distance a Jeep can penetrate into 
\color{red} 1-D \color{black} desert.}
\end{center}
{\LARGE
\begin{itemize}
\item Jeep can move a unit distance with a unit fuel.
\item Jeep can unload its fuels anywhere in the desert.
\item Fuels can be filled only at the base.
\item Jeep can go back to the base \color{red} $n$ \color{black} times to re-fill its tank.
\end{itemize}
}


\clearpage  % =========================================
\hspace*{1in}\\
\begin{center}\color{blue}
{\bf A success at a small scale \\is not a royal road to a real success.}
\end{center}

%% with the task being ...
\clearpage  % =========================================
\begin{center}\color{blue}
{\bf An extension to 2-D Desert}\\
\end{center}
\color{black}
\noindent
{\LARGE
\begin{center}
\color{red}``Maximize the penetration''
\color{black} or 
\color{red}``Find the needle'' \\ 
\color{black}by repeating the procedure:
\end{center}
\begin{itemize}
\item[] (1) Start the base.
\item[] (2) Navigate the desert.
\item[] (3) Put fuels somewhere, or find the fuels to get.
\item[] (4) Return to the base.
\end{itemize}
}

\clearpage  % =========================================
\begin{center}
\color{blue}{\bf Still resists to be solved!}\\
\bigskip
\color{black}{\LARGE
Evolutionary Computation \\
...... \\
Reinforcement learning \\
$\Downarrow$ \\
\color{red} An extremely tough benchmark\\
\color{black}
%%$\Downarrow$ \\
A challenge to a navigation robot community.}
\end{center}

\clearpage  % =========================================
\begin{center} \color{blue}
{\bf Summary}
\end{center}
\color{black} {\LARGE
\begin{itemize}
\setlength{\itemsep}{0pt}
\item Two challenges:
   \begin{itemize}
   \item[$\star$] A needle hidden in a huge grid world.
                  %%{\Large (n=1 of the below)}
   \item[$\star$] \color{red}2-D Jeep Problem by a learning navigation robot.\color{black}
         \begin{itemize}
         \item[] {\Large Can a robot learn to solve this? Then how, if any?}
         \end{itemize}
     \end{itemize}
\item We have to be careful because we \\
\hspace*{0.6in}{\it tend-to-hear-what-we-want-to-hear.}
%%\item Success at a small scale is not a success in a real world.
\item Success report sometimes holds true only in a small world.


\end{itemize}
}






\end{huge}
\end{document}


