Update.

diff --git a/autoencoder.tex b/autoencoder.tex
new file mode 100644 (file)
index 0000000..705ab21
--- /dev/null
+++ b/autoencoder.tex
@@ -0,0 +1,45 @@
+%% -*- mode: latex; mode: reftex; mode: flyspell; coding: utf-8; tex-command: "pdflatex.sh" -*-
+
+\documentclass[11pt,a4paper,twoside]{article}
+\usepackage[a4paper,top=2.5cm,bottom=2cm,left=2.5cm,right=2.5cm]{geometry}
+\usepackage{amsmath,amssymb,dsfont}
+\usepackage[pdftex]{graphicx}
+\usepackage[colorlinks=true,linkcolor=blue,urlcolor=blue,citecolor=blue]{hyperref}
+\usepackage{tikz}
+\usetikzlibrary{arrows.meta}
+\usepackage{cmbright}
+
+\begin{document}
+
+\definecolor{green}{rgb}{0.0,0.50,0.0}
+\tikzset{>={Straight Barb[angle'=80,scale=1.1]}}
+
+\begin{tikzpicture}
+
+\draw[->] (0, 0) -- ++(0, 2);
+\draw[->] (0, 0) -- ++(2.5, 0.6);
+\draw[->] (0, 0) -- ++(3, 0) node[midway,below,yshift=-0.5em] {Original space ${\cal X}$};
+
+\draw[fill=green!50,draw=none,shift={(0.2, 0.7)},scale=0.5]
+  (0, 0) to[out=20,in=140] (1.5, -0.2) to [out=60,in=160]
+  (5, 0.5) to[out=130,in=60]
+  cycle;
+
+\shade[thin,left color=green!10,right color=green!50,draw=none,shift={(0.2, 0.7)},scale=0.5]
+  (0, 0) to[out=10,in=140] (3.3, -0.8) to [out=60,in=190] (5, 0.5) to[out=130,in=60] cycle;
+
+  \draw[->] (4.8, 0.8) -- ++(0, 2);
+  \draw[->] (4.8, 0.8) -- ++(2, 0) node[midway,below,yshift=-0.5em] {Latent space ${\cal F}$};
+
+  \draw[thin,fill=green!30,draw=none,shift={(5.4, 1.1)},rotate=20]
+    (0, 0) -- (1, 0) -- (1, 1) -- (0, 1) -- cycle;
+
+  \draw[thick,->,red]
+    (1.5, 1.3) to [out=55,in=150] node[midway,above,xshift=6pt,yshift=2pt] {$f$} (5.7, 2);
+
+  \draw[thick,->,blue] (1.5, 1.3) ++(4.03, 0.3) to [out=150,in=55]
+    node[midway,below,xshift=2pt,yshift=-2pt] {$g$} ++(-3.6, -0.5);
+
+\end{tikzpicture}
+
+\end{document}

diff --git a/single-attention.tex b/single-attention.tex
index 01a181c..d21595e 100644 (file)
--- a/single-attention.tex
+++ b/single-attention.tex
@@ -113,4 +113,70 @@ Single-head attention
 
 \end{frame}
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+
+\begin{center}
+
+\begin{tikzpicture}
+
+\node[value,    minimum height=0.8cm,minimum width=0.7cm] (K) at (0, 0) {$K$};
+\draw[very thick,yellow] ([yshift=1pt]K.north west) -- ([yshift=1pt]K.north east);
+\draw[very thick,red] ([xshift=-1pt]K.north west) -- ([xshift=-1pt]K.south west);
+
+\node[value,    minimum height=1.2cm,minimum width=0.7cm] (Q) [above=1cm of K] {$Q$};
+\draw[very thick,yellow] ([yshift=1pt]Q.north west) -- ([yshift=1pt]Q.north east);
+\draw[very thick,cyan] ([xshift=-1pt]Q.north west) -- ([xshift=-1pt]Q.south west);
+
+\node[value,    minimum height=0.8cm,minimum width=1.0cm] (V) [below=1cm of K] {$V$};
+\draw[very thick,orange] ([yshift=1pt]V.north west) -- ([yshift=1pt]V.north east);
+\draw[very thick,red] ([xshift=-1pt]V.north west) -- ([xshift=-1pt]V.south west);
+
+\node[operation,minimum height=0.4cm] (mulWq) [left=1cm of Q.center] {$\cdot$};
+\node[operation,minimum height=0.4cm] (mulWk) [left=1cm of K.center] {$\cdot$};
+\node[operation,minimum height=0.4cm] (mulWv) [left=1cm of V.center] {$\cdot$};
+
+\node[value,    minimum height=1.2cm,minimum width=0.5cm] (X) [left=1cm of mulWq] {$X$};
+\draw[very thick,cyan] ([xshift=-1pt]X.north west) -- ([xshift=-1pt]X.south west);
+\draw[very thick,green] ([yshift=1pt]X.north west) -- ([yshift=1pt]X.north east);
+
+\node[parameter,    minimum height=0.5cm,minimum width=0.7cm] (Wq) [above=0.25 cm of X] {$W^Q$};
+\draw[very thick,green] ([xshift=-1pt]Wq.north west) -- ([xshift=-1pt]Wq.south west);
+\draw[very thick,yellow] ([yshift=1pt]Wq.north west) -- ([yshift=1pt]Wq.north east);
+
+\node[value,    minimum height=0.8cm,minimum width=0.3cm] (X') [below=1.2cm of X]  {$X'$};
+
+\node[operation,minimum height=0.4cm,minimum width=0.4cm] (att) [right=0.5cm of K] {$\cdot\transpose$};
+\node[operation,minimum height=0.4cm,minimum width=0.4cm] (sm) [right=0.25cm of att] {$\softmax$};
+
+\node[value,    minimum height=1.2cm,minimum width=0.8cm] (A) [right=0.5cm of sm] {$A$};
+\draw[very thick,cyan] ([xshift=-1pt]A.north west) -- ([xshift=-1pt]A.south west);
+\draw[very thick,red] ([yshift=1pt]A.north west) -- ([yshift=1pt]A.north east);
+
+\node[operation,minimum height=0.4cm,minimum width=0.4cm] (prod) [right=0.5cm of A] {$\cdot$};
+
+\node[value,    minimum height=1.2cm,minimum width=1.0cm] (Y) [right=0.5cm of prod] {$Y$};
+\draw[very thick,orange] ([yshift=1pt]Y.north west) -- ([yshift=1pt]Y.north east);
+\draw[very thick,cyan] ([xshift=-1pt]Y.north west) -- ([xshift=-1pt]Y.south west);
+
+\draw[v2f,rounded corners=1mm] (X) -- (mulWq);
+\draw[v2f,rounded corners=1mm] (Wq) -| (mulWq);
+\draw[f2v,rounded corners=1mm] (mulWq) -- ([xshift=-1pt]Q.west);
+
+\draw[v2f,rounded corners=1mm] (K) -- (att);
+\draw[v2f,rounded corners=1mm] (Q) -| (att);
+\draw[f2f,rounded corners=1mm] (att) -- (sm);
+\draw[f2v,rounded corners=1mm] (sm) -- ([xshift=-1pt]A.west);
+
+\draw[v2f,rounded corners=1mm] (A) -- (prod);
+\draw[v2f,rounded corners=1mm] (V) -| (prod);
+\draw[f2v,rounded corners=1mm] (prod) -- ([xshift=-1pt]Y.west);
+
+\end{tikzpicture}
+
+\end{center}
+
+\end{frame}
+
 \end{document}