From 070904ec73b192aff94b8f2e1db3a9fd0d80c320 Mon Sep 17 00:00:00 2001 From: Francois Fleuret Date: Sat, 20 Aug 2022 11:59:40 +0200 Subject: [PATCH] Update. --- autoencoder.tex | 45 ++++++++++++++++++++++++++++++ single-attention.tex | 66 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 autoencoder.tex diff --git a/autoencoder.tex b/autoencoder.tex new file mode 100644 index 0000000..705ab21 --- /dev/null +++ b/autoencoder.tex @@ -0,0 +1,45 @@ +%% -*- mode: latex; mode: reftex; mode: flyspell; coding: utf-8; tex-command: "pdflatex.sh" -*- + +\documentclass[11pt,a4paper,twoside]{article} +\usepackage[a4paper,top=2.5cm,bottom=2cm,left=2.5cm,right=2.5cm]{geometry} +\usepackage{amsmath,amssymb,dsfont} +\usepackage[pdftex]{graphicx} +\usepackage[colorlinks=true,linkcolor=blue,urlcolor=blue,citecolor=blue]{hyperref} +\usepackage{tikz} +\usetikzlibrary{arrows.meta} +\usepackage{cmbright} + +\begin{document} + +\definecolor{green}{rgb}{0.0,0.50,0.0} +\tikzset{>={Straight Barb[angle'=80,scale=1.1]}} + +\begin{tikzpicture} + +\draw[->] (0, 0) -- ++(0, 2); +\draw[->] (0, 0) -- ++(2.5, 0.6); +\draw[->] (0, 0) -- ++(3, 0) node[midway,below,yshift=-0.5em] {Original space ${\cal X}$}; + +\draw[fill=green!50,draw=none,shift={(0.2, 0.7)},scale=0.5] + (0, 0) to[out=20,in=140] (1.5, -0.2) to [out=60,in=160] + (5, 0.5) to[out=130,in=60] + cycle; + +\shade[thin,left color=green!10,right color=green!50,draw=none,shift={(0.2, 0.7)},scale=0.5] + (0, 0) to[out=10,in=140] (3.3, -0.8) to [out=60,in=190] (5, 0.5) to[out=130,in=60] cycle; + + \draw[->] (4.8, 0.8) -- ++(0, 2); + \draw[->] (4.8, 0.8) -- ++(2, 0) node[midway,below,yshift=-0.5em] {Latent space ${\cal F}$}; + + \draw[thin,fill=green!30,draw=none,shift={(5.4, 1.1)},rotate=20] + (0, 0) -- (1, 0) -- (1, 1) -- (0, 1) -- cycle; + + \draw[thick,->,red] + (1.5, 1.3) to [out=55,in=150] node[midway,above,xshift=6pt,yshift=2pt] {$f$} (5.7, 2); + + \draw[thick,->,blue] (1.5, 1.3) ++(4.03, 0.3) to [out=150,in=55] + node[midway,below,xshift=2pt,yshift=-2pt] {$g$} ++(-3.6, -0.5); + +\end{tikzpicture} + +\end{document} diff --git a/single-attention.tex b/single-attention.tex index 01a181c..d21595e 100644 --- a/single-attention.tex +++ b/single-attention.tex @@ -113,4 +113,70 @@ Single-head attention \end{frame} +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\begin{frame} + +\begin{center} + +\begin{tikzpicture} + +\node[value, minimum height=0.8cm,minimum width=0.7cm] (K) at (0, 0) {$K$}; +\draw[very thick,yellow] ([yshift=1pt]K.north west) -- ([yshift=1pt]K.north east); +\draw[very thick,red] ([xshift=-1pt]K.north west) -- ([xshift=-1pt]K.south west); + +\node[value, minimum height=1.2cm,minimum width=0.7cm] (Q) [above=1cm of K] {$Q$}; +\draw[very thick,yellow] ([yshift=1pt]Q.north west) -- ([yshift=1pt]Q.north east); +\draw[very thick,cyan] ([xshift=-1pt]Q.north west) -- ([xshift=-1pt]Q.south west); + +\node[value, minimum height=0.8cm,minimum width=1.0cm] (V) [below=1cm of K] {$V$}; +\draw[very thick,orange] ([yshift=1pt]V.north west) -- ([yshift=1pt]V.north east); +\draw[very thick,red] ([xshift=-1pt]V.north west) -- ([xshift=-1pt]V.south west); + +\node[operation,minimum height=0.4cm] (mulWq) [left=1cm of Q.center] {$\cdot$}; +\node[operation,minimum height=0.4cm] (mulWk) [left=1cm of K.center] {$\cdot$}; +\node[operation,minimum height=0.4cm] (mulWv) [left=1cm of V.center] {$\cdot$}; + +\node[value, minimum height=1.2cm,minimum width=0.5cm] (X) [left=1cm of mulWq] {$X$}; +\draw[very thick,cyan] ([xshift=-1pt]X.north west) -- ([xshift=-1pt]X.south west); +\draw[very thick,green] ([yshift=1pt]X.north west) -- ([yshift=1pt]X.north east); + +\node[parameter, minimum height=0.5cm,minimum width=0.7cm] (Wq) [above=0.25 cm of X] {$W^Q$}; +\draw[very thick,green] ([xshift=-1pt]Wq.north west) -- ([xshift=-1pt]Wq.south west); +\draw[very thick,yellow] ([yshift=1pt]Wq.north west) -- ([yshift=1pt]Wq.north east); + +\node[value, minimum height=0.8cm,minimum width=0.3cm] (X') [below=1.2cm of X] {$X'$}; + +\node[operation,minimum height=0.4cm,minimum width=0.4cm] (att) [right=0.5cm of K] {$\cdot\transpose$}; +\node[operation,minimum height=0.4cm,minimum width=0.4cm] (sm) [right=0.25cm of att] {$\softmax$}; + +\node[value, minimum height=1.2cm,minimum width=0.8cm] (A) [right=0.5cm of sm] {$A$}; +\draw[very thick,cyan] ([xshift=-1pt]A.north west) -- ([xshift=-1pt]A.south west); +\draw[very thick,red] ([yshift=1pt]A.north west) -- ([yshift=1pt]A.north east); + +\node[operation,minimum height=0.4cm,minimum width=0.4cm] (prod) [right=0.5cm of A] {$\cdot$}; + +\node[value, minimum height=1.2cm,minimum width=1.0cm] (Y) [right=0.5cm of prod] {$Y$}; +\draw[very thick,orange] ([yshift=1pt]Y.north west) -- ([yshift=1pt]Y.north east); +\draw[very thick,cyan] ([xshift=-1pt]Y.north west) -- ([xshift=-1pt]Y.south west); + +\draw[v2f,rounded corners=1mm] (X) -- (mulWq); +\draw[v2f,rounded corners=1mm] (Wq) -| (mulWq); +\draw[f2v,rounded corners=1mm] (mulWq) -- ([xshift=-1pt]Q.west); + +\draw[v2f,rounded corners=1mm] (K) -- (att); +\draw[v2f,rounded corners=1mm] (Q) -| (att); +\draw[f2f,rounded corners=1mm] (att) -- (sm); +\draw[f2v,rounded corners=1mm] (sm) -- ([xshift=-1pt]A.west); + +\draw[v2f,rounded corners=1mm] (A) -- (prod); +\draw[v2f,rounded corners=1mm] (V) -| (prod); +\draw[f2v,rounded corners=1mm] (prod) -- ([xshift=-1pt]Y.west); + +\end{tikzpicture} + +\end{center} + +\end{frame} + \end{document} -- 2.20.1