From 5c3ff032a4d2fc50d96f8f94672086ddde45ca75 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fran=C3=A7ois=20Fleuret?= Date: Sat, 2 Mar 2024 01:04:42 +0100 Subject: [PATCH 1/8] Update. --- elbo.tex | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/elbo.tex b/elbo.tex index 4c6cb24..563ec3c 100644 --- a/elbo.tex +++ b/elbo.tex @@ -148,4 +148,20 @@ $\theta$ and $\alpha$ to maximize it maximizes $\log \, p_\theta(x_n)$ and brings $q_\alpha(z \mid x_n)$ close to $p_\theta(z \mid x_n)$. +\medskip + +A point that may be important in practice is +% +\begin{align*} + & \expect_{Z \sim q_\alpha(z \mid x_n)} \left[ \log \frac{p_\theta(x_n,Z)}{q_\alpha(Z \mid x_n)} \right] \\ + & = \expect_{Z \sim q_\alpha(z \mid x_n)} \left[ \log \frac{p_\theta(x_n \mid Z) p_\theta(Z)}{q_\alpha(Z \mid x_n)} \right] \\ + & = \expect_{Z \sim q_\alpha(z \mid x_n)} \left[ \log \, p_\theta(x_n \mid Z) \right] \\ + & \hspace*{7em} - \dkl(q_\alpha(z \mid x_n) \, \| \, p_\theta(z)). +\end{align*} +% +This form is useful because for certain $p_\theta$ and $q_\alpha$, for +instance if they are Gaussian, the KL term can be computed exactly +instead of through sampling, which removes one source of noise in the +optimization process. + \end{document} -- 2.20.1 From 00ccb7a22366144caa8278b72f62ea2b5f331d8e Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fran=C3=A7ois=20Fleuret?= Date: Thu, 25 Apr 2024 08:29:45 +0200 Subject: [PATCH 2/8] Update. --- dlscore.tex | 163 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 dlscore.tex diff --git a/dlscore.tex b/dlscore.tex new file mode 100644 index 0000000..b72c511 --- /dev/null +++ b/dlscore.tex @@ -0,0 +1,163 @@ +%% -*- mode: latex; mode: reftex; mode: flyspell; coding: utf-8; tex-command: "pdflatex.sh" -*- + +\documentclass[11pt,a4paper,twocolumn,twoside]{article} +\usepackage[a4paper,top=2.5cm,bottom=2cm,left=2.5cm,right=2.5cm]{geometry} +\usepackage[utf8]{inputenc} +\usepackage{cmbright} + +\begin{document} + +\noindent One point per item if you know precisely the meaning of the +listed word(s) + +\section{Machine Learning} + +\begin{enumerate} + + \item VC dimension + \item over-fitting, under-fitting + \item logistic regression + \item Q-value + \item kernel trick + \item boosting + \item feature design + \item linear regression + \item expectation-maximization, GMM + \item SVM + \item Bellman equation + \item decision tree + \item train/validation/test sets + \item naive Bayesian model + \item autoregressive model + \item bias-variance dilemma + \item policy gradient + \item random forest + \item k-NN + \item perceptron algorithm + +\end{enumerate} + + +\section{Deep-Learning} + +\begin{enumerate} + + \item Adam + \item softmax + \item residual connections + \item autograd + \item ReLU + \item dropout + \item CLIP + \item Xavier's initialisation + \item Vanishing gradient + \item LeNet + \item ViT + \item transposed convolution layer + \item checkpoint (during the forward pass) + \item minibatch + \item masked model + \item supervised / unsupervised + \item data augmentation + \item attention block + \item SGD + \item batchnorm + \item gradient clipping + \item tokenizer + \item VAE + \item weight decay + \item GELU + \item LSTM, GRU + \item GAN + \item resnet + \item straight-through estimator + \item convolution layer + \item pre-training / fine-tuning + \item perplexity + \item logits + \item cls token + \item forward pass + \item Transformer (original one), GPT + \item backward pass + \item autoencoder, denoising autoencoder + \item layer norm + \item GNN + \item diffusion model + \item cross-entropy + \item max pooling, average pooling + \item RNN + \item contrastive loss + \item positional encoding + \item causal model + \item attention layer + \item SSL + \item MSE + \item positional encoding + \item tensor + +\end{enumerate} + +\section{Math} + +\begin{enumerate} + \item Hessian + \item random variable + \item matrix + \item entropy, mutual information + \item dot product + \item mean, variance + \item L2 norm + \item chain rule (differentiation) + \item Fourier transform + \item continuity, Lipschitz continuity + \item chain rule (probability) + \item polynomial + \item Cantor's diagonal argument + \item Jacobian + \item linear operator + \item gradient + \item Bayes' thorem + \item vector + \item joint law, product law + \item Gaussian distribution + \item distribution + \item determinant, rank + \item eigen-decomposition, svd + \item maximum likelihood + \item Central Limit Theorem + +\end{enumerate} + +\section{Compute Science} + +\begin{enumerate} + + \item polymorphism + \item recursion + \item value passed by reference + \item binary search + \item quick sort + \item parallel scan + \item mutability + \item Turing machine + \item FP32 + \item iterator + \item interpreter, compiler + \item anonymous function + \item set + \item binary heap + \item mutex + \item cache memory + \item scope of a variable or function + \item dynamic programming + \item hash table + \item big-O notation + \item Turing complete + \item class inheritance + \item closure + \item loop unrolling + \item complexity + +\end{enumerate} + +\end{document} -- 2.20.1 From d3ec58e881d629993d490e7b6b3a6a5f7492fc8b Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fran=C3=A7ois=20Fleuret?= Date: Thu, 25 Apr 2024 08:29:58 +0200 Subject: [PATCH 3/8] Update. --- dlscore.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlscore.tex b/dlscore.tex index b72c511..ae997f5 100644 --- a/dlscore.tex +++ b/dlscore.tex @@ -49,7 +49,7 @@ listed word(s) \item ReLU \item dropout \item CLIP - \item Xavier's initialisation + \item Xavier's initialization \item Vanishing gradient \item LeNet \item ViT -- 2.20.1 From 449f7b43b1fb2b9e30cf099c02037b1dc51276c4 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fran=C3=A7ois=20Fleuret?= Date: Thu, 25 Apr 2024 08:35:05 +0200 Subject: [PATCH 4/8] Update. --- dlscore.tex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dlscore.tex b/dlscore.tex index ae997f5..5097381 100644 --- a/dlscore.tex +++ b/dlscore.tex @@ -1,7 +1,7 @@ %% -*- mode: latex; mode: reftex; mode: flyspell; coding: utf-8; tex-command: "pdflatex.sh" -*- \documentclass[11pt,a4paper,twocolumn,twoside]{article} -\usepackage[a4paper,top=2.5cm,bottom=2cm,left=2.5cm,right=2.5cm]{geometry} +\usepackage[a4paper,top=2cm,bottom=2cm,left=2.5cm,right=2.5cm]{geometry} \usepackage[utf8]{inputenc} \usepackage{cmbright} @@ -128,10 +128,12 @@ listed word(s) \end{enumerate} -\section{Compute Science} +\section{Computer Science} \begin{enumerate} +%% \itemsep0em + \item polymorphism \item recursion \item value passed by reference -- 2.20.1 From cc8788660c8f69048778d7bc5781100b7a54fbe8 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fran=C3=A7ois=20Fleuret?= Date: Thu, 25 Apr 2024 08:38:15 +0200 Subject: [PATCH 5/8] Update. --- dlscore.tex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dlscore.tex b/dlscore.tex index 5097381..e23cf19 100644 --- a/dlscore.tex +++ b/dlscore.tex @@ -100,6 +100,7 @@ listed word(s) \section{Math} \begin{enumerate} + \item Hessian \item random variable \item matrix @@ -132,8 +133,6 @@ listed word(s) \begin{enumerate} -%% \itemsep0em - \item polymorphism \item recursion \item value passed by reference -- 2.20.1 From 6f02a4dbc2799135ef135da72a1c1f83b690c9e5 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fran=C3=A7ois=20Fleuret?= Date: Thu, 25 Apr 2024 08:40:09 +0200 Subject: [PATCH 6/8] Update. --- dlscore.tex | 1 - 1 file changed, 1 deletion(-) diff --git a/dlscore.tex b/dlscore.tex index e23cf19..dd742d1 100644 --- a/dlscore.tex +++ b/dlscore.tex @@ -92,7 +92,6 @@ listed word(s) \item attention layer \item SSL \item MSE - \item positional encoding \item tensor \end{enumerate} -- 2.20.1 From 321e2a37cbb0e723e6a768541d0d793fa68b2faa Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fran=C3=A7ois=20Fleuret?= Date: Thu, 25 Apr 2024 08:54:05 +0200 Subject: [PATCH 7/8] Update. --- dlscore.tex | 1 + 1 file changed, 1 insertion(+) diff --git a/dlscore.tex b/dlscore.tex index dd742d1..743ad62 100644 --- a/dlscore.tex +++ b/dlscore.tex @@ -82,6 +82,7 @@ listed word(s) \item autoencoder, denoising autoencoder \item layer norm \item GNN + \item learning rate schedule \item diffusion model \item cross-entropy \item max pooling, average pooling -- 2.20.1 From cc493838b758b04d940b4cf7f57deee9b12548d4 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fran=C3=A7ois=20Fleuret?= Date: Thu, 25 Apr 2024 09:01:36 +0200 Subject: [PATCH 8/8] Update. --- dlscore.tex | 1 + 1 file changed, 1 insertion(+) diff --git a/dlscore.tex b/dlscore.tex index 743ad62..6fd06ac 100644 --- a/dlscore.tex +++ b/dlscore.tex @@ -20,6 +20,7 @@ listed word(s) \item Q-value \item kernel trick \item boosting + \item PCA \item feature design \item linear regression \item expectation-maximization, GMM -- 2.20.1