sigmoid grad

1 files changed, 700 insertions, 0 deletions

diff --git a/learn_torch/grad/01_sigmoid.ipynb b/learn_torch/grad/01_sigmoid.ipynb
new file mode 100644
index 0000000..568b259
--- /dev/null
+++ b/learn_torch/grad/01_sigmoid.ipynb
@@ -0,0 +1,700 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "3e0fd009",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:18:25.694270Z",
+     "start_time": "2022-12-21T14:18:25.689308Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import torch\n",
+    "from torch import nn"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "19752d02",
+   "metadata": {},
+   "source": [
+    "## 1. sigmoid"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "61c39ff9",
+   "metadata": {},
+   "source": [
+    "$$\n",
+    "\\begin{equation}\n",
+    "\\begin{split}\n",
+    "\\sigma(x)=&\\frac{1}{1+\\exp(-x)}\\\\\n",
+    "\\sigma'(x)=&\\frac{1}{1+\\exp(-x)}\\cdot\\left(\\frac{\\exp(-x)}{1+\\exp(-x)}\\right)\\\\\n",
+    "=&\\sigma(x)\\cdot(1-\\sigma(x))\n",
+    "\\end{split}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "712ada20",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:18:56.364919Z",
+     "start_time": "2022-12-21T14:18:56.361697Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "x = torch.tensor(1., requires_grad=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "b59eb7b2",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:18:58.039226Z",
+     "start_time": "2022-12-21T14:18:58.033116Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor(1., requires_grad=True)"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "2954b719",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:19:03.135490Z",
+     "start_time": "2022-12-21T14:19:03.132685Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "x.grad"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "52f8345a",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:17:13.664355Z",
+     "start_time": "2022-12-21T14:17:13.661315Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "y = x.sigmoid()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "11834af2",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:17:16.692371Z",
+     "start_time": "2022-12-21T14:17:16.685271Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor(0.7311, grad_fn=<SigmoidBackward0>)"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "y"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "85058642",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:17:24.778205Z",
+     "start_time": "2022-12-21T14:17:24.772897Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.7310585786300049"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "1/(1+np.exp(-1))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "aee2b514",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:17:42.045216Z",
+     "start_time": "2022-12-21T14:17:42.041184Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "y.backward()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "1db231de",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:17:45.510450Z",
+     "start_time": "2022-12-21T14:17:45.504834Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor(0.1966)"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x.grad"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "752f7c65",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:18:10.236761Z",
+     "start_time": "2022-12-21T14:18:10.230619Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor(0.1966, grad_fn=<MulBackward0>)"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x.sigmoid()*(1-x.sigmoid())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "de599e8a",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:18:31.682109Z",
+     "start_time": "2022-12-21T14:18:31.679012Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "sigmoid = nn.Sigmoid()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "4aeb6c11",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:18:42.674460Z",
+     "start_time": "2022-12-21T14:18:42.667354Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor(0.1966, grad_fn=<MulBackward0>)"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sigmoid(x)*(1-sigmoid(x))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0d705c70",
+   "metadata": {},
+   "source": [
+    "### 1.1 multi variables 多元形式"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1f78cda0",
+   "metadata": {},
+   "source": [
+    "$$\n",
+    "\\begin{split}\n",
+    "&y: \\mathbb R^n \\rightarrow \\mathbb R\\\\\n",
+    "&y=\\sum_i\\sigma(x_i)=\\sigma(x_1)+\\sigma(x_2)+\\sigma(x_3)+\\cdots\\\\\n",
+    "&\\frac{\\partial y}{\\partial x_1}=\\frac{\\partial y}{\\partial \\sigma(x_1)}\\cdot\\frac{\\partial \\sigma(x_1)}{\\partial x_1}=1\\cdot \\sigma'(x_1)=\\sigma(x_1)\\cdot(1-\\sigma(x_1))\\\\\n",
+    "&\\frac{\\partial y}{\\partial x_2}=\\frac{\\partial y}{\\partial \\sigma(x_2)}\\cdot\\frac{\\partial \\sigma(x_2)}{\\partial x_2}=1\\cdot \\sigma'(x_2)=\\sigma(x_2)\\cdot(1-\\sigma(x_2))\n",
+    "\\end{split}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "a170b4f2",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:21:16.930507Z",
+     "start_time": "2022-12-21T14:21:16.927246Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "x = torch.tensor([1., 2., 3.], requires_grad=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "28ad92b0",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:21:20.400132Z",
+     "start_time": "2022-12-21T14:21:20.394956Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([1., 2., 3.], requires_grad=True)"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "207aed3d",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:21:33.866170Z",
+     "start_time": "2022-12-21T14:21:33.863052Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "y = x.sigmoid().sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "aae94138",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:21:37.659070Z",
+     "start_time": "2022-12-21T14:21:37.653139Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor(2.5644, grad_fn=<SumBackward0>)"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "y"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "768fc89e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:21:58.365597Z",
+     "start_time": "2022-12-21T14:21:58.362737Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "x.grad"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "66ada1a0",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:22:04.651730Z",
+     "start_time": "2022-12-21T14:22:04.648331Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "y.backward()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "6106351f",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:22:15.633303Z",
+     "start_time": "2022-12-21T14:22:15.627301Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([0.1966, 0.1050, 0.0452])"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x.grad"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "7c8fb8e2",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:22:52.283229Z",
+     "start_time": "2022-12-21T14:22:52.277048Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([0.1966, 0.1050, 0.0452], grad_fn=<MulBackward0>)"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x.sigmoid() * (1-x.sigmoid())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "752e44c8",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:23:04.923198Z",
+     "start_time": "2022-12-21T14:23:04.920210Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "sigmoid = nn.Sigmoid()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "67c3ec8e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:23:14.293035Z",
+     "start_time": "2022-12-21T14:23:14.287461Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([0.1966, 0.1050, 0.0452], grad_fn=<MulBackward0>)"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sigmoid(x)*(1-sigmoid(x))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "07ace81a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5d320185",
+   "metadata": {},
+   "source": [
+    "### 1.2 自定义函数"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5f7bccd6",
+   "metadata": {},
+   "source": [
+    "$$\n",
+    "\\begin{split}\n",
+    "&y=\\sigma^2(x)\\\\\n",
+    "&y'=2\\sigma(x)\\cdot(\\sigma'(x))=2\\sigma(x)\\cdot(\\sigma(x)(1-\\sigma(x)))\n",
+    "\\end{split}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "8f933296",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:23:49.314325Z",
+     "start_time": "2022-12-21T14:23:49.311149Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "x = torch.tensor(2., requires_grad=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "41f93718",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:23:57.314962Z",
+     "start_time": "2022-12-21T14:23:57.311915Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "y = x.sigmoid()*x.sigmoid()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "7e776d6d",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:23:58.838686Z",
+     "start_time": "2022-12-21T14:23:58.833639Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor(0.7758, grad_fn=<MulBackward0>)"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "y"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "44cff663",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:24:24.895720Z",
+     "start_time": "2022-12-21T14:24:24.893088Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "x.grad"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "1adcff5a",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:24:28.985525Z",
+     "start_time": "2022-12-21T14:24:28.981695Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "y.backward()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "id": "0d02d2e6",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:24:31.996468Z",
+     "start_time": "2022-12-21T14:24:31.989945Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor(0.1850)"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x.grad"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "id": "0b75dad1",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-12-21T14:24:55.668346Z",
+     "start_time": "2022-12-21T14:24:55.661259Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor(0.1850, grad_fn=<MulBackward0>)"
+      ]
+     },
+     "execution_count": 35,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "2*x.sigmoid()*(x.sigmoid() * (1-x.sigmoid()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d5af58cb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}