{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3e0fd009",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:18:25.694270Z",
     "start_time": "2022-12-21T14:18:25.689308Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19752d02",
   "metadata": {},
   "source": [
    "## 1. sigmoid"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61c39ff9",
   "metadata": {},
   "source": [
    "$$\n",
    "\\begin{equation}\n",
    "\\begin{split}\n",
    "\\sigma(x)=&\\frac{1}{1+\\exp(-x)}\\\\\n",
    "\\sigma'(x)=&\\frac{1}{1+\\exp(-x)}\\cdot\\left(\\frac{\\exp(-x)}{1+\\exp(-x)}\\right)\\\\\n",
    "=&\\sigma(x)\\cdot(1-\\sigma(x))\n",
    "\\end{split}\n",
    "\\end{equation}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "712ada20",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:18:56.364919Z",
     "start_time": "2022-12-21T14:18:56.361697Z"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.tensor(1., requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b59eb7b2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:18:58.039226Z",
     "start_time": "2022-12-21T14:18:58.033116Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1., requires_grad=True)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2954b719",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:19:03.135490Z",
     "start_time": "2022-12-21T14:19:03.132685Z"
    }
   },
   "outputs": [],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "52f8345a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:17:13.664355Z",
     "start_time": "2022-12-21T14:17:13.661315Z"
    }
   },
   "outputs": [],
   "source": [
    "y = x.sigmoid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "11834af2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:17:16.692371Z",
     "start_time": "2022-12-21T14:17:16.685271Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.7311, grad_fn=<SigmoidBackward0>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "85058642",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:17:24.778205Z",
     "start_time": "2022-12-21T14:17:24.772897Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7310585786300049"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1/(1+np.exp(-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "aee2b514",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:17:42.045216Z",
     "start_time": "2022-12-21T14:17:42.041184Z"
    }
   },
   "outputs": [],
   "source": [
    "y.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1db231de",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:17:45.510450Z",
     "start_time": "2022-12-21T14:17:45.504834Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.1966)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "752f7c65",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:18:10.236761Z",
     "start_time": "2022-12-21T14:18:10.230619Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.1966, grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.sigmoid()*(1-x.sigmoid())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "de599e8a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:18:31.682109Z",
     "start_time": "2022-12-21T14:18:31.679012Z"
    }
   },
   "outputs": [],
   "source": [
    "sigmoid = nn.Sigmoid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4aeb6c11",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:18:42.674460Z",
     "start_time": "2022-12-21T14:18:42.667354Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.1966, grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigmoid(x)*(1-sigmoid(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d705c70",
   "metadata": {},
   "source": [
    "### 1.1 multi variables 多元形式"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f78cda0",
   "metadata": {},
   "source": [
    "$$\n",
    "\\begin{split}\n",
    "&y: \\mathbb R^n \\rightarrow \\mathbb R\\\\\n",
    "&y=\\sum_i\\sigma(x_i)=\\sigma(x_1)+\\sigma(x_2)+\\sigma(x_3)+\\cdots\\\\\n",
    "&\\frac{\\partial y}{\\partial x_1}=\\frac{\\partial y}{\\partial \\sigma(x_1)}\\cdot\\frac{\\partial \\sigma(x_1)}{\\partial x_1}=1\\cdot \\sigma'(x_1)=\\sigma(x_1)\\cdot(1-\\sigma(x_1))\\\\\n",
    "&\\frac{\\partial y}{\\partial x_2}=\\frac{\\partial y}{\\partial \\sigma(x_2)}\\cdot\\frac{\\partial \\sigma(x_2)}{\\partial x_2}=1\\cdot \\sigma'(x_2)=\\sigma(x_2)\\cdot(1-\\sigma(x_2))\n",
    "\\end{split}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a170b4f2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:21:16.930507Z",
     "start_time": "2022-12-21T14:21:16.927246Z"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.tensor([1., 2., 3.], requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "28ad92b0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:21:20.400132Z",
     "start_time": "2022-12-21T14:21:20.394956Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 2., 3.], requires_grad=True)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "207aed3d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:21:33.866170Z",
     "start_time": "2022-12-21T14:21:33.863052Z"
    }
   },
   "outputs": [],
   "source": [
    "y = x.sigmoid().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "aae94138",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:21:37.659070Z",
     "start_time": "2022-12-21T14:21:37.653139Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.5644, grad_fn=<SumBackward0>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "768fc89e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:21:58.365597Z",
     "start_time": "2022-12-21T14:21:58.362737Z"
    }
   },
   "outputs": [],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "66ada1a0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:22:04.651730Z",
     "start_time": "2022-12-21T14:22:04.648331Z"
    }
   },
   "outputs": [],
   "source": [
    "y.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "6106351f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:22:15.633303Z",
     "start_time": "2022-12-21T14:22:15.627301Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.1966, 0.1050, 0.0452])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7c8fb8e2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:22:52.283229Z",
     "start_time": "2022-12-21T14:22:52.277048Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.1966, 0.1050, 0.0452], grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.sigmoid() * (1-x.sigmoid())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "752e44c8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:23:04.923198Z",
     "start_time": "2022-12-21T14:23:04.920210Z"
    }
   },
   "outputs": [],
   "source": [
    "sigmoid = nn.Sigmoid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "67c3ec8e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:23:14.293035Z",
     "start_time": "2022-12-21T14:23:14.287461Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.1966, 0.1050, 0.0452], grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigmoid(x)*(1-sigmoid(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07ace81a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "5d320185",
   "metadata": {},
   "source": [
    "### 1.2 自定义函数"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f7bccd6",
   "metadata": {},
   "source": [
    "$$\n",
    "\\begin{split}\n",
    "&y=\\sigma^2(x)\\\\\n",
    "&y'=2\\sigma(x)\\cdot(\\sigma'(x))=2\\sigma(x)\\cdot(\\sigma(x)(1-\\sigma(x)))\n",
    "\\end{split}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8f933296",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:23:49.314325Z",
     "start_time": "2022-12-21T14:23:49.311149Z"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.tensor(2., requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "41f93718",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:23:57.314962Z",
     "start_time": "2022-12-21T14:23:57.311915Z"
    }
   },
   "outputs": [],
   "source": [
    "y = x.sigmoid()*x.sigmoid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7e776d6d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:23:58.838686Z",
     "start_time": "2022-12-21T14:23:58.833639Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.7758, grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "44cff663",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:24:24.895720Z",
     "start_time": "2022-12-21T14:24:24.893088Z"
    }
   },
   "outputs": [],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "1adcff5a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:24:28.985525Z",
     "start_time": "2022-12-21T14:24:28.981695Z"
    }
   },
   "outputs": [],
   "source": [
    "y.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "0d02d2e6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:24:31.996468Z",
     "start_time": "2022-12-21T14:24:31.989945Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.1850)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "0b75dad1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-21T14:24:55.668346Z",
     "start_time": "2022-12-21T14:24:55.661259Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.1850, grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2*x.sigmoid()*(x.sigmoid() * (1-x.sigmoid()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5af58cb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}