Activation Functions

import torch
import torch.nn.functional as F

x = torch.tensor([-2.0, -1.0, 0.0, 1.0, 2.0])

print(F.relu(x))         # [0, 0, 0, 1, 2]
print(torch.sigmoid(x))  # [0.12, 0.27, 0.50, 0.73, 0.88]
print(torch.tanh(x))     # [-0.96, -0.76, 0.00, 0.76, 0.96]
print(F.leaky_relu(x, 0.01))  # [-0.02, -0.01, 0, 1, 2]
print(F.elu(x, 1.0))     # [-0.865, -0.632, 0, 1, 2]
print(F.gelu(x))         # [-0.045, -0.159, 0, 0.841, 1.955]

# Trong mạng nơ-ron:
model = torch.nn.Sequential(
    torch.nn.Linear(784, 256),
    torch.nn.ReLU(),          # hàm kích hoạt sau mỗi lớp ẩn
    torch.nn.Linear(256, 10),
    # Không cần Softmax ở đây — CrossEntropyLoss tự xử lý log-softmax
)
criterion = torch.nn.CrossEntropyLoss()

import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_grad(x):
    s = sigmoid(x)
    return s * (1 - s)

def relu(x):
    return np.maximum(0, x)

def relu_grad(x):
    return (x > 0).astype(float)

def leaky_relu(x, alpha=0.01):
    return np.where(x > 0, x, alpha * x)

def leaky_relu_grad(x, alpha=0.01):
    return np.where(x > 0, 1.0, alpha)

def elu(x, alpha=1.0):
    return np.where(x > 0, x, alpha * (np.exp(x) - 1))

def elu_grad(x, alpha=1.0):
    return np.where(x > 0, 1.0, alpha * np.exp(x))

def gelu(x):
    # Hendrycks-Gimpel tanh approximation
    return 0.5 * x * (1 + np.tanh(
        np.sqrt(2 / np.pi) * (x + 0.044715 * x ** 3)
    ))

# So sánh 6 hàm trên cùng đầu vào
xs = np.linspace(-3, 3, 7)
for name, fn in [("sigmoid", sigmoid), ("relu", relu),
                 ("leaky", leaky_relu), ("elu", elu),
                 ("gelu", gelu), ("tanh", np.tanh)]:
    print(name, np.round(fn(xs), 3))