Recurrent Neural Networks (RNN) Explained: Hidden State, Shared Weights and Why They Forget

import numpy as np
import torch
import torch.nn as nn

rng = np.random.default_rng(0)
words = ["I", "love", "machine", "learning"]
x = rng.normal(size=(4, 3))                    # each word is a vector of 3 numbers (an "embedding")
Wx = rng.normal(size=(3, 5)) * 0.5            # input -> hidden weights   (shared by every step)
Wh = rng.normal(size=(5, 5)) * 0.5            # hidden -> hidden weights  (shared by every step)
b = np.zeros(5)

h = np.zeros(5)                                # h0: the starting memory is empty
for t, word in enumerate(words, start=1):
    h = np.tanh(x[t - 1] @ Wx + h @ Wh + b)    # the SAME formula, the SAME weights, every time step
    print(f"t={t}  read '{word:8s}'  hidden state = {h.round(2)}")

# check against PyTorch's own RNN layer, loaded with the same weights
rnn = nn.RNN(input_size=3, hidden_size=5, batch_first=True)
with torch.no_grad():
    rnn.weight_ih_l0.copy_(torch.tensor(Wx.T)); rnn.weight_hh_l0.copy_(torch.tensor(Wh.T))
    rnn.bias_ih_l0.zero_(); rnn.bias_hh_l0.zero_()
    out, h_last = rnn(torch.tensor(x, dtype=torch.float32).unsqueeze(0))
print("\nmatches torch.nn.RNN:", np.allclose(h_last[0, 0].numpy(), h, atol=1e-5))
print("parameters in this one RNN layer:", sum(p.numel() for p in rnn.parameters()), "(the same count for 4 steps or 4,000)")

import matplotlib.pyplot as plt
import torch
import torch.nn as nn

torch.manual_seed(0)
T = 60
rnn = nn.RNN(1, 32, batch_first=True)          # untrained, default start
x = torch.randn(1, T, 1, requires_grad=True)
out, _ = rnn(x)
out[0, -1].sum().backward()                    # the loss depends on the LAST step only
grad = x.grad[0, :, 0].abs()                   # how much does each input position influence it?

for t in (T - 1, T - 5, T - 10, T - 20, T - 40, 0):
    print(f"influence of the input {T - 1 - t:2d} steps back: {grad[t].item():.2e}")

fig, ax = plt.subplots(figsize=(6.4, 3.6))
ax.semilogy(range(T), grad.numpy() + 1e-30, lw=2, color="#DC2626")
ax.set_xlabel("position in the sequence (the last step is on the right)"); ax.set_ylabel("gradient size (log)")
ax.spines[["top", "right"]].set_visible(False)
fig.tight_layout()

import torch
import torch.nn as nn

# The task: the label is the sign of the FIRST number. Everything after it is noise.
def make_batch(batch, T):
    x = torch.randn(batch, T, 1)
    y = (x[:, 0, 0] > 0).float()
    return x, y

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.rnn = nn.RNN(1, 32, batch_first=True)
        self.head = nn.Linear(32, 1)
    def forward(self, x):
        _, h = self.rnn(x)                       # h: the memory after reading the whole sequence
        return self.head(h[0]).squeeze(-1)

print("sequence length   accuracy of a plain RNN (mean of 3 runs)   (50% = pure guessing)")
for T in (5, 15, 30, 60):
    accs = []
    for seed in range(3):                                # three different random starts
        torch.manual_seed(seed)
        net = Net(); opt = torch.optim.Adam(net.parameters(), lr=0.01)
        for step in range(500):
            x, y = make_batch(64, T)
            loss = nn.functional.binary_cross_entropy_with_logits(net(x), y)
            opt.zero_grad(); loss.backward()
            nn.utils.clip_grad_norm_(net.parameters(), 1.0)
            opt.step()
        x, y = make_batch(2000, T)
        accs.append(((net(x) > 0).float() == y).float().mean().item())
    print(f"{T:10d}        {sum(accs) / 3:.3f}      (runs: {', '.join(f'{a:.2f}' for a in accs)})")

import torch
import torch.nn as nn

text = ("practice makes perfect. slow and steady wins the race. a stitch in time saves nine. "
        "well begun is half done. knowledge is power. patience is a virtue. ") * 6
chars = sorted(set(text)); idx = {c: i for i, c in enumerate(chars)}
data = torch.tensor([idx[c] for c in text])
print(f"{len(text)} characters, {len(chars)} different symbols")

class CharRNN(nn.Module):
    def __init__(self, vocab, hidden=96):
        super().__init__()
        self.emb = nn.Embedding(vocab, 24); self.rnn = nn.RNN(24, hidden, batch_first=True); self.out = nn.Linear(hidden, vocab)
    def forward(self, x, h=None):
        y, h = self.rnn(self.emb(x), h)
        return self.out(y), h

torch.manual_seed(0)
model = CharRNN(len(chars)); opt = torch.optim.Adam(model.parameters(), lr=0.01)
seq = 40
for step in range(1, 501):
    starts = torch.randint(0, len(data) - seq - 1, (64,))
    x = torch.stack([data[s:s + seq] for s in starts]); y = torch.stack([data[s + 1:s + seq + 1] for s in starts])
    logits, _ = model(x)
    loss = nn.functional.cross_entropy(logits.reshape(-1, len(chars)), y.reshape(-1))
    opt.zero_grad(); loss.backward(); nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
    if step in (1, 50, 200, 500):
        print(f"step {step:3d}   loss {loss.item():.3f}")

# generate: feed the model its own output one character at a time
model.eval(); c = torch.tensor([[idx["p"]]]); h = None; out = ["p"]
with torch.no_grad():
    for _ in range(150):
        logits, h = model(c, h)
        c = torch.multinomial(torch.softmax(logits[0, -1] / 0.7, dim=0), 1).view(1, 1)
        out.append(chars[c.item()])
print("\ngenerated:", "".join(out))

import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn

# A made-up daily series with a yearly cycle, a slow trend and noise (think: average daily temperature)
rng = np.random.default_rng(1)
t = np.arange(1500)
series = 25 + 10 * np.sin(2 * np.pi * t / 365) + 0.004 * t + rng.normal(0, 1.5, len(t))
series = (series - series.mean()) / series.std()

def make_windows(s, seq_len):
    X = np.array([s[i:i + seq_len] for i in range(len(s) - seq_len)])
    y = s[seq_len:]
    return X[..., None], y                    # X shape: (samples, seq_len, 1)

seq_len, split = 30, 1100
X, y = make_windows(series, seq_len)
X_tr, y_tr, X_te, y_te = [torch.tensor(a, dtype=torch.float32) for a in (X[:split], y[:split], X[split:], y[split:])]

class Forecaster(nn.Module):
    def __init__(self):
        super().__init__(); self.rnn = nn.RNN(1, 32, batch_first=True); self.head = nn.Linear(32, 1)
    def forward(self, x):
        _, h = self.rnn(x); return self.head(h[0]).squeeze(-1)

torch.manual_seed(0)
model = Forecaster(); opt = torch.optim.Adam(model.parameters(), lr=0.005)
for epoch in range(40):
    perm = torch.randperm(len(X_tr))
    for i in range(0, len(perm), 64):
        b = perm[i:i + 64]
        loss = nn.functional.mse_loss(model(X_tr[b]), y_tr[b]); opt.zero_grad(); loss.backward(); opt.step()

with torch.no_grad():
    pred = model(X_te)
mse = lambda a: ((a - y_te) ** 2).mean().item()
print(f"RNN test MSE            : {mse(pred):.3f}")
print(f"'same as yesterday' MSE : {mse(X_te[:, -1, 0]):.3f}   <- the baseline every model must beat")

fig, ax = plt.subplots(figsize=(8.2, 3.4))
ax.plot(y_te[:120].numpy(), color="#9CA3AF", lw=2, label="actual"); ax.plot(pred[:120].numpy(), color="#2563EB", lw=2, label="RNN prediction")
ax.set_xlabel("day (test period)"); ax.set_ylabel("standardised value"); ax.legend(frameon=False); ax.spines[["top", "right"]].set_visible(False)
fig.tight_layout()

import torch.nn as nn

rnn = nn.RNN(input_size=1, hidden_size=50, batch_first=True)        # input shape: (batch, time steps, features)
# drop-in replacements with gates:
# nn.GRU(...)   nn.LSTM(...)   bidirectional=True for a bidirectional layer

out, h_last = rnn(x)      # out: the hidden state at every step; h_last: the final state