In [12]:

import numpy as np
import torch
from torch import Tensor, nn

import numpy as np import torch from torch import Tensor, nn

Normalising for NLLLoss¶

Divide NLL by log(N), where N is the size of the choice.

In [13]:

for n in range(1, 9):
    o = torch.rand(128, n)
    p = nn.Softmax(-1)(o)
    lp = nn.LogSoftmax(-1)(o)
    norm = np.log(n)
    labels = torch.argmax(p, -1)

    loss = nn.NLLLoss()(lp, labels)
    nloss = loss / norm
    print(loss, nloss)

for n in range(1, 9): o = torch.rand(128, n) p = nn.Softmax(-1)(o) lp = nn.LogSoftmax(-1)(o) norm = np.log(n) labels = torch.argmax(p, -1) loss = nn.NLLLoss()(lp, labels) nloss = loss / norm print(loss, nloss)

tensor(0.) tensor(nan)
tensor(0.5516) tensor(0.7958)
tensor(0.8900) tensor(0.8101)
tensor(1.1101) tensor(0.8008)
tensor(1.2883) tensor(0.8004)
tensor(1.4584) tensor(0.8140)
tensor(1.6098) tensor(0.8273)
tensor(1.7186) tensor(0.8265)

Normalising for KLD¶

In [19]:

def kld(mu: Tensor, log_var: Tensor) -> Tensor:
    return torch.mean(
        -0.5 * torch.sum(1 + log_var - mu**2 - log_var.exp(), dim=1), dim=0
    )

def kld(mu: Tensor, log_var: Tensor) -> Tensor: return torch.mean( -0.5 * torch.sum(1 + log_var - mu**2 - log_var.exp(), dim=1), dim=0 )

In [22]:

for n in range(1, 9):
    fake = torch.rand(128, 128, n)
    mu = fake.mean(dim=0)
    log_var = torch.log(fake.var(dim=0))
    loss = kld(mu, log_var)
    nloss = loss / n
    print(loss, nloss)

for n in range(1, 9): fake = torch.rand(128, 128, n) mu = fake.mean(dim=0) log_var = torch.log(fake.var(dim=0)) loss = kld(mu, log_var) nloss = loss / n print(loss, nloss)

tensor(0.9069) tensor(0.9069)
tensor(1.8243) tensor(0.9121)
tensor(2.7375) tensor(0.9125)
tensor(3.6475) tensor(0.9119)
tensor(4.5428) tensor(0.9086)
tensor(5.4654) tensor(0.9109)
tensor(6.3581) tensor(0.9083)
tensor(7.3023) tensor(0.9128)

In [ ]: