Skip to content

Commit

Permalink
Feat: We added the horizon weighing to the distribution losses, simil… (
Browse files Browse the repository at this point in the history 
#1233)

Co-authored-by: Marco <[email protected]>
  • Loading branch information
@mwamsojo @marcopeix
mwamsojo and marcopeix authored Jan 22, 2025
1 parent cefaf73 commit 5ebaa22
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 27 deletions.
84 changes: 58 additions & 26 deletions nbs/losses.pytorch.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2475,7 +2475,7 @@
"\n",
" \"\"\"\n",
" def __init__(self, distribution, level=[80, 90], quantiles=None,\n",
" num_samples=1000, return_params=False, **distribution_kwargs):\n",
" num_samples=1000, return_params=False, horizon_weight = None, **distribution_kwargs):\n",
" super(DistributionLoss, self).__init__()\n",
"\n",
" qs, self.output_names = level_to_outputs(level)\n",
Expand All @@ -2489,6 +2489,12 @@
" self.quantiles = torch.nn.Parameter(qs, requires_grad=False)\n",
" num_qk = len(self.quantiles)\n",
"\n",
" # Generate a horizon weight tensor from the array\n",
" if horizon_weight is not None:\n",
" horizon_weight = torch.Tensor(horizon_weight.flatten())\n",
" self.horizon_weight = horizon_weight\n",
"\n",
"\n",
" if \"num_pieces\" not in distribution_kwargs:\n",
" num_pieces = 5\n",
" else:\n",
Expand Down Expand Up @@ -2610,36 +2616,62 @@
"\n",
" return samples, sample_mean, quants\n",
"\n",
" def __call__(self,\n",
" y: torch.Tensor,\n",
" distr_args: torch.Tensor,\n",
" mask: Union[torch.Tensor, None] = None):\n",
"\n",
"\n",
" def _compute_weights(self, y, mask):\n",
" \"\"\"\n",
" Compute final weights for each datapoint (based on all weights and all masks)\n",
" Set horizon_weight to a ones[H] tensor if not set.\n",
" If set, check that it has the same length as the horizon in x.\n",
" \"\"\"\n",
" Computes the negative log-likelihood objective function. \n",
" To estimate the following predictive distribution:\n",
" if mask is None:\n",
" mask = torch.ones_like(y, device=y.device)\n",
" else:\n",
" mask = mask.unsqueeze(1) # Add Q dimension.\n",
"\n",
" $$\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta) \\\\quad \\mathrm{and} \\\\quad -\\log(\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta))$$\n",
"\n",
" where $\\\\theta$ represents the distributions parameters. It aditionally \n",
" summarizes the objective signal using a weighted average using the `mask` tensor. \n",
" # get uniform weights if none\n",
" if self.horizon_weight is None:\n",
" self.horizon_weight = torch.ones(mask.shape[-1])\n",
" else:\n",
" assert mask.shape[-1] == len(self.horizon_weight), \\\n",
" 'horizon_weight must have same length as Y'\n",
" weights = self.horizon_weight.clone()\n",
" weights = torch.ones_like(mask, device=mask.device) * weights.to(mask.device)\n",
" return weights * mask\n",
" \n",
"\n",
" **Parameters**<br>\n",
" `y`: tensor, Actual values.<br>\n",
" `distr_args`: Constructor arguments for the underlying Distribution type.<br>\n",
" `loc`: Optional tensor, of the same shape as the batch_shape + event_shape\n",
" of the resulting distribution.<br>\n",
" `scale`: Optional tensor, of the same shape as the batch_shape+event_shape \n",
" of the resulting distribution.<br>\n",
" `mask`: tensor, Specifies date stamps per serie to consider in loss.<br>\n",
"\n",
" **Returns**<br>\n",
" `loss`: scalar, weighted loss function against which backpropagation will be performed.<br>\n",
" \"\"\"\n",
" # Instantiate Scaled Decoupled Distribution\n",
" distr = self.get_distribution(distr_args=distr_args, **self.distribution_kwargs)\n",
" loss_values = -distr.log_prob(y)\n",
" loss_weights = mask\n",
" return weighted_average(loss_values, weights=loss_weights)"
" def __call__(self,\n",
" y: torch.Tensor,\n",
" distr_args: torch.Tensor,\n",
" mask: Union[torch.Tensor, None] = None):\n",
" \"\"\"\n",
" Computes the negative log-likelihood objective function. \n",
" To estimate the following predictive distribution:\n",
"\n",
" $$\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta) \\\\quad \\mathrm{and} \\\\quad -\\log(\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta))$$\n",
"\n",
" where $\\\\theta$ represents the distributions parameters. It aditionally \n",
" summarizes the objective signal using a weighted average using the `mask` tensor. \n",
" \n",
" **Parameters**<br>\n",
" `y`: tensor, Actual values.<br>\n",
" `distr_args`: Constructor arguments for the underlying Distribution type.<br>\n",
" `loc`: Optional tensor, of the same shape as the batch_shape + event_shape\n",
" of the resulting distribution.<br>\n",
" `scale`: Optional tensor, of the same shape as the batch_shape+event_shape \n",
" of the resulting distribution.<br>\n",
" `mask`: tensor, Specifies date stamps per serie to consider in loss.<br>\n",
"\n",
" **Returns**<br>\n",
" `loss`: scalar, weighted loss function against which backpropagation will be performed.<br>\n",
" \"\"\"\n",
" # Instantiate Scaled Decoupled Distribution\n",
" distr = self.get_distribution(distr_args=distr_args, **self.distribution_kwargs)\n",
" loss_values = -distr.log_prob(y)\n",
" loss_weights = self._compute_weights(y=y, mask=mask)\n",
" return weighted_average(loss_values, weights=loss_weights)"
]
},
{
Expand Down
2 changes: 2 additions & 0 deletions neuralforecast/_modidx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,8 @@
'neuralforecast/losses/pytorch.py'),
'neuralforecast.losses.pytorch.DistributionLoss.__init__': ( 'losses.pytorch.html#distributionloss.__init__',
'neuralforecast/losses/pytorch.py'),
'neuralforecast.losses.pytorch.DistributionLoss._compute_weights': ( 'losses.pytorch.html#distributionloss._compute_weights',
'neuralforecast/losses/pytorch.py'),
'neuralforecast.losses.pytorch.DistributionLoss.get_distribution': ( 'losses.pytorch.html#distributionloss.get_distribution',
'neuralforecast/losses/pytorch.py'),
'neuralforecast.losses.pytorch.DistributionLoss.sample': ( 'losses.pytorch.html#distributionloss.sample',
Expand Down
30 changes: 29 additions & 1 deletion neuralforecast/losses/pytorch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1873,6 +1873,7 @@ def __init__(
quantiles=None,
num_samples=1000,
return_params=False,
horizon_weight=None,
**distribution_kwargs,
):
super(DistributionLoss, self).__init__()
Expand All @@ -1888,6 +1889,11 @@ def __init__(
self.quantiles = torch.nn.Parameter(qs, requires_grad=False)
num_qk = len(self.quantiles)

# Generate a horizon weight tensor from the array
if horizon_weight is not None:
horizon_weight = torch.Tensor(horizon_weight.flatten())
self.horizon_weight = horizon_weight

if "num_pieces" not in distribution_kwargs:
num_pieces = 5
else:
Expand Down Expand Up @@ -2011,6 +2017,28 @@ def sample(self, distr_args: torch.Tensor, num_samples: Optional[int] = None):

return samples, sample_mean, quants

def _compute_weights(self, y, mask):
"""
Compute final weights for each datapoint (based on all weights and all masks)
Set horizon_weight to a ones[H] tensor if not set.
If set, check that it has the same length as the horizon in x.
"""
if mask is None:
mask = torch.ones_like(y, device=y.device)
else:
mask = mask.unsqueeze(1) # Add Q dimension.

# get uniform weights if none
if self.horizon_weight is None:
self.horizon_weight = torch.ones(mask.shape[-1])
else:
assert mask.shape[-1] == len(
self.horizon_weight
), "horizon_weight must have same length as Y"
weights = self.horizon_weight.clone()
weights = torch.ones_like(mask, device=mask.device) * weights.to(mask.device)
return weights * mask

def __call__(
self,
y: torch.Tensor,
Expand Down Expand Up @@ -2041,7 +2069,7 @@ def __call__(
# Instantiate Scaled Decoupled Distribution
distr = self.get_distribution(distr_args=distr_args, **self.distribution_kwargs)
loss_values = -distr.log_prob(y)
loss_weights = mask
loss_weights = self._compute_weights(y=y, mask=mask)
return weighted_average(loss_values, weights=loss_weights)

# %% ../../nbs/losses.pytorch.ipynb 74
Expand Down

0 comments on commit 5ebaa22

Please sign in to comment.