From f9dc2acce452a0d7bdc415a079520239ca8977a2 Mon Sep 17 00:00:00 2001 From: Tom Diethe Date: Wed, 17 Oct 2018 23:10:08 +0100 Subject: [PATCH 01/70] Added NormalMeanPrecision and MultivariateNormalMeanPrecision - versions of the Normal and MultivariateNormal that take mean and precision as parameters instead of mean and variance Other minor fixes. --- mxfusion/components/distributions/__init__.py | 6 +- mxfusion/components/distributions/normal.py | 328 ++++++++++++++++++ mxfusion/components/distributions/wishart.py | 2 +- mxfusion/util/special.py | 2 +- .../normal_mean_precision_test.py | 305 ++++++++++++++++ 5 files changed, 638 insertions(+), 5 deletions(-) create mode 100644 testing/distributions/normal_mean_precision_test.py diff --git a/mxfusion/components/distributions/__init__.py b/mxfusion/components/distributions/__init__.py index 7a188e1..fc1275e 100644 --- a/mxfusion/components/distributions/__init__.py +++ b/mxfusion/components/distributions/__init__.py @@ -15,11 +15,11 @@ gp """ -__all__ = ['categorical', 'distribution', 'normal', 'pointmass', 'rand_gen', - 'univariate','gp', 'wishart', 'beta'] +__all__ = ['categorical', 'distribution', 'normal', 'pointmass', 'random_gen', + 'univariate', 'gp', 'wishart', 'beta'] from .distribution import Distribution -from .normal import Normal, MultivariateNormal +from .normal import Normal, MultivariateNormal, NormalMeanPrecision, MultivariateNormalMeanPrecision from .pointmass import PointMass from .categorical import Categorical from .gp import GaussianProcess, ConditionalGaussianProcess diff --git a/mxfusion/components/distributions/normal.py b/mxfusion/components/distributions/normal.py index ab0d105..711772c 100644 --- a/mxfusion/components/distributions/normal.py +++ b/mxfusion/components/distributions/normal.py @@ -1,5 +1,8 @@ import numpy as np import mxnet as mx +import itertools + +from ...util.special import log_determinant from ...common.config import get_default_MXNet_mode from ...common.exceptions import InferenceError from ..variables import Variable @@ -321,3 +324,328 @@ def _generate_outputs(self, shape): :type shape: tuple """ self.outputs = [('random_variable', Variable(value=self, shape=shape))] + + +class NormalMeanPrecision(UnivariateDistribution): + """ + The one-dimensional normal distribution, parameterized by mean and precision rather than mean and variance. + The normal distribution can be defined over a scalar random variable + or an array of random variables. In case of an array of random variables, the mean and precisions are broadcasted + to the shape of the output random variable (array). + + :param mean: Mean of the normal distribution. + :type mean: Variable + :param precision: Precision of the normal distribution. + :type precision: Variable + :param rand_gen: the random generator (default: MXNetRandomGenerator). + :type rand_gen: RandomGenerator + :param dtype: the data type for float point numbers. + :type dtype: numpy.float32 or numpy.float64 + :param ctx: the mxnet context (default: None/current context). + :type ctx: None or mxnet.cpu or mxnet.gpu + """ + def __init__(self, mean, precision, rand_gen=None, dtype=None, ctx=None): + if not isinstance(mean, Variable): + mean = Variable(value=mean) + if not isinstance(precision, Variable): + precision = Variable(value=precision) + + inputs = [('mean', mean), ('precision', precision)] + input_names = [k for k, _ in inputs] + output_names = ['random_variable'] + super(NormalMeanPrecision, self).__init__(inputs=inputs, outputs=None, + input_names=input_names, + output_names=output_names, + rand_gen=rand_gen, dtype=dtype, ctx=ctx) + + @UnivariateLogPDFDecorator() + def log_pdf(self, mean, precision, random_variable, F=None): + """ + Computes the logarithm of the probability density function (PDF) of the normal distribution. + + :param mean: the mean of the normal distribution. + :type mean: MXNet NDArray or MXNet Symbol + :param precision: the precision of the normal distributions. + :type precision: MXNet NDArray or MXNet Symbol + :param random_variable: the random variable of the normal distribution. + :type random_variable: MXNet NDArray or MXNet Symbol + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray). + :returns: log pdf of the distribution. + :rtypes: MXNet NDArray or MXNet Symbol + """ + F = get_default_MXNet_mode() if F is None else F + logvar = (F.log(precision) - np.log(2 * np.pi)) / 2 + logL = F.broadcast_add(logvar, F.broadcast_mul(F.square( + F.broadcast_minus(random_variable, mean)), -precision / 2)) * self.log_pdf_scaling + return logL + + @UnivariateDrawSamplesDecorator() + def draw_samples(self, mean, precision, rv_shape, num_samples=1, F=None): + """ + Draw samples from the normal distribution. + + :param mean: the mean of the normal distribution. + :type mean: MXNet NDArray or MXNet Symbol + :param precision: the precision of the normal distributions. + :type precision: MXNet NDArray or MXNet Symbol + :param rv_shape: the shape of each sample. + :type rv_shape: tuple + :param num_samples: the number of drawn samples (default: one). + :int num_samples: int + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray). + :returns: a set samples of the normal distribution. + :rtypes: MXNet NDArray or MXNet Symbol + """ + F = get_default_MXNet_mode() if F is None else F + out_shape = (num_samples,) + rv_shape + return F.broadcast_add(F.broadcast_div(self._rand_gen.sample_normal( + shape=out_shape, dtype=self.dtype, ctx=self.ctx), + F.sqrt(precision)), mean) + + @staticmethod + def define_variable(mean=0., precision=1., shape=None, rand_gen=None, + dtype=None, ctx=None): + """ + Creates and returns a random variable drawn from a normal distribution. + + :param mean: Mean of the distribution. + :param precision: Precision of the distribution. + :param shape: the shape of the random variable(s). + :type shape: tuple or [tuple] + :param rand_gen: the random generator (default: MXNetRandomGenerator). + :type rand_gen: RandomGenerator + :param dtype: the data type for float point numbers. + :type dtype: numpy.float32 or numpy.float64 + :param ctx: the mxnet context (default: None/current context). + :type ctx: None or mxnet.cpu or mxnet.gpu + :returns: the random variables drawn from the normal distribution. + :rtypes: Variable + """ + normal = NormalMeanPrecision(mean=mean, precision=precision, rand_gen=rand_gen, dtype=dtype, ctx=ctx) + normal._generate_outputs(shape=shape) + return normal.random_variable + + +class MultivariateNormalMeanPrecisionLogPDFDecorator(LogPDFDecorator): + + def _wrap_log_pdf_with_broadcast(self, func): + def log_pdf_broadcast(self, F, **kw): + """ + Computes the logarithm of the probability density/mass function (PDF/PMF) of the distribution. The inputs and outputs variables are in RTVariable format. + + Shape assumptions: + * mean is S x N x D + * precision is S x N x D x D + * random_variable is S x N x D + + Where: + * S, the number of samples, is optional. If more than one of the variables has samples, the number of samples in each variable must be the same. S is 1 by default if not a sampled variable. + * N is the number of data points. N can be any number of dimensions (N_1, N_2, ...) but must be broadcastable to the shape of random_variable. + * D is the dimension of the distribution. + + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) + :param kw: the dict of input and output variables of the distribution + :type kw: {str (name): MXNet NDArray or MXNet Symbol} + :returns: log pdf of the distribution + :rtypes: MXNet NDArray or MXNet Symbol + """ + variables = {name: kw[name] for name, _ in self.inputs} + variables['random_variable'] = kw['random_variable'] + rv_shape = variables['random_variable'].shape[1:] + + n_samples = max([get_num_samples(F, v) for v in variables.values()]) + + shapes_map = dict( + mean=(n_samples,) + rv_shape, + precision=(n_samples,) + rv_shape + (rv_shape[-1],), + random_variable=(n_samples,) + rv_shape) + variables = {name: broadcast_to_w_samples(F, v, shapes_map[name]) + for name, v in variables.items()} + res = func(self, F=F, **variables) + return res + return log_pdf_broadcast + + +class MultivariateNormalMeanPrecisionDrawSamplesDecorator(DrawSamplesDecorator): + + def _wrap_draw_samples_with_broadcast(self, func): + def draw_samples_broadcast(self, F, rv_shape, num_samples=1, + always_return_tuple=False, **kw): + """ + Draw a number of samples from the distribution. The inputs and outputs variables are in RTVariable format. + + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) + :param rv_shape: the shape of each sample + :type rv_shape: tuple + :param num_samples: the number of drawn samples (default: one) + :int num_samples: int + :param always_return_tuple: Whether return a tuple even if there is only one variables in outputs. + :type always_return_tuple: boolean + :param kw: the dict of input variables of the distribution + :type kw: {name: MXNet NDArray or MXNet Symbol} + :returns: a set samples of the distribution + :rtypes: MXNet NDArray or MXNet Symbol or [MXNet NDArray or MXNet Symbol] + """ + rv_shape = list(rv_shape.values())[0] + variables = {name: kw[name] for name, _ in self.inputs} + + isSamples = any([is_sampled_array(F, v) for v in variables.values()]) + if isSamples: + num_samples_inferred = max([get_num_samples(F, v) for v in + variables.values()]) + if num_samples_inferred != num_samples: + raise InferenceError("The number of samples in the nSamples argument of draw_samples of " + "Normal distribution must be the same as the number of samples given " + "to the inputs. nSamples: {} the inferred number of samples from " + "inputs: {}.".format(num_samples, num_samples_inferred)) + + shapes_map = dict( + mean=(num_samples,) + rv_shape, + precision=(num_samples,) + rv_shape + (rv_shape[-1],), + random_variable=(num_samples,) + rv_shape) + variables = {name: broadcast_to_w_samples(F, v, shapes_map[name]) + for name, v in variables.items()} + + res = func(self, F=F, rv_shape=rv_shape, num_samples=num_samples, + **variables) + if always_return_tuple: + res = (res,) + return res + return draw_samples_broadcast + + +class MultivariateNormalMeanPrecision(Distribution): + """ + The multi-dimensional normal distribution parameterized by mean and precision rather than mean and variance. + + :param mean: Mean of the normal distribution. + :type mean: Variable + :param precision: Precision matrix of the distribution. + :type precision: Variable + :param rand_gen: the random generator (default: MXNetRandomGenerator). + :type rand_gen: RandomGenerator + :param dtype: the data type for float point numbers. + :type dtype: numpy.float32 or numpy.float64 + :param ctx: the mxnet context (default: None/current context). + :type ctx: None or mxnet.cpu or mxnet.gpu + """ + def __init__(self, mean, precision, rand_gen=None, minibatch_ratio=1., + dtype=None, ctx=None): + self.minibatch_ratio = minibatch_ratio + if not isinstance(mean, Variable): + mean = Variable(value=mean) + if not isinstance(precision, Variable): + precision = Variable(value=precision) + + inputs = [('mean', mean), ('precision', precision)] + input_names = ['mean', 'precision'] + output_names = ['random_variable'] + super(MultivariateNormalMeanPrecision, self).__init__(inputs=inputs, outputs=None, + input_names=input_names, + output_names=output_names, + rand_gen=rand_gen, dtype=dtype, ctx=ctx) + + def replicate_self(self, attribute_map=None): + """ + Replicates this Factor, using new inputs, outputs, and a new uuid. + Used during model replication to functionally replicate a factor into a new graph. + + :param inputs: new input variables of the factor. + :type inputs: a dict of {'name' : Variable} or None + :param outputs: new output variables of the factor. + :type outputs: a dict of {'name' : Variable} or None + """ + replicant = super(MultivariateNormalMeanPrecision, self).replicate_self(attribute_map) + replicant.minibatch_ratio = self.minibatch_ratio + return replicant + + @MultivariateNormalMeanPrecisionLogPDFDecorator() + def log_pdf(self, mean, precision, random_variable, F=None): + """ + Computes the logarithm of the probability density function (PDF) of the normal distribution. + + :param mean: the mean of the normal distribution. + :type mean: MXNet NDArray or MXNet Symbol + :param precision: the precision of the distribution. + :type precision: MXNet NDArray or MXNet Symbol + :param random_variable: the random variable of the normal distribution. + :type random_variable: MXNet NDArray or MXNet Symbol + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray). + :returns: log pdf of the distribution. + :rtypes: MXNet NDArray or MXNet Symbol + """ + F = get_default_MXNet_mode() if F is None else F + N = mean.shape[-1] + c = N * np.log(2 * np.pi) + logdetl = -log_determinant(precision) + targets = random_variable - mean + + # TODO: Should be a way to do this without loops + sqnorm_z = F.zeros(random_variable.shape[:-1], dtype=self.dtype) + for ix in itertools.product(*map(range, random_variable.shape[:-1])): + sqnorm_z[ix] = F.dot(F.dot(targets[ix], precision[ix], transpose_a=True), targets[ix]) + + return -0.5 * (sqnorm_z + c + logdetl) + + @MultivariateNormalMeanPrecisionDrawSamplesDecorator() + def draw_samples(self, mean, precision, rv_shape, num_samples=1, F=None): + """ + Draw a number of samples from the normal distribution. + + :param mean: the mean of the normal distribution. + :type mean: MXNet NDArray or MXNet Symbol + :param precision: the precision of the normal distributions. + :type precision: MXNet NDArray or MXNet Symbol + :param rv_shape: the shape of each sample. + :type rv_shape: tuple + :param num_samples: the number of drawn samples (default: one). + :int num_samples: int + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray). + :returns: a set samples of the normal distribution + :rtypes: MXNet NDArray or MXNet Symbol + """ + F = get_default_MXNet_mode() if F is None else F + out_shape = (num_samples,) + rv_shape + (1,) + + # Use potri instead of potrf: + # https://mxnet.incubator.apache.org/api/python/symbol/linalg.html#mxnet.symbol.linalg.potri + lmat = F.linalg.potri(precision) + epsilon = self._rand_gen.sample_normal( + shape=out_shape, dtype=self.dtype, ctx=self.ctx) + lmat_eps = F.linalg.trmm(lmat, epsilon) + return F.broadcast_add(lmat_eps.sum(-1), mean) + + @staticmethod + def define_variable(shape, mean=0., precision=None, rand_gen=None, + minibatch_ratio=1., dtype=None, ctx=None): + """ + Creates and returns a random variable drawn from a normal distribution. + + :param mean: Mean of the distribution. + :param precision: Precision of the distribution. + :param shape: the shape of the random variable(s). + :type shape: tuple or [tuple] + :param rand_gen: the random generator (default: MXNetRandomGenerator). + :type rand_gen: RandomGenerator + :param dtype: the data type for float point numbers. + :type dtype: numpy.float32 or numpy.float64 + :param ctx: the mxnet context (default: None/current context). + :type ctx: None or mxnet.cpu or mxnet.gpu + :returns: the random variables drawn from the normal distribution. + :rtypes: Variable + """ + precision = precision if precision is not None else mx.nd.array(np.eye(N=shape[-1]), dtype=dtype, ctx=ctx) + normal = MultivariateNormalMeanPrecision(mean=mean, precision=precision, + rand_gen=rand_gen, + dtype=dtype, ctx=ctx) + normal._generate_outputs(shape=shape) + return normal.random_variable + + def _generate_outputs(self, shape): + """ + Set the output variable of the distribution. + + :param shape: the shape of the random distribution. + :type shape: tuple + """ + self.outputs = [('random_variable', Variable(value=self, shape=shape))] diff --git a/mxfusion/components/distributions/wishart.py b/mxfusion/components/distributions/wishart.py index 1af4537..548f9bb 100644 --- a/mxfusion/components/distributions/wishart.py +++ b/mxfusion/components/distributions/wishart.py @@ -169,7 +169,7 @@ def log_pdf(self, degrees_of_freedom, scale, random_variable, F=None): num_samples, num_data_points, dimension, _ = scale.shape # Note that the degrees of freedom should be a float for most of the remaining calculations - df = degrees_of_freedom.astype(random_variable.dtype) + df = degrees_of_freedom.astype(self.dtype) a = df - dimension - 1 b = df * dimension * np.log(2) diff --git a/mxfusion/util/special.py b/mxfusion/util/special.py index 56c434c..9902377 100644 --- a/mxfusion/util/special.py +++ b/mxfusion/util/special.py @@ -16,7 +16,7 @@ def log_determinant(A, F=None): """ F = get_default_MXNet_mode() if F is None else F - return 2 * F.linalg.sumlogdiag(F.linalg.potrf(A)) + return 2 * F.linalg.sumlogdiag(F.abs(F.linalg.potrf(A))) # noinspection PyPep8Naming diff --git a/testing/distributions/normal_mean_precision_test.py b/testing/distributions/normal_mean_precision_test.py new file mode 100644 index 0000000..54bc81c --- /dev/null +++ b/testing/distributions/normal_mean_precision_test.py @@ -0,0 +1,305 @@ +import pytest +import mxnet as mx +import numpy as np +from scipy.stats import norm, multivariate_normal + +from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.distributions import NormalMeanPrecision, MultivariateNormalMeanPrecision +from mxfusion.util.testutils import numpy_array_reshape +from mxfusion.util.testutils import MockMXNetRandomGenerator + + +@pytest.mark.usefixtures("set_seed") +class TestNormalPrecisionDistribution(object): + + @pytest.mark.parametrize( + "dtype, mean, mean_is_samples, precision, precision_is_samples, rv, rv_is_samples, num_samples", [ + (np.float64, np.random.rand(5, 2), True, np.random.rand(2) + 0.1, False, np.random.rand(5, 3, 2), True, 5), + (np.float64, np.random.rand(5, 2), True, np.random.rand(2) + 0.1, False, np.random.rand(3, 2), False, 5), + (np.float64, np.random.rand(2), False, np.random.rand(2) + 0.1, False, np.random.rand(3, 2), False, 5), + (np.float64, np.random.rand(5, 2), True, np.random.rand(5, 3, 2) + 0.1, True, np.random.rand(5, 3, 2), + True, 5), + (np.float32, np.random.rand(5, 2), True, np.random.rand(2) + 0.1, False, np.random.rand(5, 3, 2), True, 5), + ]) + def test_log_pdf(self, dtype, mean, mean_is_samples, precision, precision_is_samples, + rv, rv_is_samples, num_samples): + is_samples_any = any([mean_is_samples, precision_is_samples, rv_is_samples]) + rv_shape = rv.shape[1:] if rv_is_samples else rv.shape + n_dim = 1 + len(rv.shape) if is_samples_any and not rv_is_samples else len(rv.shape) + mean_np = numpy_array_reshape(mean, mean_is_samples, n_dim) + precision_np = numpy_array_reshape(precision, precision_is_samples, n_dim) + rv_np = numpy_array_reshape(rv, rv_is_samples, n_dim) + log_pdf_np = norm.logpdf(rv_np, mean_np, np.power(precision_np, -0.5)) + + var = NormalMeanPrecision.define_variable(shape=rv_shape, dtype=dtype).factor + mean_mx = mx.nd.array(mean, dtype=dtype) + if not mean_is_samples: + mean_mx = add_sample_dimension(mx.nd, mean_mx) + precision_mx = mx.nd.array(precision, dtype=dtype) + if not precision_is_samples: + precision_mx = add_sample_dimension(mx.nd, precision_mx) + rv_mx = mx.nd.array(rv, dtype=dtype) + if not rv_is_samples: + rv_mx = add_sample_dimension(mx.nd, rv_mx) + variables = {var.mean.uuid: mean_mx, var.precision.uuid: precision_mx, var.random_variable.uuid: rv_mx} + log_pdf_rt = var.log_pdf(F=mx.nd, variables=variables) + + assert np.issubdtype(log_pdf_rt.dtype, dtype) + assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + if is_samples_any: + assert get_num_samples(mx.nd, log_pdf_rt) == num_samples + if np.issubdtype(dtype, np.float64): + rtol, atol = 1e-7, 1e-10 + else: + rtol, atol = 1e-4, 1e-5 + assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy(), rtol=rtol, atol=atol) + + @pytest.mark.parametrize( + "dtype, mean, mean_is_samples, precision, precision_is_samples, rv_shape, num_samples", [ + (np.float64, np.random.rand(5, 2), True, np.random.rand(2) + 0.1, False, (3, 2), 5), + (np.float64, np.random.rand(2), False, np.random.rand(5, 2) + 0.1, True, (3, 2), 5), + (np.float64, np.random.rand(2), False, np.random.rand(2) + 0.1, False, (3, 2), 5), + (np.float64, np.random.rand(5, 2), True, np.random.rand(5, 3, 2) + 0.1, True, (3, 2), 5), + (np.float32, np.random.rand(5, 2), True, np.random.rand(2) + 0.1, False, (3, 2), 5), + ]) + def test_draw_samples(self, dtype, mean, mean_is_samples, precision, + precision_is_samples, rv_shape, num_samples): + n_dim = 1 + len(rv_shape) + mean_np = numpy_array_reshape(mean, mean_is_samples, n_dim) + precision_np = numpy_array_reshape(precision, precision_is_samples, n_dim) + + rand = np.random.randn(num_samples, *rv_shape) + rv_samples_np = mean_np + rand * np.power(precision_np, -0.5) + + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + + var = NormalMeanPrecision.define_variable(shape=rv_shape, dtype=dtype, + rand_gen=rand_gen).factor + mean_mx = mx.nd.array(mean, dtype=dtype) + if not mean_is_samples: + mean_mx = add_sample_dimension(mx.nd, mean_mx) + precision_mx = mx.nd.array(precision, dtype=dtype) + if not precision_is_samples: + precision_mx = add_sample_dimension(mx.nd, precision_mx) + variables = {var.mean.uuid: mean_mx, var.precision.uuid: precision_mx} + rv_samples_rt = var.draw_samples( + F=mx.nd, variables=variables, num_samples=num_samples) + + assert np.issubdtype(rv_samples_rt.dtype, dtype) + assert is_sampled_array(mx.nd, rv_samples_rt) + assert get_num_samples(mx.nd, rv_samples_rt) == num_samples + + if np.issubdtype(dtype, np.float64): + rtol, atol = 1e-7, 1e-10 + else: + rtol, atol = 1e-4, 1e-5 + assert np.allclose(rv_samples_np, rv_samples_rt.asnumpy(), rtol=rtol, atol=atol) + + +def make_symmetric(array): + original_shape = array.shape + d3_array = np.reshape(array, (-1,)+array.shape[-2:]) + d3_array = (d3_array[:,:,:,None]*d3_array[:,:,None,:]).sum(-3)+np.eye(2) + return np.reshape(d3_array, original_shape) + + +@pytest.mark.usefixtures("set_seed") +class TestMultivariateNormalMeanPrecisionDistribution(object): + + @pytest.mark.parametrize( + "dtype, mean, mean_is_samples, precision, precision_is_samples, rv, rv_is_samples, num_samples", [ + (np.float32, np.random.rand(2), False, make_symmetric(np.random.rand(2, 2) + 0.1), False, + np.random.rand(5, 3, 2), True, 5), + ]) + def test_log_pdf_with_broadcast(self, dtype, mean, mean_is_samples, precision, precision_is_samples, + rv, rv_is_samples, num_samples): + + mean_mx = mx.nd.array(mean, dtype=dtype) + if not mean_is_samples: + mean_mx = add_sample_dimension(mx.nd, mean_mx) + mean = mean_mx.asnumpy() + + precision_mx = mx.nd.array(precision, dtype=dtype) + if not precision_is_samples: + precision_mx = add_sample_dimension(mx.nd, precision_mx) + precision = precision_mx.asnumpy() + + rv_mx = mx.nd.array(rv, dtype=dtype) + if not rv_is_samples: + rv_mx = add_sample_dimension(mx.nd, rv_mx) + rv = rv_mx.asnumpy() + + is_samples_any = any([mean_is_samples, precision_is_samples, rv_is_samples]) + rv_shape = rv.shape[1:] + + n_dim = 1 + len(rv.shape) if is_samples_any and not rv_is_samples else len(rv.shape) + mean_np = np.broadcast_to(mean, (5, 3, 2)) + precision_np = np.broadcast_to(precision, (5, 3, 2, 2)) + rv_np = numpy_array_reshape(rv, is_samples_any, n_dim) + + rand = np.random.rand(num_samples, *rv_shape) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + + r = [] + for s in range(len(rv_np)): + a = [] + for i in range(len(rv_np[s])): + a.append(multivariate_normal.logpdf(rv_np[s][i], mean_np[s][i], np.linalg.inv(precision_np[s][i]))) + r.append(a) + log_pdf_np = np.array(r) + + normal = MultivariateNormalMeanPrecision.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + variables = { + normal.mean.uuid: mean_mx, normal.precision.uuid: precision_mx, normal.random_variable.uuid: rv_mx} + log_pdf_rt = normal.log_pdf(F=mx.nd, variables=variables) + + assert np.issubdtype(log_pdf_rt.dtype, dtype) + assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + if is_samples_any: + assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) + assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) + + @pytest.mark.parametrize( + "dtype, mean, mean_is_samples, precision, precision_is_samples, rv, rv_is_samples, num_samples", [ + (np.float64, np.random.rand(5, 3, 2), True, make_symmetric(np.random.rand(5, 3, 2, 2) + 0.1), True, + np.random.rand(5, 3, 2), True, 5), + ]) + def test_log_pdf_no_broadcast(self, dtype, mean, mean_is_samples, precision, precision_is_samples, + rv, rv_is_samples, num_samples): + + mean_mx = mx.nd.array(mean, dtype=dtype) + if not mean_is_samples: + mean_mx = add_sample_dimension(mx.nd, mean_mx) + mean = mean_mx.asnumpy() + + precision_mx = mx.nd.array(precision, dtype=dtype) + if not precision_is_samples: + precision_mx = add_sample_dimension(mx.nd, precision_mx) + precision = precision_mx.asnumpy() + + rv_mx = mx.nd.array(rv, dtype=dtype) + if not rv_is_samples: + rv_mx = add_sample_dimension(mx.nd, rv_mx) + rv = rv_mx.asnumpy() + + is_samples_any = any([mean_is_samples, precision_is_samples, rv_is_samples]) + rv_shape = rv.shape[1:] + + n_dim = 1 + len(rv.shape) if is_samples_any and not rv_is_samples else len(rv.shape) + mean_np = numpy_array_reshape(mean, is_samples_any, n_dim) + precision_np = numpy_array_reshape(precision, is_samples_any, n_dim) + rv_np = numpy_array_reshape(rv, is_samples_any, n_dim) + + rand = np.random.rand(num_samples, *rv_shape) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + + r = [] + for s in range(len(rv_np)): + a = [] + for i in range(len(rv_np[s])): + a.append(multivariate_normal.logpdf(rv_np[s][i], mean_np[s][i], np.linalg.inv(precision_np[s][i]))) + r.append(a) + log_pdf_np = np.array(r) + + normal = MultivariateNormalMeanPrecision.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + variables = {normal.mean.uuid: mean_mx, normal.precision.uuid: precision_mx, normal.random_variable.uuid: rv_mx} + log_pdf_rt = normal.log_pdf(F=mx.nd, variables=variables) + + assert np.issubdtype(log_pdf_rt.dtype, dtype) + assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + if is_samples_any: + assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) + assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) + + @pytest.mark.parametrize( + "dtype, mean, mean_is_samples, precision, precision_is_samples, rv_shape, num_samples", [ + (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2, 2) + 0.1), False, (5, 3, 2), 5), + ]) + def test_draw_samples_with_broadcast(self, dtype, mean, mean_is_samples, precision, + precision_is_samples, rv_shape, num_samples): + + mean_mx = mx.nd.array(mean, dtype=dtype) + if not mean_is_samples: + mean_mx = add_sample_dimension(mx.nd, mean_mx) + precision_mx = mx.nd.array(precision, dtype=dtype) + if not precision_is_samples: + precision_mx = add_sample_dimension(mx.nd, precision_mx) + # precision = precision_mx.asnumpy() + + is_samples_any = any([mean_is_samples, precision_is_samples]) + rand = np.random.rand(num_samples, *rv_shape) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + # rv_samples_np = mean + np.matmul(np.linalg.cholesky(precision), np.expand_dims(rand, axis=-1)).sum(-1) + + normal = MultivariateNormalMeanPrecision.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + variables = {normal.mean.uuid: mean_mx, normal.precision.uuid: precision_mx} + draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables) + + assert np.issubdtype(draw_samples_rt.dtype, dtype) + assert is_sampled_array(mx.nd, draw_samples_rt) == is_samples_any + if is_samples_any: + assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, \ + (get_num_samples(mx.nd, draw_samples_rt), num_samples) + + @pytest.mark.parametrize( + "dtype, mean, mean_is_samples, precision, precision_is_samples, rv_shape, num_samples", [ + (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2, 2) + 0.1), False, (5, 3, 2), 5), + (np.float64, np.random.rand(5, 2), True, make_symmetric(np.random.rand(2, 2) + 0.1), False, (5, 3, 2), 5), + (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(5, 2, 2) + 0.1), True, (5, 3, 2), 5), + (np.float64, np.random.rand(5, 2), True, make_symmetric(np.random.rand(5, 2, 2) + 0.1), True, (5, 3, 2), 5), + ]) + def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype, mean, mean_is_samples, precision, + precision_is_samples, rv_shape, num_samples): + + mean_mx = mx.nd.array(mean, dtype=dtype) + if not mean_is_samples: + mean_mx = add_sample_dimension(mx.nd, mean_mx) + precision_mx = mx.nd.array(precision, dtype=dtype) + if not precision_is_samples: + precision_mx = add_sample_dimension(mx.nd, precision_mx) + # precision = precision_mx.asnumpy() + + rand = np.random.rand(num_samples, *rv_shape) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + + normal = MultivariateNormalMeanPrecision.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + variables = {normal.mean.uuid: mean_mx, normal.precision.uuid: precision_mx} + draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) + + assert np.issubdtype(draw_samples_rt.dtype, dtype) + assert is_sampled_array(mx.nd, draw_samples_rt) + + @pytest.mark.parametrize( + "dtype, mean, mean_is_samples, precision, precision_is_samples, rv_shape, num_samples", [ + (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2, 2) + 0.1), False, (3, 2), 5), + (np.float64, np.random.rand(5, 3, 2), True, make_symmetric(np.random.rand(5, 3, 2, 2) + 0.1), + True, (5, 3, 2), 5), + ]) + def test_draw_samples_no_broadcast(self, dtype, mean, mean_is_samples, precision, + precision_is_samples, rv_shape, num_samples): + + mean_mx = mx.nd.array(mean, dtype=dtype) + if not mean_is_samples: + mean_mx = add_sample_dimension(mx.nd, mean_mx) + precision_mx = mx.nd.array(precision, dtype=dtype) + if not precision_is_samples: + precision_mx = add_sample_dimension(mx.nd, precision_mx) + # precision = precision_mx.asnumpy() + + # n_dim = 1 + len(rv.shape) if is_samples_any else len(rv.shape) + rand = np.random.rand(num_samples, *rv_shape) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + # rand_exp = np.expand_dims(rand, axis=-1) + # lmat = np.linalg.cholesky(precision) + # temp1 = np.matmul(lmat, rand_exp).sum(-1) + # rv_samples_np = mean + temp1 + + normal = MultivariateNormalMeanPrecision.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + + variables = {normal.mean.uuid: mean_mx, normal.precision.uuid: precision_mx} + draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) + + assert np.issubdtype(draw_samples_rt.dtype, dtype) + assert is_sampled_array(mx.nd, draw_samples_rt) + assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, \ + (get_num_samples(mx.nd, draw_samples_rt), num_samples) From 7eb39a798c55c114ccdbed451a492c2a2dc34978 Mon Sep 17 00:00:00 2001 From: Tom Diethe Date: Thu, 18 Oct 2018 00:19:33 +0100 Subject: [PATCH 02/70] Added Bernoulli distribution + tests --- mxfusion/components/distributions/__init__.py | 6 +- .../components/distributions/bernoulli.py | 181 ++++++++++++++++++ .../components/distributions/random_gen.py | 21 ++ mxfusion/util/testutils.py | 6 +- testing/distributions/bernoulli_test.py | 72 +++++++ 5 files changed, 282 insertions(+), 4 deletions(-) create mode 100644 mxfusion/components/distributions/bernoulli.py create mode 100644 testing/distributions/bernoulli_test.py diff --git a/mxfusion/components/distributions/__init__.py b/mxfusion/components/distributions/__init__.py index c69b422..e17c24a 100644 --- a/mxfusion/components/distributions/__init__.py +++ b/mxfusion/components/distributions/__init__.py @@ -6,6 +6,7 @@ .. autosummary:: :toctree: _autosummary + bernoulli categorical distribution normal @@ -15,13 +16,14 @@ gp """ -__all__ = ['categorical', 'distribution', 'normal', 'pointmass', 'rand_gen', - 'univariate','gp', 'wishart', 'beta'] +__all__ = ['bernoulli', 'categorical', 'distribution', 'normal', 'pointmass', 'random_gen', + 'univariate', 'gp', 'wishart', 'beta'] from .distribution import Distribution from .gamma import Gamma, GammaMeanVariance from .normal import Normal, MultivariateNormal from .pointmass import PointMass +from .bernoulli import Bernoulli from .categorical import Categorical from .gp import GaussianProcess, ConditionalGaussianProcess from .wishart import Wishart diff --git a/mxfusion/components/distributions/bernoulli.py b/mxfusion/components/distributions/bernoulli.py new file mode 100644 index 0000000..26926f9 --- /dev/null +++ b/mxfusion/components/distributions/bernoulli.py @@ -0,0 +1,181 @@ +from ..variables import Variable +from .univariate import UnivariateDistribution +from .distribution import LogPDFDecorator, DrawSamplesDecorator +from ...util.customop import broadcast_to_w_samples +from ..variables import get_num_samples, is_sampled_array +from ...common.config import get_default_MXNet_mode +from ...common.exceptions import InferenceError + + +class BernoulliLogPDFDecorator(LogPDFDecorator): + + def _wrap_log_pdf_with_broadcast(self, func): + def log_pdf_broadcast(self, F, **kw): + """ + Computes the logarithm of the probability density/mass function (PDF/PMF) of the distribution. + + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) + :param kw: the dict of input and output variables of the distribution + :type kw: {name: MXNet NDArray or MXNet Symbol} + :returns: log pdf of the distribution + :rtypes: MXNet NDArray or MXNet Symbol + """ + variables = {name: kw[name] for name, _ in self.inputs} + variables['random_variable'] = kw['random_variable'] + rv_shape = variables['random_variable'].shape[1:] + + n_samples = max([get_num_samples(F, v) for v in variables.values()]) + full_shape = (n_samples,) + rv_shape + + variables = { + name: broadcast_to_w_samples(F, v, full_shape[:-1]+(v.shape[-1],)) for name, v in variables.items()} + res = func(self, F=F, **variables) + return res + return log_pdf_broadcast + + +class BernoulliDrawSamplesDecorator(DrawSamplesDecorator): + + def _wrap_draw_samples_with_broadcast(self, func): + def draw_samples_broadcast(self, F, rv_shape, num_samples=1, + always_return_tuple=False, **kw): + """ + Draw a number of samples from the distribution. + + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) + :param rv_shape: the shape of each sample + :type rv_shape: tuple + :param num_samples: the number of drawn samples (default: one) + :int n_samples: int + :param always_return_tuple: Whether return a tuple even if there is only one variables in outputs. + :type always_return_tuple: boolean + :param kw: the dict of input variables of the distribution + :type kw: {name: MXNet NDArray or MXNet Symbol} + :returns: a set samples of the distribution + :rtypes: MXNet NDArray or MXNet Symbol or [MXNet NDArray or MXNet Symbol] + """ + rv_shape = list(rv_shape.values())[0] + variables = {name: kw[name] for name, _ in self.inputs} + + is_samples = any([is_sampled_array(F, v) for v in variables.values()]) + if is_samples: + num_samples_inferred = max([get_num_samples(F, v) for v in variables.values()]) + if num_samples_inferred != num_samples: + raise InferenceError("The number of samples in the n_samples argument of draw_samples of " + "Bernoulli has to be the same as the number of samples given " + "to the inputs. n_samples: {} the inferred number of samples from " + "inputs: {}.".format(num_samples, num_samples_inferred)) + full_shape = (num_samples,) + rv_shape + + variables = { + name: broadcast_to_w_samples(F, v, full_shape[:-1]+(v.shape[-1],)) for name, v in + variables.items()} + res = func(self, F=F, rv_shape=rv_shape, num_samples=num_samples, + **variables) + if always_return_tuple: + res = (res,) + return res + return draw_samples_broadcast + + +class Bernoulli(UnivariateDistribution): + """ + The Bernoulli distribution. + + :param prob_true: the probability of being true. + :type prob_true: Variable + :param rand_gen: the random generator (default: MXNetRandomGenerator). + :type rand_gen: RandomGenerator + :param dtype: the data type for float point numbers. + :type dtype: numpy.float32 or numpy.float64 + :param ctx: the mxnet context (default: None/current context). + :type ctx: None or mxnet.cpu or mxnet.gpu + """ + def __init__(self, prob_true, rand_gen=None, dtype=None, ctx=None): + if not isinstance(prob_true, Variable): + prob_true = Variable(value=prob_true) + inputs = [('prob_true', prob_true)] + input_names = ['prob_true'] + output_names = ['random_variable'] + super(Bernoulli, self).__init__( + inputs=inputs, outputs=None, + input_names=input_names, + output_names=output_names, + rand_gen=rand_gen, dtype=dtype, + ctx=ctx) + + def replicate_self(self, attribute_map=None): + """ + This functions as a copy constructor for the object. + In order to do a copy constructor we first call ``__new__`` on the class which creates a blank object. + We then initialize that object using the methods standard init procedures, and do any extra copying of attributes. + + Replicates this Factor, using new inputs, outputs, and a new uuid. + Used during model replication to functionally replicate a factor into a new graph. + + :param inputs: new input variables of the factor. + :type inputs: List of tuples of name to node e.g. [('random_variable': Variable y)] or None + :param outputs: new output variables of the factor. + :type outputs: List of tuples of name to node e.g. [('random_variable': Variable y)] or None + """ + replicant = super(Bernoulli, self).replicate_self(attribute_map=attribute_map) + return replicant + + @BernoulliLogPDFDecorator() + def log_pdf(self, prob_true, random_variable, F=None): + """ + Computes the logarithm of probabilistic mass function of the Bernoulli distribution. + + :param F: MXNet computation type . + :param prob_true: the probability of being true. + :type prob_true: MXNet NDArray or MXNet Symbol + :param random_variable: the point to compute the logpdf for. + :type random_variable: MXNet NDArray or MXNet Symbol + :returns: log pdf of the distribution. + :rtypes: MXNet NDArray or MXNet Symbol + """ + F = get_default_MXNet_mode() if F is None else F + + logL = random_variable * F.log(prob_true) + (1 - random_variable) * F.log(1 - prob_true) + logL = logL * self.log_pdf_scaling + return logL + + @BernoulliDrawSamplesDecorator() + def draw_samples(self, prob_true, rv_shape, num_samples=1, F=None): + """ + Draw a number of samples from the Bernoulli distribution. + + :param prob_true: the probability being true. + :type prob_true: MXNet NDArray or MXNet Symbol + :param rv_shape: the shape of each sample. + :type rv_shape: tuple + :param num_samples: the number of drawn samples (default: one). + :int num_samples: int + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray). + :returns: a set samples of the Bernoulli distribution + :rtypes: MXNet NDArray or MXNet Symbol + """ + F = get_default_MXNet_mode() if F is None else F + return self._rand_gen.sample_bernoulli(prob_true, shape=(num_samples,) + rv_shape, F=F) + + @staticmethod + def define_variable(prob_true, shape=None, rand_gen=None, dtype=None, ctx=None): + """ + Creates and returns a random variable drawn from a Bernoulli distribution. + + :param prob_true: the probability being true. + :type prob_true: Variable + :param shape: the shape of the Bernoulli variable. + :type shape: tuple of int + :param rand_gen: the random generator (default: MXNetRandomGenerator). + :type rand_gen: RandomGenerator + :param dtype: the data type for float point numbers. + :type dtype: numpy.float32 or numpy.float64 + :param ctx: the mxnet context (default: None/current context). + :type ctx: None or mxnet.cpu or mxnet.gpu + :returns: RandomVariable drawn from the Bernoulli distribution. + :rtypes: Variable + """ + bernoulli = Bernoulli(prob_true=prob_true, rand_gen=rand_gen, dtype=dtype, ctx=ctx) + bernoulli._generate_outputs(shape=shape) + return bernoulli.random_variable diff --git a/mxfusion/components/distributions/random_gen.py b/mxfusion/components/distributions/random_gen.py index 7989952..3300c77 100644 --- a/mxfusion/components/distributions/random_gen.py +++ b/mxfusion/components/distributions/random_gen.py @@ -16,6 +16,14 @@ def sample_normal(loc=0, scale=1, shape=None, dtype=None, out=None, ctx=None): def sample_gamma(alpha=1, beta=1, shape=None, dtype=None, out=None, ctx=None): pass + @staticmethod + def sample_multinomial(data, get_prob=True, dtype='int32', F=None): + pass + + @staticmethod + def sample_bernoulli(prob_true=0.5, dtype='bool', F=None): + pass + class MXNetRandomGenerator(RandomGenerator): """ @@ -83,6 +91,19 @@ def sample_multinomial(data, get_prob=True, dtype='int32', F=None): return F.random.multinomial( data=data, get_prob=get_prob, dtype=dtype) + @staticmethod + def sample_bernoulli(prob_true=0.5, dtype='bool', F=None): + """ + Sample Bernoulli distributed variables + + :param prob_true: Probability of being true + :param dtype: data type + :param F: MXNet node + :return: Array of samples + """ + F = get_default_MXNet_mode() if F is None else F + return F.random.normal() > 0 + @staticmethod def sample_gamma(alpha=1, beta=1, shape=None, dtype=None, out=None, ctx=None, F=None): """ diff --git a/mxfusion/util/testutils.py b/mxfusion/util/testutils.py index 008d7f3..0620ea5 100644 --- a/mxfusion/util/testutils.py +++ b/mxfusion/util/testutils.py @@ -57,10 +57,12 @@ def sample_normal(self, loc=0, scale=1, shape=None, dtype=None, out=None, out[:] = res return res - def sample_multinomial(self, data, shape=None, get_prob=True, dtype='int32', - F=None): + def sample_multinomial(self, data, shape=None, get_prob=True, dtype=None, F=None): return mx.nd.reshape(self._samples[:np.prod(data.shape[:-1])], shape=data.shape[:-1]) + def sample_bernoulli(self, prob_true=0.5, shape=None, dtype=None, out=None, F=None): + return mx.nd.reshape(self._samples[:np.prod(shape)], shape=shape) + def sample_gamma(self, alpha=1, beta=1, shape=None, dtype=None, out=None, ctx=None, F=None): if shape is None: shape = (1,) diff --git a/testing/distributions/bernoulli_test.py b/testing/distributions/bernoulli_test.py new file mode 100644 index 0000000..a99cef6 --- /dev/null +++ b/testing/distributions/bernoulli_test.py @@ -0,0 +1,72 @@ +import pytest +import mxnet as mx +import numpy as np +from scipy.stats import bernoulli + +from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.distributions import Bernoulli +from mxfusion.util.testutils import numpy_array_reshape +from mxfusion.util.testutils import MockMXNetRandomGenerator + + +@pytest.mark.usefixtures("set_seed") +class TestBernoulliDistribution(object): + + @pytest.mark.parametrize( + "dtype, prob_true, prob_true_is_samples, rv, rv_is_samples, num_samples", [ + (np.float64, np.random.beta(a=1, b=1, size=(5, 4, 3)), True, np.random.normal(size=(5, 4, 1)) > 0, True, 5), + (np.float64, np.random.beta(a=1, b=1, size=(4, 3)), False, np.random.normal(size=(4, 1)) > 0, False, 1), + (np.float64, np.random.beta(a=1, b=1, size=(5, 4, 3)), True, np.random.normal(size=(4, 1)) > 0, False, 5), + (np.float64, np.random.beta(a=1, b=1, size=(4, 3)), False, np.random.normal(size=(5, 4, 1)) > 0, True, 5), + ]) + def test_log_pdf(self, dtype, prob_true, prob_true_is_samples, rv, rv_is_samples, num_samples): + + rv_shape = rv.shape[1:] if rv_is_samples else rv.shape + n_dim = 1 + len(rv.shape) if not rv_is_samples else len(rv.shape) + prob_true_np = numpy_array_reshape(prob_true, prob_true_is_samples, n_dim) + rv_np = numpy_array_reshape(rv, rv_is_samples, n_dim) + rv_full_shape = (num_samples,)+rv_shape + rv_np = np.broadcast_to(rv_np, rv_full_shape) + + log_pdf_np = bernoulli.logpmf(k=rv_np, p=prob_true_np) + + var = Bernoulli.define_variable(0, shape=rv_shape, dtype=dtype).factor + prob_true_mx = mx.nd.array(prob_true, dtype=dtype) + if not prob_true_is_samples: + prob_true_mx = add_sample_dimension(mx.nd, prob_true_mx) + rv_mx = mx.nd.array(rv, dtype=dtype) + if not rv_is_samples: + rv_mx = add_sample_dimension(mx.nd, rv_mx) + variables = {var.prob_true.uuid: prob_true_mx, var.random_variable.uuid: rv_mx} + log_pdf_rt = var.log_pdf(F=mx.nd, variables=variables) + + assert np.issubdtype(log_pdf_rt.dtype, dtype) + assert get_num_samples(mx.nd, log_pdf_rt) == num_samples + assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) + + @pytest.mark.parametrize( + "dtype, prob_true, prob_true_is_samples, rv_shape, num_samples", [ + (np.float64, np.random.rand(5, 4, 3), True, (4, 1), 5), + (np.float64, np.random.rand(4, 3), False, (4, 1), 5), + (np.float64, np.random.rand(5, 4, 3), True, (4, 3), 5), + (np.float64, np.random.rand(4, 3), False, (4, 3), 5), + ]) + def test_draw_samples(self, dtype, prob_true, prob_true_is_samples, rv_shape, num_samples): + rv_full_shape = (num_samples,) + rv_shape + + rand_np = np.random.normal(size=rv_full_shape) > 0 + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand_np.flatten(), dtype=dtype)) + + rv_samples_np = rand_np + + var = Bernoulli.define_variable(0, shape=rv_shape, rand_gen=rand_gen, dtype=dtype).factor + prob_true_mx = mx.nd.array(prob_true, dtype=dtype) + if not prob_true_is_samples: + prob_true_mx = add_sample_dimension(mx.nd, prob_true_mx) + variables = {var.prob_true.uuid: prob_true_mx} + rv_samples_rt = var.draw_samples( + F=mx.nd, variables=variables, num_samples=num_samples) + + assert is_sampled_array(mx.nd, rv_samples_rt) + assert get_num_samples(mx.nd, rv_samples_rt) == num_samples + assert np.array_equal(rv_samples_np, rv_samples_rt.asnumpy().astype(bool)) From 25444578a24c95b32c45eaa998b14473c4c2daa4 Mon Sep 17 00:00:00 2001 From: Cliff McCollum Date: Thu, 18 Oct 2018 14:38:37 +0100 Subject: [PATCH 03/70] Corrected some text in the PPCA example notebook. --- examples/notebooks/ppca_tutorial.ipynb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/notebooks/ppca_tutorial.ipynb b/examples/notebooks/ppca_tutorial.ipynb index 46a4c60..f1bc168 100644 --- a/examples/notebooks/ppca_tutorial.ipynb +++ b/examples/notebooks/ppca_tutorial.ipynb @@ -119,7 +119,9 @@ { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "data": { @@ -150,7 +152,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We now our project our $K$ dimensional ```r``` into a high-dimensional $D$ space using a random matrix of random weights $W$. Now that ```r``` is embedded in a $D$ dimensional space the goal of PPCA will be to recover ```r``` in it's original low-dimensional $K$ space." + "We now project our $K$ dimensional ```r``` into a high-dimensional $D$ space using a random matrix of random weights $W$. Now that ```r``` is embedded in a $D$ dimensional space the goal of PPCA will be to recover ```r``` in it's original low-dimensional $K$ space." ] }, { @@ -171,7 +173,7 @@ "source": [ "# from sklearn.decomposition import PCA\n", "# pca = PCA(n_components=2)\n", - "# new_r = pca.fit_transform(r_high)\n", + "# new_r = pca.fit_transform(x_train)\n", "# plt.plot(new_r[:,0], new_r[:,1],'.')" ] }, @@ -303,7 +305,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now we define ```m.z``` which has an identity matrix covariance, ```cov```, and zero mean.\n", + "Now we define ```m.z``` which has an identity matrix covariance ```cov``` and zero mean.\n", "\n", "```m.z``` and ```sigma_2``` are then used to define ```m.x```.\n", "\n", @@ -335,7 +337,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The covariance matrix must continue to be positive definite throughout the optimization process in order to succeed in the Cholesky decomposition when drawing samples or computing the log pdf of ```q.z```. To satisfy this, we pass the covariance matrix parameters through a Gluon function that forces it into a Symmetric matrix which for suitable initialization values should maintain positive definite-ness throughout the optimization procedure. " + "The covariance matrix must continue to be positive definite throughout the optimization process in order to succeed in the Cholesky decomposition when drawing samples or computing the log pdf of ```q.z```. To satisfy this, we pass the covariance matrix parameters through a Gluon function that forces it into a Symmetric matrix for which suitable initialization values should maintain positive definite-ness throughout the optimization procedure. " ] }, { @@ -482,7 +484,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.3" } }, "nbformat": 4, From b45626c69c7150fba44574d5abd5f5ffa685f87b Mon Sep 17 00:00:00 2001 From: Mark Pullin Date: Thu, 18 Oct 2018 16:08:10 +0100 Subject: [PATCH 04/70] Add multiply kernel --- .../distributions/gp/kernels/kernel.py | 22 ++++++ .../gp/kernels/multiply_kernel.py | 72 +++++++++++++++++++ testing/distributions/gp/kernel_test.py | 22 ++++++ 3 files changed, 116 insertions(+) create mode 100644 mxfusion/components/distributions/gp/kernels/multiply_kernel.py diff --git a/mxfusion/components/distributions/gp/kernels/kernel.py b/mxfusion/components/distributions/gp/kernels/kernel.py index 708557f..5a043c2 100644 --- a/mxfusion/components/distributions/gp/kernels/kernel.py +++ b/mxfusion/components/distributions/gp/kernels/kernel.py @@ -149,6 +149,28 @@ def __add__(self, other): """ return self.add(other) + def multiply(self, other, name='mul'): + """ + Construct a new kernel by multiplying this kernel with another kernel. + + :param other: the other kernel to be added. + :type other: Kernel + :return: the kernel which is the sum of the current kernel with the specified kernel. + :rtype: Kernel + """ + if not isinstance(other, Kernel): + raise ModelSpecificationError( + "Only a Gaussian Process Kernel can be added to a Gaussian Process Kernel.") + from .multiply_kernel import MultiplyKernel + return MultiplyKernel([self, other], name=name, ctx=self.ctx, + dtype=self.dtype) + + def __mul__(self, other): + """ + Overload the "*" operator to perform multiplication of kernels + """ + return self.multiply(other) + def _compute_K(self, F, X, X2=None, **kernel_params): """ The internal interface for the actual covariance matrix computation. diff --git a/mxfusion/components/distributions/gp/kernels/multiply_kernel.py b/mxfusion/components/distributions/gp/kernels/multiply_kernel.py new file mode 100644 index 0000000..188602d --- /dev/null +++ b/mxfusion/components/distributions/gp/kernels/multiply_kernel.py @@ -0,0 +1,72 @@ +from .kernel import CombinationKernel + + +class MultiplyKernel(CombinationKernel): + """ + The multiply kernel that computes a covariance matrix by multiplying the covariance + matrices of a list of kernels. + + :param sub_kernels: a list of kernels that are combined to compute a covariance matrix. + :type sub_kernels: [Kernel] + :param name: the name of the kernel. The name is used to access kernel parameters. + :type name: str + :param dtype: the data type for float point numbers. + :type dtype: numpy.float32 or numpy.float64 + :param ctx: the mxnet context (default: None/current context). + :type ctx: None or mxnet.cpu or mxnet.gpu + """ + def __init__(self, sub_kernels, name='add', dtype=None, ctx=None): + kernels = [] + for k in sub_kernels: + if isinstance(k, CombinationKernel): + for k2 in k.sub_kernels: + kernels.append(k2) + else: + kernels.append(k) + super(MultiplyKernel, self).__init__( + sub_kernels=kernels, name=name, dtype=dtype, ctx=ctx) + + def _compute_K(self, F, X, X2=None, **kernel_params): + """ + The internal interface for the actual covariance matrix computation. + + This function takes as an assumption: The prefix in the keys of + *kernel_params* that corresponds to the name of the kernel has been + removed. The dimensions of *X* and *X2* have been sliced according to + *active_dims*. + + :param F: MXNet computation type . + :param X: the first set of inputs to the kernel. + :type X: MXNet NDArray or MXNet Symbol + :param X2: (optional) the second set of arguments to the kernel. If X2 is None, this computes a square covariance matrix of X. In other words, + X2 is internally treated as X. + :type X2: MXNet NDArray or MXNet Symbol + :param **kernel_params: the set of kernel parameters, provided as keyword arguments. + :type **kernel_params: {str: MXNet NDArray or MXNet Symbol} + :return: The covariance matrix. + :rtype: MXNet NDArray or MXNet Symbol + """ + K = self.sub_kernels[0].K(F=F, X=X, X2=X2, **kernel_params) + for k in self.sub_kernels[1:]: + K = K * k.K(F=F, X=X, X2=X2, **kernel_params) + return K + + def _compute_Kdiag(self, F, X, **kernel_params): + """ + The internal interface for the actual computation for the diagonal of the covariance matrix. + + This function takes as an assumption: The prefix in the keys of *kernel_params* that corresponds to the name of the kernel has been + removed. The dimensions of *X* has been sliced according to *active_dims*. + + :param F: MXNet computation type . + :param X: the first set of inputs to the kernel. + :type X: MXNet NDArray or MXNet Symbol + :param **kernel_params: the set of kernel parameters, provided as keyword arguments. + :type **kernel_params: {str: MXNet NDArray or MXNet Symbol} + :return: The covariance matrix. + :rtype: MXNet NDArray or MXNet Symbol + """ + K = self.sub_kernels[0].Kdiag(F=F, X=X, **kernel_params) + for k in self.sub_kernels[1:]: + K = K * k.Kdiag(F=F, X=X, **kernel_params) + return K diff --git a/testing/distributions/gp/kernel_test.py b/testing/distributions/gp/kernel_test.py index 79ae49a..5994d1e 100644 --- a/testing/distributions/gp/kernel_test.py +++ b/testing/distributions/gp/kernel_test.py @@ -262,5 +262,27 @@ def create_gpy_rbf_plus_linear(): gpy_comb_kernel_test(X, X_isSamples, X2, X2_isSamples, kernel_params, num_samples, dtype, create_rbf_plus_linear, create_gpy_rbf_plus_linear) + + @pytest.mark.parametrize("dtype, X, X_isSamples, X2, X2_isSamples, rbf_lengthscale, rbf_lengthscale_isSamples, rbf_variance, rbf_variance_isSamples, linear_variances, linear_variances_isSamples, num_samples, input_dim", + [(np.float64, np.random.rand(5, 2), False, np.random.rand(4, 2), False, np.random.rand(2) + 1e-4, False, np.random.rand(1) + 1e-4, False, np.random.rand(2) + 1e-4, False, 1, 2), + (np.float64, np.random.rand(3, 5, 2), True, np.random.rand(3, 4, 2), True, np.random.rand(2) + 1e-4, False, np.random.rand(1) + 1e-4, False, np.random.rand(2) + 1e-4, False, 3, 2), + (np.float64, np.random.rand(5, 2), False, np.random.rand(4, 2), False, np.random.rand(3, 2) + 1e-4, True, np.random.rand(1) + 1e-4, False, np.random.rand(2) + 1e-4, False, 3, 2), + (np.float64, np.random.rand(5, 2), False, np.random.rand(4, 2), False, np.random.rand(2) + 1e-4, False, np.random.rand(1) + 1e-4, False, np.random.rand(3, 2) + 1e-4, True, 3, 2), + (np.float64, np.random.rand(3, 5, 2), True, np.random.rand(3, 4, 2), True, np.random.rand(3, 2) + 1e-4, True, np.random.rand(3, 1) + 1e-4, True, np.random.rand(3, 2) + 1e-4, True, 3, 2)]) + def test_mul_kernel(self, dtype, X, X_isSamples, X2, X2_isSamples, rbf_lengthscale, rbf_lengthscale_isSamples, rbf_variance, rbf_variance_isSamples, linear_variances, linear_variances_isSamples, num_samples, input_dim): + def create_rbf_plus_linear(): + return RBF(input_dim, True, 1., 1., 'rbf', None, dtype) * Linear(input_dim, True, 1, 'linear', None, dtype) + + def create_gpy_rbf_plus_linear(): + return GPy.kern.RBF(input_dim=input_dim, ARD=True) * GPy.kern.Linear(input_dim=input_dim, ARD=True) + + kernel_params = {'rbf': {'lengthscale': (rbf_lengthscale, rbf_lengthscale_isSamples), 'variance': (rbf_variance, rbf_variance_isSamples)}, + 'linear': {'variances': (linear_variances, linear_variances_isSamples)} + } + + gpy_comb_kernel_test(X, X_isSamples, X2, X2_isSamples, kernel_params, + num_samples, dtype, create_rbf_plus_linear, + create_gpy_rbf_plus_linear) + except ImportError: pass From da95fe04dcd72e793103f5f420a17d87a6bf51c5 Mon Sep 17 00:00:00 2001 From: Tom Diethe Date: Thu, 18 Oct 2018 22:16:12 +0100 Subject: [PATCH 05/70] Fix to Bernoulli sampling --- mxfusion/components/distributions/random_gen.py | 5 +++-- testing/distributions/bernoulli_test.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mxfusion/components/distributions/random_gen.py b/mxfusion/components/distributions/random_gen.py index 3300c77..734647f 100644 --- a/mxfusion/components/distributions/random_gen.py +++ b/mxfusion/components/distributions/random_gen.py @@ -92,17 +92,18 @@ def sample_multinomial(data, get_prob=True, dtype='int32', F=None): data=data, get_prob=get_prob, dtype=dtype) @staticmethod - def sample_bernoulli(prob_true=0.5, dtype='bool', F=None): + def sample_bernoulli(prob_true=0.5, dtype='bool', shape=None, F=None): """ Sample Bernoulli distributed variables + :param shape: Array shape of samples :param prob_true: Probability of being true :param dtype: data type :param F: MXNet node :return: Array of samples """ F = get_default_MXNet_mode() if F is None else F - return F.random.normal() > 0 + return F.random.uniform(low=0, high=1, shape=shape) > prob_true @staticmethod def sample_gamma(alpha=1, beta=1, shape=None, dtype=None, out=None, ctx=None, F=None): diff --git a/testing/distributions/bernoulli_test.py b/testing/distributions/bernoulli_test.py index a99cef6..84ff4f1 100644 --- a/testing/distributions/bernoulli_test.py +++ b/testing/distributions/bernoulli_test.py @@ -69,4 +69,5 @@ def test_draw_samples(self, dtype, prob_true, prob_true_is_samples, rv_shape, nu assert is_sampled_array(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples + # TODO: should we have to cast to bool? or can mxnet handle bool natively? assert np.array_equal(rv_samples_np, rv_samples_rt.asnumpy().astype(bool)) From e79204192be95d5ca0286093342c3083053a3e35 Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 08:58:16 +0100 Subject: [PATCH 06/70] Implement correct log pdf --- .../components/distributions/dirichlet.py | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 mxfusion/components/distributions/dirichlet.py diff --git a/mxfusion/components/distributions/dirichlet.py b/mxfusion/components/distributions/dirichlet.py new file mode 100644 index 0000000..4aa1c0a --- /dev/null +++ b/mxfusion/components/distributions/dirichlet.py @@ -0,0 +1,166 @@ +from ..variables import Variable +from ...common.config import get_default_MXNet_mode +from .distribution import Distribution, LogPDFDecorator, DrawSamplesDecorator +from ..variables import is_sampled_array, get_num_samples +from ...util.customop import broadcast_to_w_samples +from ...common.exceptions import InferenceError + + +class DirichletLogPDFDecorator(LogPDFDecorator): + + def _wrap_log_pdf_with_broadcast(self, func): + def log_pdf_broadcast(self, F, **kw): + """ + Computes the logarithm of the probability density/mass function (PDF/PMF) of the distribution. The inputs and outputs variables are in RTVariable format. + + Shape assumptions: + * a is S x N x D + * random_variable is S x N x D + + Where: + * S, the number of samples, is optional. If more than one of the variables has samples, the number of samples in each variable must be the same. S is 1 by default if not a sampled variable. + * N is the number of data points. N can be any number of dimensions (N_1, N_2, ...) but must be broadcastable to the shape of random_variable. + * D is the dimension of the distribution. + + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) + :param kw: the dict of input and output variables of the distribution + :type kw: {str (name): MXNet NDArray or MXNet Symbol} + :returns: log pdf of the distribution + :rtypes: MXNet NDArray or MXNet Symbol + """ + variables = {name: kw[name] for name, _ in self.inputs} + variables['random_variable'] = kw['random_variable'] + rv_shape = variables['random_variable'].shape[1:] + + nSamples = max([get_num_samples(F, v) for v in variables.values()]) + + shapes_map = {} + shapes_map['a'] = (nSamples,) + rv_shape + shapes_map['random_variable'] = (nSamples,) + rv_shape + variables = {name: broadcast_to_w_samples(F, v, shapes_map[name]) + for name, v in variables.items()} + res = func(self, F=F, **variables) + return res + return log_pdf_broadcast + + +class DirichletDrawSamplesDecorator(DrawSamplesDecorator): + + def _wrap_draw_samples_with_broadcast(self, func): + def draw_samples_broadcast(self, F, rv_shape, num_samples=1, + always_return_tuple=False, **kw): + """ + Draw a number of samples from the distribution. The inputs and outputs variables are in RTVariable format. + + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) + :param rv_shape: the shape of each sample + :type rv_shape: tuple + :param num_samples: the number of drawn samples (default: one) + :int num_samples: int + :param always_return_tuple: Whether return a tuple even if there is only one variables in outputs. + :type always_return_tuple: boolean + :param kw: the dict of input variables of the distribution + :type kw: {name: MXNet NDArray or MXNet Symbol} + :returns: a set samples of the distribution + :rtypes: MXNet NDArray or MXNet Symbol or [MXNet NDArray or MXNet Symbol] + """ + rv_shape = list(rv_shape.values())[0] + variables = {name: kw[name] for name, _ in self.inputs} + + isSamples = any([is_sampled_array(F, v) for v in variables.values()]) + if isSamples: + num_samples_inferred = max([get_num_samples(F, v) for v in + variables.values()]) + if num_samples_inferred != num_samples: + raise InferenceError("The number of samples in the nSamples argument of draw_samples of Normal distribution must be the same as the number of samples given to the inputs. nSamples: "+str(num_samples)+" the inferred number of samples from inputs: "+str(num_samples_inferred)+".") + + shapes_map = {} + shapes_map['a'] = (num_samples,) + rv_shape + shapes_map['random_variable'] = (num_samples,) + rv_shape + variables = {name: broadcast_to_w_samples(F, v, shapes_map[name]) + for name, v in variables.items()} + + res = func(self, F=F, rv_shape=rv_shape, num_samples=num_samples, + **variables) + if always_return_tuple: + res = (res,) + return res + return draw_samples_broadcast + + +class Dirichlet(Distribution): + def __init__(self, a, normalization=True, + rand_gen=None, dtype=None, ctx=None): + # self.minibatch_ratio = minibatch_ratio + if not isinstance(a, Variable): + a = Variable(value=a) + + inputs = [('a', a)] + input_names = ['a'] + output_names = ['random_variable'] + super().__init__(inputs=inputs, outputs=None, input_names=input_names, + output_names=output_names, rand_gen=rand_gen, + dtype=dtype, ctx=ctx) + self.normalization = normalization + + @DirichletLogPDFDecorator() + def log_pdf(self, a, random_variable, F=None): + F = get_default_MXNet_mode() if F is None else F + + if self.normalization: + random_variable = F.broadcast_div(random_variable, F.expand_dims(F.norm(random_variable, ord=1, axis=2), + axis=2)) + power = F.broadcast_power(random_variable, a - 1) + prod = F.prod(power, axis=2) + beta = F.prod(F.gamma(a), axis=2)/F.gamma(F.sum(a, axis=2)) + logL = F.log(prod/beta) + return logL + + @DirichletDrawSamplesDecorator() + def draw_samples(self, a, rv_shape, num_samples=1, F=None): + F = get_default_MXNet_mode() if F is None else F + out_shape = (num_samples,) + rv_shape + (1,) + + ones = F.ones_like(a) + + y = self._rand_gen.sample_gamma(alpha=a, beta=ones, + shape=out_shape, + dtype=self.dtype, ctx=self.ctx) + x = F.broadcast_div(y, F.broadcast_sum(y)) + return x + + @staticmethod + def define_variable(a, shape=None, normalization=True, + rand_gen=None, dtype=None, ctx=None): + dirichlet = Dirichlet(a=a, normalization=normalization, + rand_gen=rand_gen, dtype=dtype, ctx=ctx) + dirichlet._generate_outputs(shape=shape) + return dirichlet.random_variable + + def _generate_outputs(self, shape): + """ + Set the output variable of the distribution. + + :param shape: the shape of the random distribution. + :type shape: tuple + """ + self.outputs = [('random_variable', Variable(value=self, shape=shape))] + + def replicate_self(self, attribute_map=None): + """ + This functions as a copy constructor for the object. + In order to do a copy constructor we first call ``__new__`` on the class which creates a blank object. + We then initialize that object using the methods standard init procedures, and do any extra copying of + attributes. + + Replicates this Factor, using new inputs, outputs, and a new uuid. + Used during model replication to functionally replicate a factor into a new graph. + + :param inputs: new input variables of the factor. + :type inputs: List of tuples of name to node e.g. [('random_variable': Variable y)] or None + :param outputs: new output variables of the factor. + :type outputs: List of tuples of name to node e.g. [('random_variable': Variable y)] or None + """ + replicant = super().replicate_self(attribute_map=attribute_map) + replicant.normalization = self.normalization + return replicant From 3b369939aa9ebb1143581734b387ba9db84331b7 Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 09:29:05 +0100 Subject: [PATCH 07/70] Implement test_log_pdf_with_broadcast --- testing/distributions/dirichlet_test.py | 207 ++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 testing/distributions/dirichlet_test.py diff --git a/testing/distributions/dirichlet_test.py b/testing/distributions/dirichlet_test.py new file mode 100644 index 0000000..7f9261b --- /dev/null +++ b/testing/distributions/dirichlet_test.py @@ -0,0 +1,207 @@ +import pytest +import numpy as np +import mxnet as mx +from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.distributions import Dirichlet +from mxfusion.util.testutils import numpy_array_reshape +from mxfusion.util.testutils import MockMXNetRandomGenerator + + +def make_symmetric(array): + original_shape = array.shape + d3_array = np.reshape(array, (-1,)+array.shape[-2:]) + d3_array = (d3_array[:,:,:,None]*d3_array[:,:,None,:]).sum(-3)+np.eye(2) + return np.reshape(d3_array, original_shape) + + +@pytest.mark.usefixtures("set_seed") +class TestDirichletDistribution(object): + # dtype must be float64 to ensure that sum of x_i is always 1 + @pytest.mark.parametrize("dtype, a, a_is_samples, rv, rv_is_samples, num_samples", [ + (np.float64, np.random.rand(2), False, np.random.rand(5, 3, 2), True, 5), + (np.float64, np.random.rand(2), False, np.random.rand(10, 3, 2), True, 10), + (np.float64, np.random.rand(2), False, np.random.rand(3, 2), False, 5) + ]) + def test_log_pdf_with_broadcast(self, dtype, a, a_is_samples, rv, rv_is_samples, num_samples): + # Add sample dimension if varaible is not samples + a_mx = mx.nd.array(a, dtype=dtype) + if not a_is_samples: + a_mx = add_sample_dimension(mx.nd, a_mx) + a = a_mx.asnumpy() + print('a.shape', a.shape) + + rv_mx = mx.nd.array(rv, dtype=dtype) + if not rv_is_samples: + rv_mx = add_sample_dimension(mx.nd, rv_mx) + rv = rv_mx.asnumpy() + print('rv.shape', rv.shape) + + is_samples_any = a_is_samples or rv_is_samples + rv_shape = rv.shape[1:] + + n_dim = 1 + len(rv.shape) if is_samples_any and not rv_is_samples else len(rv.shape) + a_np = np.broadcast_to(a, (num_samples, 3, 2)) + rv_np = numpy_array_reshape(rv, is_samples_any, n_dim) + + # Initialize rand_gen + rand = np.random.rand(num_samples, *rv_shape) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + + # Calculate correct Dirichlet logpdf + from scipy.stats import dirichlet as scipy_dirichlet + r = [] + for s in range(len(rv_np)): + a = [] + for i in range(len(rv_np[s])): + print('rv_np[s][i].shape', rv_np[s][i].shape) + print('a_np[s][i].shape', a_np[s][i].shape) + a.append(scipy_dirichlet.logpdf(rv_np[s][i]/sum(rv_np[s][i]), a_np[s][i])) # NORMALISING + r.append(a) + log_pdf_np = np.array(r) + + dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + variables = {dirichlet.a.uuid: a_mx, dirichlet.random_variable.uuid: rv_mx} + log_pdf_rt = dirichlet.log_pdf(F=mx.nd, variables=variables) + + assert np.issubdtype(log_pdf_rt.dtype, dtype) + assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + if is_samples_any: + print('log_pdf_rt.shape', log_pdf_rt.shape) + print('num_samples', num_samples) + assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) + assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) + + # @pytest.mark.parametrize("dtype, a, a_is_samples, var, var_isSamples, rv, rv_is_samples, num_samples", [ + # (np.float64, np.random.rand(5,3,2), True, np.random.rand(5,3,2), True, 5), + # ]) + # def test_log_pdf_no_broadcast(self, dtype, a, a_is_samples, + # rv, rv_is_samples, num_samples): + # + # a_mx = mx.nd.array(a, dtype=dtype) + # if not a_is_samples: + # a_mx = add_sample_dimension(mx.nd, a_mx) + # a = a_mx.asnumpy() + # + # rv_mx = mx.nd.array(rv, dtype=dtype) + # if not rv_is_samples: + # rv_mx = add_sample_dimension(mx.nd, rv_mx) + # rv = rv_mx.asnumpy() + # + # from scipy.stats import dirichlet as scipy_dirichlet + # is_samples_any = any([a_is_samples, rv_is_samples]) + # rv_shape = rv.shape[1:] + # + # n_dim = 1 + len(rv.shape) if is_samples_any and not rv_is_samples else len(rv.shape) + # a_np = numpy_array_reshape(a, is_samples_any, n_dim) + # rv_np = numpy_array_reshape(rv, is_samples_any, n_dim) + # + # rand = np.random.rand(num_samples, *rv_shape) + # rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + # + # r = [] + # for s in range(len(rv_np)): + # a = [] + # for i in range(len(rv_np[s])): + # a.append(scipy_dirichlet.logpdf(rv_np[s][i], a_np[s][i], var_np[s][i])) + # r.append(a) + # log_pdf_np = np.array(r) + # + # dirichlet = Dirichlet.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + # variables = {dirichlet.a.uuid: a_mx, dirichlet.random_variable.uuid: rv_mx} + # log_pdf_rt = dirichlet.log_pdf(F=mx.nd, variables=variables) + # + # assert np.issubdtype(log_pdf_rt.dtype, dtype) + # assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + # if is_samples_any: + # assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) + # assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) + # + # @pytest.mark.parametrize( + # "dtype, a, a_is_samples, var, var_isSamples, rv_shape, num_samples",[ + # (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2,2)+0.1), False, (5,3,2), 5), + # ]) + # def test_draw_samples_with_broadcast(self, dtype, a, a_is_samples, var, + # var_isSamples, rv_shape, num_samples): + # + # a_mx = mx.nd.array(a, dtype=dtype) + # if not a_is_samples: + # a_mx = add_sample_dimension(mx.nd, a_mx) + # var_mx = mx.nd.array(var, dtype=dtype) + # if not var_isSamples: + # var_mx = add_sample_dimension(mx.nd, var_mx) + # var = var_mx.asnumpy() + # + # is_samples_any = any([a_is_samples, var_isSamples]) + # rand = np.random.rand(num_samples, *rv_shape) + # rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + # rv_samples_np = a + np.matmul(np.linalg.cholesky(var), np.expand_dims(rand, axis=-1)).sum(-1) + # + # normal = Dirichlet.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + # variables = {normal.a.uuid: a_mx, normal.covariance.uuid: var_mx} + # draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables) + # + # assert np.issubdtype(draw_samples_rt.dtype, dtype) + # assert is_sampled_array(mx.nd, draw_samples_rt) == is_samples_any + # if is_samples_any: + # assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, (get_num_samples(mx.nd, draw_samples_rt), num_samples) + # + # @pytest.mark.parametrize( + # "dtype, a, a_is_samples, var, var_isSamples, rv_shape, num_samples",[ + # (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2,2)+0.1), False, (5,3,2), 5), + # (np.float64, np.random.rand(5,2), True, make_symmetric(np.random.rand(2,2)+0.1), False, (5,3,2), 5), + # (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(5,2,2)+0.1), True, (5,3,2), 5), + # (np.float64, np.random.rand(5,2), True, make_symmetric(np.random.rand(5,2,2)+0.1), True, (5,3,2), 5), + # ]) + # def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype, a, a_is_samples, var, + # var_isSamples, rv_shape, num_samples): + # + # a_mx = mx.nd.array(a, dtype=dtype) + # if not a_is_samples: + # a_mx = add_sample_dimension(mx.nd, a_mx) + # var_mx = mx.nd.array(var, dtype=dtype) + # if not var_isSamples: + # var_mx = add_sample_dimension(mx.nd, var_mx) + # var = var_mx.asnumpy() + # + # rand = np.random.rand(num_samples, *rv_shape) + # rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + # + # normal = Dirichlet.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + # variables = {normal.a.uuid: a_mx, normal.covariance.uuid: var_mx} + # draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) + # + # assert np.issubdtype(draw_samples_rt.dtype, dtype) + # assert is_sampled_array(mx.nd, draw_samples_rt) == True + # + # @pytest.mark.parametrize( + # "dtype, a, a_is_samples, var, var_isSamples, rv_shape, num_samples",[ + # (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2,2)+0.1), False, (3,2), 5), + # (np.float64, np.random.rand(5,3,2), True, make_symmetric(np.random.rand(5,3,2,2)+0.1), True, (5,3,2), 5), + # ]) + # def test_draw_samples_no_broadcast(self, dtype, a, a_is_samples, var, + # var_isSamples, rv_shape, num_samples): + # + # a_mx = mx.nd.array(a, dtype=dtype) + # if not a_is_samples: + # a_mx = add_sample_dimension(mx.nd, a_mx) + # var_mx = mx.nd.array(var, dtype=dtype) + # if not var_isSamples: + # var_mx = add_sample_dimension(mx.nd, var_mx) + # var = var_mx.asnumpy() + # + # # n_dim = 1 + len(rv.shape) if is_samples_any else len(rv.shape) + # rand = np.random.rand(num_samples, *rv_shape) + # rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + # rand_exp = np.expand_dims(rand, axis=-1) + # lmat = np.linalg.cholesky(var) + # temp1 = np.matmul(lmat, rand_exp).sum(-1) + # rv_samples_np = a + temp1 + # + # normal = Dirichlet.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + # + # variables = {normal.a.uuid: a_mx, normal.covariance.uuid: var_mx} + # draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) + # + # assert np.issubdtype(draw_samples_rt.dtype, dtype) + # assert is_sampled_array(mx.nd, draw_samples_rt) == True + # assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, (get_num_samples(mx.nd, draw_samples_rt), num_samples) From 604e9954aa784f1a91e987fc7e1c3503bcc39464 Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 09:29:43 +0100 Subject: [PATCH 08/70] Add Dirichlet --- mxfusion/components/distributions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mxfusion/components/distributions/__init__.py b/mxfusion/components/distributions/__init__.py index c69b422..4fd2eca 100644 --- a/mxfusion/components/distributions/__init__.py +++ b/mxfusion/components/distributions/__init__.py @@ -26,3 +26,4 @@ from .gp import GaussianProcess, ConditionalGaussianProcess from .wishart import Wishart from .beta import Beta +from .dirichlet import Dirichlet From 306a1edf07669a3a2de2454675bb7655fa23dcaa Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 10:06:41 +0100 Subject: [PATCH 09/70] Write dirichlet tests --- .../components/distributions/dirichlet.py | 16 +- testing/distributions/dirichlet_test.py | 263 ++++++++---------- 2 files changed, 125 insertions(+), 154 deletions(-) diff --git a/mxfusion/components/distributions/dirichlet.py b/mxfusion/components/distributions/dirichlet.py index 4aa1c0a..b30af4d 100644 --- a/mxfusion/components/distributions/dirichlet.py +++ b/mxfusion/components/distributions/dirichlet.py @@ -11,15 +11,18 @@ class DirichletLogPDFDecorator(LogPDFDecorator): def _wrap_log_pdf_with_broadcast(self, func): def log_pdf_broadcast(self, F, **kw): """ - Computes the logarithm of the probability density/mass function (PDF/PMF) of the distribution. The inputs and outputs variables are in RTVariable format. + Computes the logarithm of the probability density/mass function (PDF/PMF) of the distribution. The inputs + and outputs variables are in RTVariable format. Shape assumptions: * a is S x N x D * random_variable is S x N x D Where: - * S, the number of samples, is optional. If more than one of the variables has samples, the number of samples in each variable must be the same. S is 1 by default if not a sampled variable. - * N is the number of data points. N can be any number of dimensions (N_1, N_2, ...) but must be broadcastable to the shape of random_variable. + * S, the number of samples, is optional. If more than one of the variables has samples, the number of + samples in each variable must be the same. S is 1 by default if not a sampled variable. + * N is the number of data points. N can be any number of dimensions (N_1, N_2, ...) but must be + broadcastable to the shape of random_variable. * D is the dimension of the distribution. :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) @@ -72,7 +75,10 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, num_samples_inferred = max([get_num_samples(F, v) for v in variables.values()]) if num_samples_inferred != num_samples: - raise InferenceError("The number of samples in the nSamples argument of draw_samples of Normal distribution must be the same as the number of samples given to the inputs. nSamples: "+str(num_samples)+" the inferred number of samples from inputs: "+str(num_samples_inferred)+".") + raise InferenceError("The number of samples in the nSamples argument of draw_samples of Dirichlet", + "distribution must be the same as the number of samples given to the inputs. ", + "nSamples: "+str(num_samples)+" the inferred number of samples from inputs: " + + str(num_samples_inferred)+".") shapes_map = {} shapes_map['a'] = (num_samples,) + rv_shape @@ -126,7 +132,7 @@ def draw_samples(self, a, rv_shape, num_samples=1, F=None): y = self._rand_gen.sample_gamma(alpha=a, beta=ones, shape=out_shape, dtype=self.dtype, ctx=self.ctx) - x = F.broadcast_div(y, F.broadcast_sum(y)) + x = F.broadcast_div(y, F.sum(y)) return x @staticmethod diff --git a/testing/distributions/dirichlet_test.py b/testing/distributions/dirichlet_test.py index 7f9261b..125efbe 100644 --- a/testing/distributions/dirichlet_test.py +++ b/testing/distributions/dirichlet_test.py @@ -1,19 +1,15 @@ import pytest + import numpy as np import mxnet as mx +from scipy.stats import dirichlet as scipy_dirichlet + from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples from mxfusion.components.distributions import Dirichlet from mxfusion.util.testutils import numpy_array_reshape from mxfusion.util.testutils import MockMXNetRandomGenerator -def make_symmetric(array): - original_shape = array.shape - d3_array = np.reshape(array, (-1,)+array.shape[-2:]) - d3_array = (d3_array[:,:,:,None]*d3_array[:,:,None,:]).sum(-3)+np.eye(2) - return np.reshape(d3_array, original_shape) - - @pytest.mark.usefixtures("set_seed") class TestDirichletDistribution(object): # dtype must be float64 to ensure that sum of x_i is always 1 @@ -28,13 +24,11 @@ def test_log_pdf_with_broadcast(self, dtype, a, a_is_samples, rv, rv_is_samples, if not a_is_samples: a_mx = add_sample_dimension(mx.nd, a_mx) a = a_mx.asnumpy() - print('a.shape', a.shape) rv_mx = mx.nd.array(rv, dtype=dtype) if not rv_is_samples: rv_mx = add_sample_dimension(mx.nd, rv_mx) rv = rv_mx.asnumpy() - print('rv.shape', rv.shape) is_samples_any = a_is_samples or rv_is_samples rv_shape = rv.shape[1:] @@ -48,14 +42,56 @@ def test_log_pdf_with_broadcast(self, dtype, a, a_is_samples, rv, rv_is_samples, rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) # Calculate correct Dirichlet logpdf - from scipy.stats import dirichlet as scipy_dirichlet r = [] for s in range(len(rv_np)): a = [] for i in range(len(rv_np[s])): - print('rv_np[s][i].shape', rv_np[s][i].shape) - print('a_np[s][i].shape', a_np[s][i].shape) - a.append(scipy_dirichlet.logpdf(rv_np[s][i]/sum(rv_np[s][i]), a_np[s][i])) # NORMALISING + a.append(scipy_dirichlet.logpdf(rv_np[s][i]/sum(rv_np[s][i]), a_np[s][i])) + r.append(a) + log_pdf_np = np.array(r) + + dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + variables = {dirichlet.a.uuid: a_mx, dirichlet.random_variable.uuid: rv_mx} + log_pdf_rt = dirichlet.log_pdf(F=mx.nd, variables=variables) + + assert np.issubdtype(log_pdf_rt.dtype, dtype) + assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + if is_samples_any: + assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) + assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) + + @pytest.mark.parametrize("dtype, a, a_is_samples, rv, rv_is_samples, num_samples", [ + (np.float64, np.random.rand(5, 3, 2), True, np.random.rand(5, 3, 2), True, 5), + (np.float64, np.random.rand(10, 3, 2), True, np.random.rand(10, 3, 2), True, 10), + ]) + def test_log_pdf_no_broadcast(self, dtype, a, a_is_samples, + rv, rv_is_samples, num_samples): + + a_mx = mx.nd.array(a, dtype=dtype) + if not a_is_samples: + a_mx = add_sample_dimension(mx.nd, a_mx) + a = a_mx.asnumpy() + + rv_mx = mx.nd.array(rv, dtype=dtype) + if not rv_is_samples: + rv_mx = add_sample_dimension(mx.nd, rv_mx) + rv = rv_mx.asnumpy() + + is_samples_any = any([a_is_samples, rv_is_samples]) + rv_shape = rv.shape[1:] + + n_dim = 1 + len(rv.shape) if is_samples_any and not rv_is_samples else len(rv.shape) + a_np = numpy_array_reshape(a, is_samples_any, n_dim) + rv_np = numpy_array_reshape(rv, is_samples_any, n_dim) + + rand = np.random.rand(num_samples, *rv_shape) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + + r = [] + for s in range(len(rv_np)): + a = [] + for i in range(len(rv_np[s])): + a.append(scipy_dirichlet.logpdf(rv_np[s][i]/sum(rv_np[s][i]), a_np[s][i])) r.append(a) log_pdf_np = np.array(r) @@ -66,142 +102,71 @@ def test_log_pdf_with_broadcast(self, dtype, a, a_is_samples, rv, rv_is_samples, assert np.issubdtype(log_pdf_rt.dtype, dtype) assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any if is_samples_any: - print('log_pdf_rt.shape', log_pdf_rt.shape) - print('num_samples', num_samples) assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) - # @pytest.mark.parametrize("dtype, a, a_is_samples, var, var_isSamples, rv, rv_is_samples, num_samples", [ - # (np.float64, np.random.rand(5,3,2), True, np.random.rand(5,3,2), True, 5), - # ]) - # def test_log_pdf_no_broadcast(self, dtype, a, a_is_samples, - # rv, rv_is_samples, num_samples): - # - # a_mx = mx.nd.array(a, dtype=dtype) - # if not a_is_samples: - # a_mx = add_sample_dimension(mx.nd, a_mx) - # a = a_mx.asnumpy() - # - # rv_mx = mx.nd.array(rv, dtype=dtype) - # if not rv_is_samples: - # rv_mx = add_sample_dimension(mx.nd, rv_mx) - # rv = rv_mx.asnumpy() - # - # from scipy.stats import dirichlet as scipy_dirichlet - # is_samples_any = any([a_is_samples, rv_is_samples]) - # rv_shape = rv.shape[1:] - # - # n_dim = 1 + len(rv.shape) if is_samples_any and not rv_is_samples else len(rv.shape) - # a_np = numpy_array_reshape(a, is_samples_any, n_dim) - # rv_np = numpy_array_reshape(rv, is_samples_any, n_dim) - # - # rand = np.random.rand(num_samples, *rv_shape) - # rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) - # - # r = [] - # for s in range(len(rv_np)): - # a = [] - # for i in range(len(rv_np[s])): - # a.append(scipy_dirichlet.logpdf(rv_np[s][i], a_np[s][i], var_np[s][i])) - # r.append(a) - # log_pdf_np = np.array(r) - # - # dirichlet = Dirichlet.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor - # variables = {dirichlet.a.uuid: a_mx, dirichlet.random_variable.uuid: rv_mx} - # log_pdf_rt = dirichlet.log_pdf(F=mx.nd, variables=variables) - # - # assert np.issubdtype(log_pdf_rt.dtype, dtype) - # assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any - # if is_samples_any: - # assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) - # assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) - # - # @pytest.mark.parametrize( - # "dtype, a, a_is_samples, var, var_isSamples, rv_shape, num_samples",[ - # (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2,2)+0.1), False, (5,3,2), 5), - # ]) - # def test_draw_samples_with_broadcast(self, dtype, a, a_is_samples, var, - # var_isSamples, rv_shape, num_samples): - # - # a_mx = mx.nd.array(a, dtype=dtype) - # if not a_is_samples: - # a_mx = add_sample_dimension(mx.nd, a_mx) - # var_mx = mx.nd.array(var, dtype=dtype) - # if not var_isSamples: - # var_mx = add_sample_dimension(mx.nd, var_mx) - # var = var_mx.asnumpy() - # - # is_samples_any = any([a_is_samples, var_isSamples]) - # rand = np.random.rand(num_samples, *rv_shape) - # rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) - # rv_samples_np = a + np.matmul(np.linalg.cholesky(var), np.expand_dims(rand, axis=-1)).sum(-1) - # - # normal = Dirichlet.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor - # variables = {normal.a.uuid: a_mx, normal.covariance.uuid: var_mx} - # draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables) - # - # assert np.issubdtype(draw_samples_rt.dtype, dtype) - # assert is_sampled_array(mx.nd, draw_samples_rt) == is_samples_any - # if is_samples_any: - # assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, (get_num_samples(mx.nd, draw_samples_rt), num_samples) - # - # @pytest.mark.parametrize( - # "dtype, a, a_is_samples, var, var_isSamples, rv_shape, num_samples",[ - # (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2,2)+0.1), False, (5,3,2), 5), - # (np.float64, np.random.rand(5,2), True, make_symmetric(np.random.rand(2,2)+0.1), False, (5,3,2), 5), - # (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(5,2,2)+0.1), True, (5,3,2), 5), - # (np.float64, np.random.rand(5,2), True, make_symmetric(np.random.rand(5,2,2)+0.1), True, (5,3,2), 5), - # ]) - # def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype, a, a_is_samples, var, - # var_isSamples, rv_shape, num_samples): - # - # a_mx = mx.nd.array(a, dtype=dtype) - # if not a_is_samples: - # a_mx = add_sample_dimension(mx.nd, a_mx) - # var_mx = mx.nd.array(var, dtype=dtype) - # if not var_isSamples: - # var_mx = add_sample_dimension(mx.nd, var_mx) - # var = var_mx.asnumpy() - # - # rand = np.random.rand(num_samples, *rv_shape) - # rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) - # - # normal = Dirichlet.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor - # variables = {normal.a.uuid: a_mx, normal.covariance.uuid: var_mx} - # draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) - # - # assert np.issubdtype(draw_samples_rt.dtype, dtype) - # assert is_sampled_array(mx.nd, draw_samples_rt) == True - # - # @pytest.mark.parametrize( - # "dtype, a, a_is_samples, var, var_isSamples, rv_shape, num_samples",[ - # (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2,2)+0.1), False, (3,2), 5), - # (np.float64, np.random.rand(5,3,2), True, make_symmetric(np.random.rand(5,3,2,2)+0.1), True, (5,3,2), 5), - # ]) - # def test_draw_samples_no_broadcast(self, dtype, a, a_is_samples, var, - # var_isSamples, rv_shape, num_samples): - # - # a_mx = mx.nd.array(a, dtype=dtype) - # if not a_is_samples: - # a_mx = add_sample_dimension(mx.nd, a_mx) - # var_mx = mx.nd.array(var, dtype=dtype) - # if not var_isSamples: - # var_mx = add_sample_dimension(mx.nd, var_mx) - # var = var_mx.asnumpy() - # - # # n_dim = 1 + len(rv.shape) if is_samples_any else len(rv.shape) - # rand = np.random.rand(num_samples, *rv_shape) - # rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) - # rand_exp = np.expand_dims(rand, axis=-1) - # lmat = np.linalg.cholesky(var) - # temp1 = np.matmul(lmat, rand_exp).sum(-1) - # rv_samples_np = a + temp1 - # - # normal = Dirichlet.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor - # - # variables = {normal.a.uuid: a_mx, normal.covariance.uuid: var_mx} - # draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) - # - # assert np.issubdtype(draw_samples_rt.dtype, dtype) - # assert is_sampled_array(mx.nd, draw_samples_rt) == True - # assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, (get_num_samples(mx.nd, draw_samples_rt), num_samples) + @pytest.mark.parametrize("dtype, a, a_is_samples, rv_shape, num_samples", [ + (np.float64, np.random.rand(2), False, (5, 3, 2), 5), + ]) + def test_draw_samples_with_broadcast(self, dtype, a, a_is_samples, rv_shape, num_samples): + + a_mx = mx.nd.array(a, dtype=dtype) + if not a_is_samples: + a_mx = add_sample_dimension(mx.nd, a_mx) + + is_samples_any = a_is_samples + rand = np.random.rand(num_samples, *rv_shape) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + + dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + variables = {dirichlet.a.uuid: a_mx} + draw_samples_rt = dirichlet.draw_samples(F=mx.nd, variables=variables) + + assert np.issubdtype(draw_samples_rt.dtype, dtype) + assert is_sampled_array(mx.nd, draw_samples_rt) == is_samples_any + if is_samples_any: + assert get_num_samples(mx.nd, draw_samples_rt) == num_samples,\ + (get_num_samples(mx.nd, draw_samples_rt), num_samples) + + @pytest.mark.parametrize("dtype, a, a_is_samples, rv_shape, num_samples", [ + (np.float64, np.random.rand(2), False, (5, 3, 2), 5), + (np.float64, np.random.rand(5, 2), True, (5, 3, 2), 5), + (np.float64, np.random.rand(2), False, (5, 3, 2), 5), + (np.float64, np.random.rand(5, 2), True, (5, 3, 2), 5), + ]) + def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype, a, a_is_samples, rv_shape, num_samples): + a_mx = mx.nd.array(a, dtype=dtype) + if not a_is_samples: + a_mx = add_sample_dimension(mx.nd, a_mx) + + rand = np.random.rand(num_samples, *rv_shape) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + + dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + variables = {dirichlet.a.uuid: a_mx} + draw_samples_rt = dirichlet.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) + + assert np.issubdtype(draw_samples_rt.dtype, dtype) + assert is_sampled_array(mx.nd, draw_samples_rt) is True + + @pytest.mark.parametrize("dtype, a, a_is_samples, rv_shape, num_samples", [ + (np.float64, np.random.rand(2), False, (3, 2), 5), + (np.float64, np.random.rand(5, 3, 2), True, (5, 3, 2), 5), + ]) + def test_draw_samples_no_broadcast(self, dtype, a, a_is_samples, rv_shape, num_samples): + a_mx = mx.nd.array(a, dtype=dtype) + if not a_is_samples: + a_mx = add_sample_dimension(mx.nd, a_mx) + + # n_dim = 1 + len(rv.shape) if is_samples_any else len(rv.shape) + rand = np.random.rand(num_samples, *rv_shape) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) + + dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + variables = {dirichlet.a.uuid: a_mx} + draw_samples_rt = dirichlet.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) + + assert np.issubdtype(draw_samples_rt.dtype, dtype) + assert is_sampled_array(mx.nd, draw_samples_rt) is True + assert get_num_samples(mx.nd, draw_samples_rt) == num_samples,\ + (get_num_samples(mx.nd, draw_samples_rt), num_samples) From 74f7b0c46e94c4b7b67692ba36542fa0bb249fad Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 10:20:13 +0100 Subject: [PATCH 10/70] Add docs --- .../components/distributions/dirichlet.py | 56 ++++++++++++++++--- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/mxfusion/components/distributions/dirichlet.py b/mxfusion/components/distributions/dirichlet.py index b30af4d..fe23729 100644 --- a/mxfusion/components/distributions/dirichlet.py +++ b/mxfusion/components/distributions/dirichlet.py @@ -56,12 +56,9 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, Draw a number of samples from the distribution. The inputs and outputs variables are in RTVariable format. :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) - :param rv_shape: the shape of each sample - :type rv_shape: tuple - :param num_samples: the number of drawn samples (default: one) - :int num_samples: int - :param always_return_tuple: Whether return a tuple even if there is only one variables in outputs. - :type always_return_tuple: boolean + :param tuple rv_shape: the shape of each sample + :param int num_samples: the number of drawn samples (default: one) + :param boolean always_return_tuple: Whether return a tuple even if there is only one variables in outputs. :param kw: the dict of input variables of the distribution :type kw: {name: MXNet NDArray or MXNet Symbol} :returns: a set samples of the distribution @@ -95,9 +92,19 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, class Dirichlet(Distribution): + """ + The Dirichlet distribution. + + :param Variable a: alpha, the concentration parameters of the distribution. + :param boolean normalization: If true, L1 normalization is applied. + :param RandomGenerator rand_gen: the random generator (default: MXNetRandomGenerator). + :param dtype: the data type for float point numbers. + :type dtype: numpy.float32 or numpy.float64 + :param ctx: the mxnet context (default: None/current context). + :type ctx: None or mxnet.cpu or mxnet.gpu + """ def __init__(self, a, normalization=True, rand_gen=None, dtype=None, ctx=None): - # self.minibatch_ratio = minibatch_ratio if not isinstance(a, Variable): a = Variable(value=a) @@ -111,6 +118,17 @@ def __init__(self, a, normalization=True, @DirichletLogPDFDecorator() def log_pdf(self, a, random_variable, F=None): + """ + Computes the logarithm of the probability density function (pdf) of the Dirichlet distribution. + + :param a: the a parameter (alpha) of the Dirichlet distribution. + :type a: MXNet NDArray or MXNet Symbol + :param random_variable: the random variable of the Dirichlet distribution. + :type random_variable: MXNet NDArray or MXNet Symbol + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray). + :returns: log pdf of the distribution. + :rtypes: MXNet NDArray or MXNet Symbol + """ F = get_default_MXNet_mode() if F is None else F if self.normalization: @@ -124,6 +142,17 @@ def log_pdf(self, a, random_variable, F=None): @DirichletDrawSamplesDecorator() def draw_samples(self, a, rv_shape, num_samples=1, F=None): + """ + Draw samples from the Dirichlet distribution. + + :param a: the a parameter (alpha) of the Dirichlet distribution. + :type a: MXNet NDArray or MXNet Symbol + :param tuple rv_shape: the shape of each sample. + :param int num_samples: the number of drawn samples (default: one). + :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray). + :returns: a set samples of the Dirichlet distribution. + :rtypes: MXNet NDArray or MXNet Symbol + """ F = get_default_MXNet_mode() if F is None else F out_shape = (num_samples,) + rv_shape + (1,) @@ -138,6 +167,19 @@ def draw_samples(self, a, rv_shape, num_samples=1, F=None): @staticmethod def define_variable(a, shape=None, normalization=True, rand_gen=None, dtype=None, ctx=None): + """ + Creates and returns a random variable drawn from a Dirichlet distribution. + + :param Variable a: alpha, the concentration parameters of the distribution. + :param boolean normalization: If true, L1 normalization is applied. + :param RandomGenerator rand_gen: the random generator (default: MXNetRandomGenerator). + :param dtype: the data type for float point numbers. + :type dtype: numpy.float32 or numpy.float64 + :param ctx: the mxnet context (default: None/current context). + :type ctx: None or mxnet.cpu or mxnet.gpu + :returns: the random variables drawn from the Dirichlet distribution. + :rtypes: Variable + """ dirichlet = Dirichlet(a=a, normalization=normalization, rand_gen=rand_gen, dtype=dtype, ctx=ctx) dirichlet._generate_outputs(shape=shape) From 6b65c6c3b0448b9ef97eb2fd5c99cd1b494a375c Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 10:20:40 +0100 Subject: [PATCH 11/70] Add dirichlet, beta, wishart to docs --- mxfusion/components/distributions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mxfusion/components/distributions/__init__.py b/mxfusion/components/distributions/__init__.py index 4fd2eca..90cc2d0 100644 --- a/mxfusion/components/distributions/__init__.py +++ b/mxfusion/components/distributions/__init__.py @@ -13,6 +13,9 @@ random_gen univariate gp + wishart + beta + dirichlet """ __all__ = ['categorical', 'distribution', 'normal', 'pointmass', 'rand_gen', From f58ae1ccaf6244d2aef54f55d3167526524851fa Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Fri, 19 Oct 2018 10:42:36 +0100 Subject: [PATCH 12/70] Update error message Co-Authored-By: marpulli --- mxfusion/components/distributions/gp/kernels/kernel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mxfusion/components/distributions/gp/kernels/kernel.py b/mxfusion/components/distributions/gp/kernels/kernel.py index 5a043c2..6d89c15 100644 --- a/mxfusion/components/distributions/gp/kernels/kernel.py +++ b/mxfusion/components/distributions/gp/kernels/kernel.py @@ -160,7 +160,7 @@ def multiply(self, other, name='mul'): """ if not isinstance(other, Kernel): raise ModelSpecificationError( - "Only a Gaussian Process Kernel can be added to a Gaussian Process Kernel.") + "Only a Gaussian Process Kernel can be multiplied with a Gaussian Process Kernel.") from .multiply_kernel import MultiplyKernel return MultiplyKernel([self, other], name=name, ctx=self.ctx, dtype=self.dtype) From 472150255d2a835ffedb9a504e380d74fedd7188 Mon Sep 17 00:00:00 2001 From: Mark Pullin Date: Fri, 19 Oct 2018 11:37:45 +0100 Subject: [PATCH 13/70] Fix divide by zero error if max_iter < n_prints --- mxfusion/inference/batch_loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mxfusion/inference/batch_loop.py b/mxfusion/inference/batch_loop.py index 611fe74..8cea773 100644 --- a/mxfusion/inference/batch_loop.py +++ b/mxfusion/inference/batch_loop.py @@ -31,7 +31,7 @@ def run(self, infr_executor, data, param_dict, ctx, optimizer='adam', optimizer=optimizer, optimizer_params={'learning_rate': learning_rate}) - iter_step = max_iter // n_prints + iter_step = max(max_iter // n_prints, 1) for i in range(max_iter): with mx.autograd.record(): loss, loss_for_gradient = infr_executor(mx.nd.zeros(1, ctx=ctx), *data) From e8cc8e6d37906c9b0b5410edeb336f6ff3fee330 Mon Sep 17 00:00:00 2001 From: Tom Diethe Date: Thu, 18 Oct 2018 21:29:05 +0100 Subject: [PATCH 14/70] Typos and tidying --- .../components/distributions/distribution.py | 2 +- .../components/distributions/pointmass.py | 2 +- .../components/distributions/univariate.py | 2 +- mxfusion/inference/grad_based_inference.py | 8 +- mxfusion/inference/inference.py | 96 ++++++++++--------- mxfusion/inference/inference_parameters.py | 28 ++++-- mxfusion/modules/module.py | 6 +- 7 files changed, 83 insertions(+), 61 deletions(-) diff --git a/mxfusion/components/distributions/distribution.py b/mxfusion/components/distributions/distribution.py index 70d94fe..85aea69 100644 --- a/mxfusion/components/distributions/distribution.py +++ b/mxfusion/components/distributions/distribution.py @@ -18,7 +18,7 @@ def _wrap_log_pdf_with_variables(self, func): def log_pdf_variables(self, F, variables, targets=None): """ - Computes the logrithm of the probability density/mass function + Computes the logarithm of the probability density/mass function (PDF/PMF) of the distribution. The inputs and outputs variables are fetched from the *variables* argument according to their UUIDs. diff --git a/mxfusion/components/distributions/pointmass.py b/mxfusion/components/distributions/pointmass.py index bf6062d..90359d0 100644 --- a/mxfusion/components/distributions/pointmass.py +++ b/mxfusion/components/distributions/pointmass.py @@ -21,7 +21,7 @@ def __init__(self, location, rand_gen=None, dtype=None, ctx=None): @UnivariateLogPDFDecorator() def log_pdf(self, location, random_variable, F=None): """ - Computes the logaorithm of probabilistic density function of the normal distribution. + Computes the logarithm of probabilistic density function of the normal distribution. :param F: MXNet computation type . :param location: the location of the point mass. diff --git a/mxfusion/components/distributions/univariate.py b/mxfusion/components/distributions/univariate.py index 95aa2ea..644dee9 100644 --- a/mxfusion/components/distributions/univariate.py +++ b/mxfusion/components/distributions/univariate.py @@ -10,7 +10,7 @@ class UnivariateLogPDFDecorator(LogPDFDecorator): def _wrap_log_pdf_with_broadcast(self, func): def log_pdf_broadcast(self, F, **kws): """ - Computes the logrithm of the probability density/mass function + Computes the logarithm of the probability density/mass function (PDF/PMF) of the distribution. :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) diff --git a/mxfusion/inference/grad_based_inference.py b/mxfusion/inference/grad_based_inference.py index d8883f8..95c4a2d 100644 --- a/mxfusion/inference/grad_based_inference.py +++ b/mxfusion/inference/grad_based_inference.py @@ -43,7 +43,7 @@ def create_executor(self): rv_scaling = self._grad_loop.rv_scaling else: rv_scaling = None - infr = self._infr_alg.create_executor( + infr = self._inference_algorithm.create_executor( data_def=self.observed_variable_UUIDs, params=self.params, var_ties=self.params.var_ties, rv_scaling=rv_scaling) if self._hybridize: @@ -52,7 +52,7 @@ def create_executor(self): return infr def run(self, optimizer='adam', learning_rate=1e-3, max_iter=2000, - verbose=False, **kw): + verbose=False, **kwargs): """ Run the inference method. @@ -67,8 +67,8 @@ def run(self, optimizer='adam', learning_rate=1e-3, max_iter=2000, :param **kwargs: The keyword arguments specify the data for inferences. The key of each argument is the name of the corresponding variable in model definition and the value of the argument is the data in numpy array format. """ - data = [kw[v] for v in self.observed_variable_names] - self.initialize(**kw) + data = [kwargs[v] for v in self.observed_variable_names] + self.initialize(**kwargs) infr = self.create_executor() return self._grad_loop.run( diff --git a/mxfusion/inference/inference.py b/mxfusion/inference/inference.py index cf8adbb..2f95e58 100644 --- a/mxfusion/inference/inference.py +++ b/mxfusion/inference/inference.py @@ -11,20 +11,17 @@ class Inference(object): """ Abstract class defining an inference method that can be applied to a model. - An inference method consists of a few components: the applied inference algorithm, the model definition (optionally a definition of posterior + An inference method consists of a few components: the applied inference algorithm, + the model definition (optionally a definition of posterior approximation), the inference parameters. :param inference_algorithm: The applied inference algorithm :type inference_algorithm: InferenceAlgorithm - :param graphs: a list of graph definitions required by the inference method. It includes the model definition and necessary posterior approximation. - :type graphs: [FactorGraph] - :param observed: A list of observed variables - :type observed: [Variable] :param constants: Specify a list of model variables as constants :type constants: {Variable: mxnet.ndarray} :param hybridize: Whether to hybridize the MXNet Gluon block of the inference method. :type hybridize: boolean - :param dtype: data type for internal numberical representation + :param dtype: data type for internal numerical representation :type dtype: {numpy.float64, numpy.float32, 'float64', 'float32'} :param context: The MXNet context :type context: {mxnet.cpu or mxnet.gpu} @@ -34,11 +31,10 @@ def __init__(self, inference_algorithm, constants=None, hybridize=False, dtype=None, context=None): self.dtype = dtype if dtype is not None else np.float32 - self.mxnet_context = context if context is not None else \ - get_default_device() + self.mxnet_context = context if context is not None else get_default_device() self._hybridize = hybridize self._graphs = inference_algorithm.graphs - self._infr_alg = inference_algorithm + self._inference_algorithm = inference_algorithm self.params = InferenceParameters(constants=constants, dtype=self.dtype, context=self.mxnet_context) @@ -46,15 +42,15 @@ def __init__(self, inference_algorithm, constants=None, @property def observed_variables(self): - return self._infr_alg.observed_variables + return self._inference_algorithm.observed_variables @property def observed_variable_UUIDs(self): - return self._infr_alg.observed_variable_UUIDs + return self._inference_algorithm.observed_variable_UUIDs @property def observed_variable_names(self): - return self._infr_alg.observed_variable_names + return self._inference_algorithm.observed_variable_names @property def graphs(self): @@ -68,15 +64,15 @@ def inference_algorithm(self): """ Return the reference to the used inference algorithm. """ - return self._infr_alg + return self._inference_algorithm def create_executor(self): """ Return a MXNet Gluon block responsible for the execution of the inference method. """ - infr = self._infr_alg.create_executor(data_def=self.observed_variable_UUIDs, - params=self.params, - var_ties=self.params.var_ties) + infr = self._inference_algorithm.create_executor(data_def=self.observed_variable_UUIDs, + params=self.params, + var_ties=self.params.var_ties) if self._hybridize: infr.hybridize() infr.initialize(ctx=self.mxnet_context) @@ -87,15 +83,18 @@ def _initialize_params(self): def initialize(self, **kw): """ - Initialize the inference method with the shapes of observed variables. The inputs of the keyword arguments are the names of the - variables in the model defintion. The values of the keyword arguments are the data of the corresponding variables (mxnet.ndarray) + Initialize the inference method with the shapes of observed variables. + The inputs of the keyword arguments are the names of the + variables in the model definition. The values of the keyword arguments are the data of the + corresponding variables (mxnet.ndarray) or their shape (tuples). """ if not self._initialized: data = [kw[v] for v in self.observed_variable_names] if len(data) > 0: if not all(isinstance(d, type(d)) for d in data): - raise InferenceError("All items in the keywords must be of the same type. Either all shapes or all data objects.") + raise InferenceError("All items in the keywords must be of the same type. " + "Either all shapes or all data objects.") if isinstance(data[0], (tuple, list)): data_shapes = {i: d for i, d in zip(self.observed_variable_UUIDs, data)} @@ -103,7 +102,8 @@ def initialize(self, **kw): data_shapes = {i: d.shape for i, d in zip(self.observed_variable_UUIDs, data)} else: - raise InferenceError("Keywords not of type mx.nd.NDArray or tuple/list for shapes passed into initialization.") + raise InferenceError("Keywords not of type mx.nd.NDArray or tuple/list " + "for shapes passed into initialization.") shape_constants = discover_shape_constants(data_shapes, self._graphs) self.params.update_constants(shape_constants) @@ -113,21 +113,22 @@ def initialize(self, **kw): warnings.warn("Trying to initialize the inference twice, skipping.") # TODO: how to handle when initialization called twice - def run(self, **kw): + def run(self, **kwargs): """ Run the inference method. - :param **kwargs: The keyword arguments specify the data for inferenceself. The key of each argument is the name of the corresponding + :param kwargs: The keyword arguments specify the data for inference. The key of each argument + is the name of the corresponding variable in model definition and the value of the argument is the data in numpy array format. :returns: the samples of target variables (if not spcified, the samples of all the latent variables) :rtype: {UUID: samples} """ - data = [kw[v] for v in self.observed_variable_names] - self.initialize(**kw) - infr = self.create_executor() - return infr(mx.nd.zeros(1, ctx=self.mxnet_context), *data) + data = [kwargs[v] for v in self.observed_variable_names] + self.initialize(**kwargs) + executor = self.create_executor() + return executor(mx.nd.zeros(1, ctx=self.mxnet_context), *data) - def set_intializer(self): + def set_initializer(self): """ Configure the inference method on how to initialize variables and parameters. """ @@ -141,21 +142,29 @@ def load(self, mxnet_constants_file=None, variable_constants_file=None): """ - Loads back everything needed to rerun an inference algorithm. The major pieces of this are the InferenceParameters, FactorGraphs, and + Loads back everything needed to rerun an inference algorithm. + The major pieces of this are the InferenceParameters, FactorGraphs, and InferenceConfiguration. :param primary_model_file: The file containing the primary model to load back for this inference algorithm. :type primary_model_file: str of filename - :param secondary_graph_files: The files containing any secondary graphs (e.g. a posterior) to load back for this inference algorithm. + :param secondary_graph_files: The files containing any secondary graphs (e.g. a posterior) to load back + for this inference algorithm. :type secondary_graph_files: [str of filename] - :param inference_configuration_file: The file containing any inference specific configuration needed to reload this inference algorithm. + :param inference_configuration_file: The file containing any inference specific configuration needed to + reload this inference algorithm. e.g. observation patterns used to train it. :type inference_configuration_file: str of filename - :param parameters_file: These are the parameters of the previous inference algorithm. These are in a {uuid: mx.nd.array} mapping. - :type mxnet_constants_file: file saved down with mx.nd.save(), so a {uuid: mx.nd.array} mapping saved in a binary format. - :param mxnet_constants_file: These are the constants in mxnet format from the previous inference algorithm. These are in a {uuid: mx.nd.array} mapping. - :type mxnet_constants_file: file saved down with mx.nd.save(), so a {uuid: mx.nd.array} mapping saved in a binary format. - :param variable_constants_file: These are the constants in primitive format from the previous inference algorithm. + :param parameters_file: These are the parameters of the previous inference algorithm. + These are in a {uuid: mx.nd.array} mapping. + :type mxnet_constants_file: file saved down with mx.nd.save(), so a {uuid: mx.nd.array} mapping saved + in a binary format. + :param mxnet_constants_file: These are the constants in mxnet format from the previous inference algorithm. + These are in a {uuid: mx.nd.array} mapping. + :type mxnet_constants_file: file saved down with mx.nd.save(), so a {uuid: mx.nd.array} mapping saved + in a binary format. + :param variable_constants_file: These are the constants in primitive format from the previous + inference algorithm. :type variable_constants_file: json dict of {uuid: constant_primitive} """ primary_model = FactorGraph('graph_0').load_graph(primary_model_file) @@ -179,7 +188,8 @@ def load(self, def load_configuration(self, config_file, uuid_map): """ - Loads relevant inference configuration back from a file. Currently only loads the observed variables UUIDs back in, using the uuid_map + Loads relevant inference configuration back from a file. + Currently only loads the observed variables UUIDs back in, using the uuid_map parameter to store the correct current observed variables. :param config_file: The file to save the configuration down into. @@ -194,7 +204,8 @@ def load_configuration(self, config_file, uuid_map): def save_configuration(self, config_file): """ - Saves relevant inference configuration down into a file. Currently only saves the observed variables UUIDs as {'observed': [observed_uuids]}. + Saves relevant inference configuration down into a file. + Currently only saves the observed variables UUIDs as {'observed': [observed_uuids]}. :param config_file: The file to save the configuration down into. :type config_file: str @@ -205,7 +216,8 @@ def save_configuration(self, config_file): def save(self, prefix=None): """ - Saves down everything needed to reload an inference algorithm. The two primary pieces of this are the InferenceParameters and FactorGraphs. + Saves down everything needed to reload an inference algorithm. + The two primary pieces of this are the InferenceParameters and FactorGraphs. :param prefix: The directory and any appending tag for the files to save this Inference as. :type prefix: str , ex. "../saved_inferences/experiment_1" @@ -220,15 +232,11 @@ def save(self, prefix=None): class TransferInference(Inference): """ - The abstract Inference method for transfering the outcome of one inference + The abstract Inference method for transferring the outcome of one inference method to another. :param inference_algorithm: The applied inference algorithm :type inference_algorithm: InferenceAlgorithm - :param graphs: a list of graph definitions required by the inference method. It includes the model definition and necessary posterior approximation. - :type graphs: [FactorGraph] - :param observed: A list of observed variables - :type observed: [Variable] :param constants: Specify a list of model variables as constants :type constants: {Variable: mxnet.ndarray} :param hybridize: Whether to hybridize the MXNet Gluon block of the inference method. @@ -254,7 +262,7 @@ def generate_executor(self, **kw): data_shapes) self._initialized = True - infr = self._infr_alg.create_executor( + infr = self._inference_algorithm.create_executor( data_def=self.observed_variable_UUIDs, params=self.params, var_ties=self.params.var_ties) if self._hybridize: diff --git a/mxfusion/inference/inference_parameters.py b/mxfusion/inference/inference_parameters.py index 8deb380..d946911 100644 --- a/mxfusion/inference/inference_parameters.py +++ b/mxfusion/inference/inference_parameters.py @@ -15,7 +15,8 @@ class InferenceParameters(object): """ The parameters and outcomes of an inference method. - InferenceParameters is a pool of memory that contains a mapping from uuid to two types of memories (MXNet ParameterDict and Constants). + InferenceParameters is a pool of memory that contains a mapping from uuid to two types of memories + (MXNet ParameterDict and Constants). :param constants: Specify a list of model variables as constants :type constants: {ModelComponent.uuid : mxnet.ndarray} @@ -72,7 +73,8 @@ def initialize_params(self, graphs, observed_uuid): for var in g.get_parameters(excluded=excluded, include_inherited=False): var_shape = realize_shape(var.shape, self._constants) - init = initializer.Constant(var.initial_value_before_transformation) if var.initial_value is not None else None + init = initializer.Constant(var.initial_value_before_transformation) \ + if var.initial_value is not None else None self._params.get(name=var.uuid, shape=var_shape, dtype=self.dtype, @@ -89,7 +91,8 @@ def initialize_with_carryover_params(self, graphs, observed_uuid, var_ties, :type graphs: a list of FactorGraph :param observed_uuid: Parameter Variables that are passed in directly as data, not to be inferred. :type observed_uuid: {UUID : mx.ndarray} - :param var_ties: A dictionary of variable maps that are tied together and use the MXNet Parameter of the dict value's uuid. + :param var_ties: A dictionary of variable maps that are tied together and use the MXNet Parameter of the dict + value's uuid. :type var_ties: { UUID to tie from : UUID to tie to } :param carryover_params: list of InferenceParameters containing the outcomes of previous inference algorithms. :type carryover_params: [InferenceParameters] @@ -173,11 +176,16 @@ def load_parameters(uuid_map=None, current_params=None): """ Loads back a sest of InferenceParameters from files. - :param parameters_file: These are the parameters of the previous inference algorithm. These are in a {uuid: mx.nd.array} mapping. - :type mxnet_constants_file: file saved down with mx.nd.save(), so a {uuid: mx.nd.array} mapping saved in a binary format. - :param mxnet_constants_file: These are the constants in mxnet format from the previous inference algorithm. These are in a {uuid: mx.nd.array} mapping. - :type mxnet_constants_file: file saved down with mx.nd.save(), so a {uuid: mx.nd.array} mapping saved in a binary format. - :param variable_constants_file: These are the constants in primitive format from the previous inference algorithm. + :param parameters_file: These are the parameters of the previous inference algorithm. + These are in a {uuid: mx.nd.array} mapping. + :type mxnet_constants_file: file saved down with mx.nd.save(), so a {uuid: mx.nd.array} mapping saved + in a binary format. + :param mxnet_constants_file: These are the constants in mxnet format from the previous inference algorithm. + These are in a {uuid: mx.nd.array} mapping. + :type mxnet_constants_file: file saved down with mx.nd.save(), so a {uuid: mx.nd.array} mapping saved + in a binary format. + :param variable_constants_file: These are the constants in primitive format from the previous + inference algorithm. :type variable_constants_file: json dict of {uuid: constant_primitive} """ def with_uuid_map(item, uuid_map): @@ -218,7 +226,9 @@ def with_uuid_map(item, uuid_map): def save(self, prefix): """ - Saves the parameters and constants down to json files as maps from {uuid : value}, where value is an mx.ndarray for parameters and either primitive number types or mx.ndarray for constants. Saves up to 3 files: prefix+["_params.json", "_variable_constants.json", "_mxnet_constants.json"] + Saves the parameters and constants down to json files as maps from {uuid : value}, + where value is an mx.ndarray for parameters and either primitive number types or mx.ndarray for constants. + Saves up to 3 files: prefix+["_params.json", "_variable_constants.json", "_mxnet_constants.json"] :param prefix: The directory and any appending tag for the files to save this Inference as. :type prefix: str , ex. "../saved_inferences/experiment_1" diff --git a/mxfusion/modules/module.py b/mxfusion/modules/module.py index f87db79..1d5d674 100644 --- a/mxfusion/modules/module.py +++ b/mxfusion/modules/module.py @@ -13,7 +13,11 @@ class Module(Factor): """ The base class for a probabilistic module. - A probabilistic module is a combination of model denfition and Inference algorithms. It acts as a factor and are defined as such during model definition, producing random variables like a plain probabilistic distribution. It differs from a plain distribution in that to compute it's log_pdf and draw_samples functions, it uses a full Inference method. + A probabilistic module is a combination of model definition and Inference algorithms. + It acts as a factor and are defined as such during model definition, + producing random variables like a plain probabilistic distribution. + It differs from a plain distribution in that to compute it's log_pdf + and draw_samples functions, it uses a full Inference method. :param inputs: the input variables :type inputs: List of tuples of name to node e.g. [('random_variable': Variable y)] or None From 58619f79d2a970bee421256cd473a47756197cbb Mon Sep 17 00:00:00 2001 From: Tom Diethe Date: Fri, 19 Oct 2018 13:54:02 +0100 Subject: [PATCH 15/70] Fixed some typos --- mxfusion/components/distributions/categorical.py | 2 +- mxfusion/components/distributions/gp/cond_gp.py | 4 ++-- mxfusion/components/distributions/gp/gp.py | 2 +- mxfusion/inference/forward_sampling.py | 8 ++++---- mxfusion/inference/grad_based_inference.py | 2 +- mxfusion/inference/inference.py | 2 +- mxfusion/inference/inference_parameters.py | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mxfusion/components/distributions/categorical.py b/mxfusion/components/distributions/categorical.py index c63c30e..eb8dfe4 100644 --- a/mxfusion/components/distributions/categorical.py +++ b/mxfusion/components/distributions/categorical.py @@ -12,7 +12,7 @@ class CategoricalLogPDFDecorator(LogPDFDecorator): def _wrap_log_pdf_with_broadcast(self, func): def log_pdf_broadcast(self, F, **kw): """ - Computes the logrithm of the probability density/mass function (PDF/PMF) of the distribution. + Computes the logarithm of the probability density/mass function (PDF/PMF) of the distribution. :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) :param kw: the dict of input and output variables of the distribution diff --git a/mxfusion/components/distributions/gp/cond_gp.py b/mxfusion/components/distributions/gp/cond_gp.py index 87570c5..1bd05be 100644 --- a/mxfusion/components/distributions/gp/cond_gp.py +++ b/mxfusion/components/distributions/gp/cond_gp.py @@ -12,7 +12,7 @@ class ConditionalGaussianProcessLogPDFDecorator(LogPDFDecorator): def _wrap_log_pdf_with_broadcast(self, func): def log_pdf_broadcast(self, F, **kw): """ - Computes the logrithm of the probability density/mass function (PDF/PMF) of the distribution. + Computes the logarithm of the probability density/mass function (PDF/PMF) of the distribution. :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) :param kw: the dict of input and output variables of the distribution @@ -175,7 +175,7 @@ def define_variable(X, X_cond, Y_cond, kernel, shape=None, mean_func=None, def log_pdf(self, X, X_cond, Y_cond, random_variable, F=None, **kernel_params): """ - Computes the logrithm of the probability density function (PDF) of the condtional Gaussian process. + Computes the logarithm of the probability density function (PDF) of the condtional Gaussian process. .. math:: \\log p(Y| X_c, Y_c, X) = \\log \\mathcal{N}(Y| K_{*c}K_{cc}^{-1}(Y_C - g(X_c)) + g(X), K_{**} - K_{*c}K_{cc}^{-1}K_{*c}^\\top) diff --git a/mxfusion/components/distributions/gp/gp.py b/mxfusion/components/distributions/gp/gp.py index dd1a50e..834a61d 100644 --- a/mxfusion/components/distributions/gp/gp.py +++ b/mxfusion/components/distributions/gp/gp.py @@ -12,7 +12,7 @@ class GaussianProcessLogPDFDecorator(LogPDFDecorator): def _wrap_log_pdf_with_broadcast(self, func): def log_pdf_broadcast(self, F, **kw): """ - Computes the logrithm of the probability density/mass function (PDF/PMF) of the distribution. + Computes the logarithm of the probability density/mass function (PDF/PMF) of the distribution. :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray) :param kw: the dict of input and output variables of the distribution diff --git a/mxfusion/inference/forward_sampling.py b/mxfusion/inference/forward_sampling.py index 565cc3b..197bfb0 100644 --- a/mxfusion/inference/forward_sampling.py +++ b/mxfusion/inference/forward_sampling.py @@ -51,8 +51,8 @@ class ForwardSampling(TransferInference): :type model: Model :param observed: A list of observed variables :type observed: [Variable] - :param var_ties: A dictionary of variables that are tied together and use the MXNet Parameter of the dict value's uuid. - :type var_ties: { UUID to tie from : UUID to tie to } + :param var_tie: A dictionary of variables that are tied together and use the MXNet Parameter of the dict value's uuid. + :type var_tie: { UUID to tie from : UUID to tie to } :param infr_params: list or single of InferenceParameters objects from previous Inference runs. :type infr_params: InferenceParameters or [InferenceParameters] :param target_variables: (optional) the target variables to sample @@ -61,7 +61,7 @@ class ForwardSampling(TransferInference): :type hybridize: boolean :param constants: Specify a list of model variables as constants :type constants: {Variable: mxnet.ndarray} - :param dtype: data type for internal numberical representation + :param dtype: data type for internal numerical representation :type dtype: {numpy.float64, numpy.float32, 'float64', 'float32'} :param context: The MXNet context :type context: {mxnet.cpu or mxnet.gpu} @@ -116,7 +116,7 @@ class VariationalPosteriorForwardSampling(ForwardSampling): :type constants: {Variable: mxnet.ndarray} :param hybridize: Whether to hybridize the MXNet Gluon block of the inference method. :type hybridize: boolean - :param dtype: data type for internal numberical representation + :param dtype: data type for internal numerical representation :type dtype: {numpy.float64, numpy.float32, 'float64', 'float32'} :param context: The MXNet context :type context: {mxnet.cpu or mxnet.gpu} diff --git a/mxfusion/inference/grad_based_inference.py b/mxfusion/inference/grad_based_inference.py index 95c4a2d..39821b6 100644 --- a/mxfusion/inference/grad_based_inference.py +++ b/mxfusion/inference/grad_based_inference.py @@ -20,7 +20,7 @@ class GradBasedInference(Inference): :type constants: {Variable: mxnet.ndarray} :param hybridize: Whether to hybridize the MXNet Gluon block of the inference method. :type hybridize: boolean - :param dtype: data type for internal numberical representation + :param dtype: data type for internal numerical representation :type dtype: {numpy.float64, numpy.float32, 'float64', 'float32'} :param context: The MXNet context :type context: {mxnet.cpu or mxnet.gpu} diff --git a/mxfusion/inference/inference.py b/mxfusion/inference/inference.py index 2f95e58..79217f7 100644 --- a/mxfusion/inference/inference.py +++ b/mxfusion/inference/inference.py @@ -241,7 +241,7 @@ class TransferInference(Inference): :type constants: {Variable: mxnet.ndarray} :param hybridize: Whether to hybridize the MXNet Gluon block of the inference method. :type hybridize: boolean - :param dtype: data type for internal numberical representation + :param dtype: data type for internal numerical representation :type dtype: {numpy.float64, numpy.float32, 'float64', 'float32'} :param context: The MXNet context :type context: {mxnet.cpu or mxnet.gpu} diff --git a/mxfusion/inference/inference_parameters.py b/mxfusion/inference/inference_parameters.py index d946911..a790d2b 100644 --- a/mxfusion/inference/inference_parameters.py +++ b/mxfusion/inference/inference_parameters.py @@ -20,7 +20,7 @@ class InferenceParameters(object): :param constants: Specify a list of model variables as constants :type constants: {ModelComponent.uuid : mxnet.ndarray} - :param dtype: data type for internal numberical representation + :param dtype: data type for internal numerical representation :type dtype: {numpy.float64, numpy.float32, 'float64', 'float32'} :param context: The MXNet context :type context: {mxnet.cpu or mxnet.gpu} From 3074c79d3e4e64e9a319ab66ed03be67f1a75220 Mon Sep 17 00:00:00 2001 From: Tom Diethe Date: Fri, 19 Oct 2018 13:58:42 +0100 Subject: [PATCH 16/70] More typos, and a missing import --- mxfusion/components/distributions/gp/cond_gp.py | 4 ++-- mxfusion/inference/score_function.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mxfusion/components/distributions/gp/cond_gp.py b/mxfusion/components/distributions/gp/cond_gp.py index 1bd05be..ded9fdd 100644 --- a/mxfusion/components/distributions/gp/cond_gp.py +++ b/mxfusion/components/distributions/gp/cond_gp.py @@ -175,7 +175,7 @@ def define_variable(X, X_cond, Y_cond, kernel, shape=None, mean_func=None, def log_pdf(self, X, X_cond, Y_cond, random_variable, F=None, **kernel_params): """ - Computes the logarithm of the probability density function (PDF) of the condtional Gaussian process. + Computes the logarithm of the probability density function (PDF) of the conditional Gaussian process. .. math:: \\log p(Y| X_c, Y_c, X) = \\log \\mathcal{N}(Y| K_{*c}K_{cc}^{-1}(Y_C - g(X_c)) + g(X), K_{**} - K_{*c}K_{cc}^{-1}K_{*c}^\\top) @@ -221,7 +221,7 @@ def log_pdf(self, X, X_cond, Y_cond, random_variable, F=None, def draw_samples(self, X, X_cond, Y_cond, rv_shape, num_samples=1, F=None, **kernel_params): """ - Draw a number of samples from the condtional Gaussian process. + Draw a number of samples from the conditional Gaussian process. :param X: the input variables on which the random variables are conditioned. :type X: MXNet NDArray or MXNet Symbol diff --git a/mxfusion/inference/score_function.py b/mxfusion/inference/score_function.py index fd78f54..532d3e9 100644 --- a/mxfusion/inference/score_function.py +++ b/mxfusion/inference/score_function.py @@ -1,4 +1,6 @@ import mxnet as mx + +from mxfusion.common.exceptions import InferenceError from .inference_alg import InferenceAlgorithm from .variational import StochasticVariationalInference from ..components.variables import VariableType @@ -164,12 +166,12 @@ def compute(self, F, variables): def _extract_descendant_blanket_params(self, graph, node): """ - Returns a set of the markov blankets of all of the descendents of the node in the graph, mapped to their parameter form. + Returns a set of the markov blankets of all of the descendants of the node in the graph, mapped to their parameter form. """ if node.graph != graph.components_graph: - raise InferenceError("Graph of node and graph to find it's descendents in differ. These should match so something went wrong.") + raise InferenceError("Graph of node and graph to find it's descendants in differ. These should match so something went wrong.") - descendents = graph.get_descendants(node) - varset = [graph.get_markov_blanket(d) for d in descendents] + descendants = graph.get_descendants(node) + varset = [graph.get_markov_blanket(d) for d in descendants] varset = set(item for s in varset for item in s) return varset From f71d8d87a4b6edb94b20180cb54c840b0edfee54 Mon Sep 17 00:00:00 2001 From: Tom Diethe Date: Fri, 19 Oct 2018 14:24:36 +0100 Subject: [PATCH 17/70] Using dtype in bernoulli sampling --- mxfusion/components/distributions/bernoulli.py | 2 +- mxfusion/components/distributions/random_gen.py | 4 ++-- testing/distributions/bernoulli_test.py | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/mxfusion/components/distributions/bernoulli.py b/mxfusion/components/distributions/bernoulli.py index 26926f9..4036396 100644 --- a/mxfusion/components/distributions/bernoulli.py +++ b/mxfusion/components/distributions/bernoulli.py @@ -156,7 +156,7 @@ def draw_samples(self, prob_true, rv_shape, num_samples=1, F=None): :rtypes: MXNet NDArray or MXNet Symbol """ F = get_default_MXNet_mode() if F is None else F - return self._rand_gen.sample_bernoulli(prob_true, shape=(num_samples,) + rv_shape, F=F) + return self._rand_gen.sample_bernoulli(prob_true, shape=(num_samples,) + rv_shape, dtype=self.dtype, F=F) @staticmethod def define_variable(prob_true, shape=None, rand_gen=None, dtype=None, ctx=None): diff --git a/mxfusion/components/distributions/random_gen.py b/mxfusion/components/distributions/random_gen.py index 734647f..958075f 100644 --- a/mxfusion/components/distributions/random_gen.py +++ b/mxfusion/components/distributions/random_gen.py @@ -92,7 +92,7 @@ def sample_multinomial(data, get_prob=True, dtype='int32', F=None): data=data, get_prob=get_prob, dtype=dtype) @staticmethod - def sample_bernoulli(prob_true=0.5, dtype='bool', shape=None, F=None): + def sample_bernoulli(prob_true=0.5, dtype=None, shape=None, F=None): """ Sample Bernoulli distributed variables @@ -103,7 +103,7 @@ def sample_bernoulli(prob_true=0.5, dtype='bool', shape=None, F=None): :return: Array of samples """ F = get_default_MXNet_mode() if F is None else F - return F.random.uniform(low=0, high=1, shape=shape) > prob_true + return F.random.uniform(low=0, high=1, shape=shape, dtype=dtype) > prob_true @staticmethod def sample_gamma(alpha=1, beta=1, shape=None, dtype=None, out=None, ctx=None, F=None): diff --git a/testing/distributions/bernoulli_test.py b/testing/distributions/bernoulli_test.py index 84ff4f1..3cdf9bd 100644 --- a/testing/distributions/bernoulli_test.py +++ b/testing/distributions/bernoulli_test.py @@ -50,6 +50,7 @@ def test_log_pdf(self, dtype, prob_true, prob_true_is_samples, rv, rv_is_samples (np.float64, np.random.rand(4, 3), False, (4, 1), 5), (np.float64, np.random.rand(5, 4, 3), True, (4, 3), 5), (np.float64, np.random.rand(4, 3), False, (4, 3), 5), + (np.float32, np.random.rand(4, 3), False, (4, 3), 5), ]) def test_draw_samples(self, dtype, prob_true, prob_true_is_samples, rv_shape, num_samples): rv_full_shape = (num_samples,) + rv_shape @@ -69,5 +70,17 @@ def test_draw_samples(self, dtype, prob_true, prob_true_is_samples, rv_shape, nu assert is_sampled_array(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples - # TODO: should we have to cast to bool? or can mxnet handle bool natively? assert np.array_equal(rv_samples_np, rv_samples_rt.asnumpy().astype(bool)) + + # Also make sure the non-mock sampler works + rand_gen = None + var = Bernoulli.define_variable(0, shape=rv_shape, rand_gen=rand_gen, dtype=dtype).factor + prob_true_mx = mx.nd.array(prob_true, dtype=dtype) + if not prob_true_is_samples: + prob_true_mx = add_sample_dimension(mx.nd, prob_true_mx) + variables = {var.prob_true.uuid: prob_true_mx} + rv_samples_rt = var.draw_samples( + F=mx.nd, variables=variables, num_samples=num_samples) + + assert is_sampled_array(mx.nd, rv_samples_rt) + assert get_num_samples(mx.nd, rv_samples_rt) == num_samples From 9cb2cfdb51a1132c7154c03af7ff05444a1bd7eb Mon Sep 17 00:00:00 2001 From: Tom Diethe Date: Fri, 19 Oct 2018 14:28:36 +0100 Subject: [PATCH 18/70] Added test for output dtype --- testing/distributions/bernoulli_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/distributions/bernoulli_test.py b/testing/distributions/bernoulli_test.py index 3cdf9bd..3f7cab2 100644 --- a/testing/distributions/bernoulli_test.py +++ b/testing/distributions/bernoulli_test.py @@ -84,3 +84,4 @@ def test_draw_samples(self, dtype, prob_true, prob_true_is_samples, rv_shape, nu assert is_sampled_array(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples + assert rv_samples_rt.dtype == dtype From 6ecc87b8ee077261b73defb653f59cce8f7bdc84 Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 17:08:19 +0100 Subject: [PATCH 19/70] Remove unused line --- mxfusion/components/distributions/normal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mxfusion/components/distributions/normal.py b/mxfusion/components/distributions/normal.py index ab0d105..358c856 100644 --- a/mxfusion/components/distributions/normal.py +++ b/mxfusion/components/distributions/normal.py @@ -181,7 +181,6 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, shapes_map = {} shapes_map['mean'] = (num_samples,) + rv_shape shapes_map['covariance'] = (num_samples,) + rv_shape + (rv_shape[-1],) - shapes_map['random_variable'] = (num_samples,) + rv_shape variables = {name: broadcast_to_w_samples(F, v, shapes_map[name]) for name, v in variables.items()} From 8cb21c4980cda435823555cbca43a4a0f6fdee89 Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 17:10:03 +0100 Subject: [PATCH 20/70] Remove samples from rv_shape and change draw_samples tests to assert that samples are close --- testing/distributions/normal_test.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/testing/distributions/normal_test.py b/testing/distributions/normal_test.py index 623b31e..f172c91 100644 --- a/testing/distributions/normal_test.py +++ b/testing/distributions/normal_test.py @@ -211,7 +211,7 @@ def test_log_pdf_no_broadcast(self, dtype, mean, mean_isSamples, var, var_isSamp @pytest.mark.parametrize( "dtype, mean, mean_isSamples, var, var_isSamples, rv_shape, num_samples",[ - (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2,2)+0.1), False, (5,3,2), 5), + (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2,2)+0.1), False, (3,2), 5), ]) def test_draw_samples_with_broadcast(self, dtype, mean, mean_isSamples, var, var_isSamples, rv_shape, num_samples): @@ -224,26 +224,22 @@ def test_draw_samples_with_broadcast(self, dtype, mean, mean_isSamples, var, var_mx = add_sample_dimension(mx.nd, var_mx) var = var_mx.asnumpy() - isSamples_any = any([mean_isSamples, var_isSamples]) rand = np.random.rand(num_samples, *rv_shape) rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) rv_samples_np = mean + np.matmul(np.linalg.cholesky(var), np.expand_dims(rand, axis=-1)).sum(-1) normal = MultivariateNormal.define_variable(shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor variables = {normal.mean.uuid: mean_mx, normal.covariance.uuid: var_mx} - draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables) + draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) == isSamples_any - if isSamples_any: - assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, (get_num_samples(mx.nd, draw_samples_rt), num_samples) + assert np.allclose(rv_samples_np, draw_samples_rt.asnumpy()) @pytest.mark.parametrize( "dtype, mean, mean_isSamples, var, var_isSamples, rv_shape, num_samples",[ - (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2,2)+0.1), False, (5,3,2), 5), - (np.float64, np.random.rand(5,2), True, make_symmetric(np.random.rand(2,2)+0.1), False, (5,3,2), 5), - (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(5,2,2)+0.1), True, (5,3,2), 5), - (np.float64, np.random.rand(5,2), True, make_symmetric(np.random.rand(5,2,2)+0.1), True, (5,3,2), 5), + (np.float64, np.random.rand(5,2), True, make_symmetric(np.random.rand(2,2)+0.1), False, (3,2), 5), + (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(5,2,2)+0.1), True, (3,2), 5), + (np.float64, np.random.rand(5,2), True, make_symmetric(np.random.rand(5,2,2)+0.1), True, (3,2), 5) ]) def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype, mean, mean_isSamples, var, var_isSamples, rv_shape, num_samples): @@ -264,12 +260,13 @@ def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype, mean, me draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) == True + assert is_sampled_array(mx.nd, draw_samples_rt) is True + assert draw_samples_rt.shape == (5,) + rv_shape @pytest.mark.parametrize( "dtype, mean, mean_isSamples, var, var_isSamples, rv_shape, num_samples",[ (np.float64, np.random.rand(2), False, make_symmetric(np.random.rand(2,2)+0.1), False, (3,2), 5), - (np.float64, np.random.rand(5,3,2), True, make_symmetric(np.random.rand(5,3,2,2)+0.1), True, (5,3,2), 5), + (np.float64, np.random.rand(5,3,2), True, make_symmetric(np.random.rand(5,3,2,2)+0.1), True, (3,2), 5), ]) def test_draw_samples_no_broadcast(self, dtype, mean, mean_isSamples, var, var_isSamples, rv_shape, num_samples): From 2f679a14ff91ae853f3f24d7793661d64f1e810d Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 19 Oct 2018 17:12:51 +0100 Subject: [PATCH 21/70] Add dtype argument to create_Gaussian_meanfield. --- mxfusion/inference/batch_loop.py | 2 +- mxfusion/inference/meanfield.py | 6 ++++-- mxfusion/inference/minibatch_loop.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mxfusion/inference/batch_loop.py b/mxfusion/inference/batch_loop.py index 611fe74..ec79129 100644 --- a/mxfusion/inference/batch_loop.py +++ b/mxfusion/inference/batch_loop.py @@ -8,7 +8,7 @@ class BatchInferenceLoop(GradLoop): """ def run(self, infr_executor, data, param_dict, ctx, optimizer='adam', - learning_rate=1e-3, max_iter=2000, n_prints=10, verbose=False): + learning_rate=1e-3, max_iter=1000, n_prints=10, verbose=False): """ :param infr_executor: The MXNet function that computes the training objective. :type infr_executor: MXNet Gluon Block diff --git a/mxfusion/inference/meanfield.py b/mxfusion/inference/meanfield.py index b0826f6..bd70222 100644 --- a/mxfusion/inference/meanfield.py +++ b/mxfusion/inference/meanfield.py @@ -3,9 +3,10 @@ from ..components.variables import Variable, VariableType from ..components.distributions.normal import Normal from ..util.inference import variables_to_UUID +from ..common.config import get_default_dtype -def create_Gaussian_meanfield(model, observed): +def create_Gaussian_meanfield(model, observed, dtype): """ Create the Meanfield posterior for Variational Inference. @@ -16,6 +17,7 @@ def create_Gaussian_meanfield(model, observed): :returns: the resulting posterior representation :rtype: Posterior """ + dtype = get_default_dtype() if dtype is None else dtype observed = variables_to_UUID(observed) q = Posterior(model) for v in model.variables.values(): @@ -23,5 +25,5 @@ def create_Gaussian_meanfield(model, observed): mean = Variable(shape=v.shape) variance = Variable(shape=v.shape, transformation=PositiveTransformation()) - q[v].set_prior(Normal(mean=mean, variance=variance)) + q[v].set_prior(Normal(mean=mean, variance=variance, dtype=dtype)) return q diff --git a/mxfusion/inference/minibatch_loop.py b/mxfusion/inference/minibatch_loop.py index 09d3227..8daa72a 100644 --- a/mxfusion/inference/minibatch_loop.py +++ b/mxfusion/inference/minibatch_loop.py @@ -25,7 +25,7 @@ def __init__(self, batch_size=100, rv_scaling=None): if rv_scaling is not None else rv_scaling def run(self, infr_executor, data, param_dict, ctx, optimizer='adam', - learning_rate=1e-3, max_iter=2000, verbose=False): + learning_rate=1e-3, max_iter=1000, verbose=False): """ :param infr_executor: The MXNet function that computes the training objective. :type infr_executor: MXNet Gluon Block From f447c70c097ca77ab5475620a099303d1249b994 Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 17:15:33 +0100 Subject: [PATCH 22/70] Refactor dirichlet to correctly use rv_shape and num_samples --- .../components/distributions/dirichlet.py | 10 ++--- testing/distributions/dirichlet_test.py | 38 +++++++------------ 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/mxfusion/components/distributions/dirichlet.py b/mxfusion/components/distributions/dirichlet.py index fe23729..7305e75 100644 --- a/mxfusion/components/distributions/dirichlet.py +++ b/mxfusion/components/distributions/dirichlet.py @@ -79,7 +79,6 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, shapes_map = {} shapes_map['a'] = (num_samples,) + rv_shape - shapes_map['random_variable'] = (num_samples,) + rv_shape variables = {name: broadcast_to_w_samples(F, v, shapes_map[name]) for name, v in variables.items()} @@ -147,22 +146,19 @@ def draw_samples(self, a, rv_shape, num_samples=1, F=None): :param a: the a parameter (alpha) of the Dirichlet distribution. :type a: MXNet NDArray or MXNet Symbol - :param tuple rv_shape: the shape of each sample. + :param tuple rv_shape: the shape of each sample (this variable is not used because the shape of the random var + is given by the shape of a) :param int num_samples: the number of drawn samples (default: one). :param F: the MXNet computation mode (mxnet.symbol or mxnet.ndarray). :returns: a set samples of the Dirichlet distribution. :rtypes: MXNet NDArray or MXNet Symbol """ F = get_default_MXNet_mode() if F is None else F - out_shape = (num_samples,) + rv_shape + (1,) ones = F.ones_like(a) - y = self._rand_gen.sample_gamma(alpha=a, beta=ones, - shape=out_shape, dtype=self.dtype, ctx=self.ctx) - x = F.broadcast_div(y, F.sum(y)) - return x + return F.broadcast_div(y, F.sum(y)) @staticmethod def define_variable(a, shape=None, normalization=True, diff --git a/testing/distributions/dirichlet_test.py b/testing/distributions/dirichlet_test.py index 125efbe..42b477e 100644 --- a/testing/distributions/dirichlet_test.py +++ b/testing/distributions/dirichlet_test.py @@ -106,60 +106,50 @@ def test_log_pdf_no_broadcast(self, dtype, a, a_is_samples, assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) @pytest.mark.parametrize("dtype, a, a_is_samples, rv_shape, num_samples", [ - (np.float64, np.random.rand(2), False, (5, 3, 2), 5), + (np.float64, np.random.rand(2), False, (3, 2), 5) ]) def test_draw_samples_with_broadcast(self, dtype, a, a_is_samples, rv_shape, num_samples): - a_mx = mx.nd.array(a, dtype=dtype) if not a_is_samples: a_mx = add_sample_dimension(mx.nd, a_mx) - is_samples_any = a_is_samples - rand = np.random.rand(num_samples, *rv_shape) + rand = np.random.gamma(shape=a, scale=np.ones(a.shape), size=(num_samples,)+rv_shape) + draw_samples_np = rand / np.sum(rand) rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor variables = {dirichlet.a.uuid: a_mx} - draw_samples_rt = dirichlet.draw_samples(F=mx.nd, variables=variables) + draw_samples_rt = dirichlet.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) == is_samples_any - if is_samples_any: - assert get_num_samples(mx.nd, draw_samples_rt) == num_samples,\ - (get_num_samples(mx.nd, draw_samples_rt), num_samples) + assert draw_samples_rt.shape == (5,) + rv_shape + assert np.allclose(draw_samples_np, draw_samples_rt.asnumpy()) @pytest.mark.parametrize("dtype, a, a_is_samples, rv_shape, num_samples", [ - (np.float64, np.random.rand(2), False, (5, 3, 2), 5), - (np.float64, np.random.rand(5, 2), True, (5, 3, 2), 5), - (np.float64, np.random.rand(2), False, (5, 3, 2), 5), - (np.float64, np.random.rand(5, 2), True, (5, 3, 2), 5), + (np.float64, np.random.rand(5, 2), True, (3, 2), 5) ]) def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype, a, a_is_samples, rv_shape, num_samples): a_mx = mx.nd.array(a, dtype=dtype) if not a_is_samples: a_mx = add_sample_dimension(mx.nd, a_mx) - rand = np.random.rand(num_samples, *rv_shape) - rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) - - dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype).factor variables = {dirichlet.a.uuid: a_mx} draw_samples_rt = dirichlet.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) is True + assert draw_samples_rt.shape == (5,) + rv_shape @pytest.mark.parametrize("dtype, a, a_is_samples, rv_shape, num_samples", [ - (np.float64, np.random.rand(2), False, (3, 2), 5), - (np.float64, np.random.rand(5, 3, 2), True, (5, 3, 2), 5), + (np.float64, np.random.rand(5, 3, 2), True, (3, 2), 5) ]) def test_draw_samples_no_broadcast(self, dtype, a, a_is_samples, rv_shape, num_samples): a_mx = mx.nd.array(a, dtype=dtype) if not a_is_samples: a_mx = add_sample_dimension(mx.nd, a_mx) - # n_dim = 1 + len(rv.shape) if is_samples_any else len(rv.shape) - rand = np.random.rand(num_samples, *rv_shape) + rand = np.random.gamma(shape=a, scale=np.ones(a.shape), size=(num_samples,)+rv_shape) + draw_samples_np = rand / np.sum(rand) rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor @@ -167,6 +157,4 @@ def test_draw_samples_no_broadcast(self, dtype, a, a_is_samples, rv_shape, num_s draw_samples_rt = dirichlet.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) is True - assert get_num_samples(mx.nd, draw_samples_rt) == num_samples,\ - (get_num_samples(mx.nd, draw_samples_rt), num_samples) + assert np.allclose(draw_samples_np, draw_samples_rt.asnumpy()) From 6b2f4a753caa2af8d9ab43020a5c5bf4359c8ec0 Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 17:17:11 +0100 Subject: [PATCH 23/70] Update mock sample_gamma to be equivalent to mx.random.gamma --- mxfusion/util/testutils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mxfusion/util/testutils.py b/mxfusion/util/testutils.py index 008d7f3..51ea43d 100644 --- a/mxfusion/util/testutils.py +++ b/mxfusion/util/testutils.py @@ -62,9 +62,7 @@ def sample_multinomial(self, data, shape=None, get_prob=True, dtype='int32', return mx.nd.reshape(self._samples[:np.prod(data.shape[:-1])], shape=data.shape[:-1]) def sample_gamma(self, alpha=1, beta=1, shape=None, dtype=None, out=None, ctx=None, F=None): - if shape is None: - shape = (1,) - res = mx.nd.reshape(self._samples[:np.prod(shape)], shape=shape) + res = mx.nd.reshape(self._samples[:np.prod(shape)], shape=alpha.shape) if out is not None: out[:] = res return res From 02a6b11d94aeaae13c224b4a0663efa7fd1ce848 Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 17:18:05 +0100 Subject: [PATCH 24/70] Allow random gen without passing shape --- mxfusion/components/distributions/random_gen.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mxfusion/components/distributions/random_gen.py b/mxfusion/components/distributions/random_gen.py index 7989952..96a0dd7 100644 --- a/mxfusion/components/distributions/random_gen.py +++ b/mxfusion/components/distributions/random_gen.py @@ -39,7 +39,11 @@ def _sample_univariate(func, shape=None, dtype=None, out=None, ctx=None, F=None, dtype = get_default_dtype() if dtype is None else dtype if F is mx.ndarray: - return func(shape=shape, dtype=dtype, ctx=ctx, out=out, **kwargs) + # This is required MXNet uses _Null instead of None as shape default + if shape is None: + return func(dtype=dtype, ctx=ctx, out=out, **kwargs) + else: + return func(shape=shape, dtype=dtype, ctx=ctx, out=out, **kwargs) else: return func(shape=shape, dtype=dtype, out=out, **kwargs) @@ -90,7 +94,9 @@ def sample_gamma(alpha=1, beta=1, shape=None, dtype=None, out=None, ctx=None, F= :param alpha: Also known as shape :param beta: Also known as rate - :param shape: Array shape of samples + :param shape: The number of samples to draw. If shape is, e.g., (m, n) and alpha and beta are scalars, output + shape will be (m, n). If alpha and beta are NDArrays with shape, e.g., (x, y), then output will have shape + (x, y, m, n), where m*n samples are drawn for each [alpha, beta) pair. :param dtype: Data type :param out: output variable :param ctx: execution context From 85d222a25d3473fc975d98d5faad6ab44e2a9f1b Mon Sep 17 00:00:00 2001 From: Massiah Date: Fri, 19 Oct 2018 17:24:39 +0100 Subject: [PATCH 25/70] is_sampled_array->array_has_samples --- mxfusion/components/distributions/categorical.py | 4 ++-- mxfusion/components/distributions/dirichlet.py | 4 ++-- mxfusion/components/distributions/gamma.py | 2 +- mxfusion/components/distributions/normal.py | 4 ++-- mxfusion/components/distributions/univariate.py | 4 ++-- mxfusion/components/distributions/wishart.py | 4 ++-- .../components/functions/function_evaluation.py | 6 +++--- mxfusion/components/variables/__init__.py | 2 +- mxfusion/components/variables/runtime_variable.py | 4 ++-- mxfusion/util/testutils.py | 4 ++-- testing/core/factor_graph_test.py | 4 ++-- testing/distributions/beta_test.py | 6 +++--- testing/distributions/categorical_test.py | 4 ++-- testing/distributions/dirichlet_test.py | 6 +++--- testing/distributions/gamma_test.py | 10 +++++----- testing/distributions/gp/cond_gp_test.py | 6 +++--- testing/distributions/gp/gp_test.py | 6 +++--- testing/distributions/gp/kernel_test.py | 2 +- testing/distributions/normal_test.py | 14 +++++++------- testing/distributions/wishart_test.py | 10 +++++----- testing/functions/function_evaluation_test.py | 4 ++-- testing/functions/mxfusion_function_test.py | 2 +- testing/functions/mxfusion_gluon_function_test.py | 6 +++--- 23 files changed, 59 insertions(+), 59 deletions(-) diff --git a/mxfusion/components/distributions/categorical.py b/mxfusion/components/distributions/categorical.py index c63c30e..86fb55e 100644 --- a/mxfusion/components/distributions/categorical.py +++ b/mxfusion/components/distributions/categorical.py @@ -2,7 +2,7 @@ from .univariate import UnivariateDistribution from .distribution import LogPDFDecorator, DrawSamplesDecorator from ...util.customop import broadcast_to_w_samples -from ..variables import get_num_samples, is_sampled_array +from ..variables import get_num_samples, array_has_samples from ...common.config import get_default_MXNet_mode from ...common.exceptions import InferenceError @@ -61,7 +61,7 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, rv_shape = list(rv_shape.values())[0] variables = {name: kw[name] for name, _ in self.inputs} - isSamples = any([is_sampled_array(F, v) for v in variables.values()]) + isSamples = any([array_has_samples(F, v) for v in variables.values()]) if isSamples: num_samples_inferred = max([get_num_samples(F, v) for v in variables.values()]) diff --git a/mxfusion/components/distributions/dirichlet.py b/mxfusion/components/distributions/dirichlet.py index 7305e75..3683526 100644 --- a/mxfusion/components/distributions/dirichlet.py +++ b/mxfusion/components/distributions/dirichlet.py @@ -1,7 +1,7 @@ from ..variables import Variable from ...common.config import get_default_MXNet_mode from .distribution import Distribution, LogPDFDecorator, DrawSamplesDecorator -from ..variables import is_sampled_array, get_num_samples +from ..variables import array_has_samples, get_num_samples from ...util.customop import broadcast_to_w_samples from ...common.exceptions import InferenceError @@ -67,7 +67,7 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, rv_shape = list(rv_shape.values())[0] variables = {name: kw[name] for name, _ in self.inputs} - isSamples = any([is_sampled_array(F, v) for v in variables.values()]) + isSamples = any([array_has_samples(F, v) for v in variables.values()]) if isSamples: num_samples_inferred = max([get_num_samples(F, v) for v in variables.values()]) diff --git a/mxfusion/components/distributions/gamma.py b/mxfusion/components/distributions/gamma.py index d1a4592..ffe792e 100644 --- a/mxfusion/components/distributions/gamma.py +++ b/mxfusion/components/distributions/gamma.py @@ -4,7 +4,7 @@ from ..variables import Variable from .univariate import UnivariateDistribution, UnivariateLogPDFDecorator, UnivariateDrawSamplesDecorator from .distribution import Distribution, LogPDFDecorator, DrawSamplesDecorator -from ..variables import is_sampled_array, get_num_samples +from ..variables import array_has_samples, get_num_samples from ...util.customop import broadcast_to_w_samples diff --git a/mxfusion/components/distributions/normal.py b/mxfusion/components/distributions/normal.py index 358c856..0035cbf 100644 --- a/mxfusion/components/distributions/normal.py +++ b/mxfusion/components/distributions/normal.py @@ -5,7 +5,7 @@ from ..variables import Variable from .univariate import UnivariateDistribution, UnivariateLogPDFDecorator, UnivariateDrawSamplesDecorator from .distribution import Distribution, LogPDFDecorator, DrawSamplesDecorator -from ..variables import is_sampled_array, get_num_samples +from ..variables import array_has_samples, get_num_samples from ...util.customop import broadcast_to_w_samples @@ -171,7 +171,7 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, rv_shape = list(rv_shape.values())[0] variables = {name: kw[name] for name, _ in self.inputs} - isSamples = any([is_sampled_array(F, v) for v in variables.values()]) + isSamples = any([array_has_samples(F, v) for v in variables.values()]) if isSamples: num_samples_inferred = max([get_num_samples(F, v) for v in variables.values()]) diff --git a/mxfusion/components/distributions/univariate.py b/mxfusion/components/distributions/univariate.py index 95aa2ea..f647381 100644 --- a/mxfusion/components/distributions/univariate.py +++ b/mxfusion/components/distributions/univariate.py @@ -1,7 +1,7 @@ from ...common.exceptions import InferenceError from ..variables import Variable from .distribution import Distribution, LogPDFDecorator, DrawSamplesDecorator -from ..variables import is_sampled_array, get_num_samples +from ..variables import array_has_samples, get_num_samples from ...util.customop import broadcast_to_w_samples @@ -56,7 +56,7 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, rv_shape = list(rv_shape.values())[0] variables = {name: kws[name] for name, _ in self.inputs} - isSamples = any([is_sampled_array(F, v) for v in variables.values()]) + isSamples = any([array_has_samples(F, v) for v in variables.values()]) if isSamples: num_samples_inferred = max([get_num_samples(F, v) for v in variables.values()]) diff --git a/mxfusion/components/distributions/wishart.py b/mxfusion/components/distributions/wishart.py index 1af4537..55c1eb9 100644 --- a/mxfusion/components/distributions/wishart.py +++ b/mxfusion/components/distributions/wishart.py @@ -6,7 +6,7 @@ from ...common.config import get_default_MXNet_mode from ..variables import Variable from .distribution import Distribution, LogPDFDecorator, DrawSamplesDecorator -from ..variables import is_sampled_array, get_num_samples +from ..variables import array_has_samples, get_num_samples from ...util.customop import broadcast_to_w_samples @@ -77,7 +77,7 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, rv_shape = list(rv_shape.values())[0] variables = {name: kw[name] for name, _ in self.inputs} - is_samples = any([is_sampled_array(F, v) for v in variables.values()]) + is_samples = any([array_has_samples(F, v) for v in variables.values()]) if is_samples: num_samples_inferred = max([get_num_samples(F, v) for v in variables.values()]) diff --git a/mxfusion/components/functions/function_evaluation.py b/mxfusion/components/functions/function_evaluation.py index 4e02330..3cc19b0 100644 --- a/mxfusion/components/functions/function_evaluation.py +++ b/mxfusion/components/functions/function_evaluation.py @@ -1,6 +1,6 @@ from abc import abstractmethod from ..factor import Factor -from ..variables import is_sampled_array, get_num_samples, as_samples +from ..variables import array_has_samples, get_num_samples, as_samples from ..variables import VariableType @@ -40,7 +40,7 @@ def eval_RT(self, F, always_return_tuple=False, **input_kws): The method handling the execution of the function with RTVariable as its input arguments and return values. """ - has_samples = any([is_sampled_array(F, v) for v in input_kws.values()]) + has_samples = any([array_has_samples(F, v) for v in input_kws.values()]) if not has_samples: # If none of the inputs are samples, directly evaluate the function nSamples = 0 @@ -65,7 +65,7 @@ def eval_RT(self, F, always_return_tuple=False, **input_kws): for sample_idx in range(nSamples): r = func( self, F=F, **{ - n: v[sample_idx] if is_sampled_array(F, v) else + n: v[sample_idx] if array_has_samples(F, v) else v[0] for n, v in input_kws.items()}) if isinstance(r, (list, tuple)): r = [F.expand_dims(i, axis=0) for i in r] diff --git a/mxfusion/components/variables/__init__.py b/mxfusion/components/variables/__init__.py index 73e89b3..9c5d9fb 100644 --- a/mxfusion/components/variables/__init__.py +++ b/mxfusion/components/variables/__init__.py @@ -13,6 +13,6 @@ __all__ = ['runtime_variable', 'var_trans', 'variable'] -from .runtime_variable import add_sample_dimension, add_sample_dimension_to_arrays, expectation, is_sampled_array, get_num_samples, as_samples +from .runtime_variable import add_sample_dimension, add_sample_dimension_to_arrays, expectation, array_has_samples, get_num_samples, as_samples from .var_trans import Softplus, PositiveTransformation from .variable import Variable, VariableType diff --git a/mxfusion/components/variables/runtime_variable.py b/mxfusion/components/variables/runtime_variable.py index 8c9c133..f516676 100644 --- a/mxfusion/components/variables/runtime_variable.py +++ b/mxfusion/components/variables/runtime_variable.py @@ -45,7 +45,7 @@ def expectation(F, array): return F.mean(array, axis=0) -def is_sampled_array(F, array): +def array_has_samples(F, array): """ Check if the array is a set of samples. @@ -78,7 +78,7 @@ def as_samples(F, array, num_samples): :param num_samples: the number of samples :type num_samples: int """ - if is_sampled_array(F, array): + if array_has_samples(F, array): return array else: return F.broadcast_axis(array, axis=0, size=num_samples) diff --git a/mxfusion/util/testutils.py b/mxfusion/util/testutils.py index 51ea43d..44acbfb 100644 --- a/mxfusion/util/testutils.py +++ b/mxfusion/util/testutils.py @@ -7,9 +7,9 @@ from ..components.distributions.random_gen import RandomGenerator from ..components.variables import add_sample_dimension -def prepare_mxnet_array(array, is_sampled_array, dtype): +def prepare_mxnet_array(array, array_has_samples, dtype): a_mx = mx.nd.array(array, dtype=dtype) - if not is_sampled_array: + if not array_has_samples: a_mx = add_sample_dimension(mx.nd, a_mx) return a_mx diff --git a/testing/core/factor_graph_test.py b/testing/core/factor_graph_test.py index e31f13f..7cb1cb7 100644 --- a/testing/core/factor_graph_test.py +++ b/testing/core/factor_graph_test.py @@ -10,7 +10,7 @@ from mxfusion.components.distributions.normal import Normal from mxfusion.components import Variable from mxfusion.models import Model -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -214,7 +214,7 @@ def test_draw_samples(self): samples_np = v_np + samples_1_np[:, None] + np.sqrt(0.1)*samples_2_np.reshape(5,10) - assert is_sampled_array(mx.nd, samples) and get_num_samples(mx.nd, samples)==5 + assert array_has_samples(mx.nd, samples) and get_num_samples(mx.nd, samples)==5 assert np.allclose(samples.asnumpy(), samples_np) def test_reconcile_simple_model(self): diff --git a/testing/distributions/beta_test.py b/testing/distributions/beta_test.py index 6a2a5b2..ea789f5 100644 --- a/testing/distributions/beta_test.py +++ b/testing/distributions/beta_test.py @@ -1,7 +1,7 @@ import pytest import mxnet as mx import numpy as np -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.components.distributions import Beta from mxfusion.util.testutils import numpy_array_reshape from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -44,7 +44,7 @@ def test_log_pdf(self, dtype, a, a_is_samples, b, b_is_samples, rv, rv_is_sample log_pdf_rt = var.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + assert array_has_samples(mx.nd, log_pdf_rt) == is_samples_any if is_samples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples if np.issubdtype(dtype, np.float64): @@ -87,7 +87,7 @@ def test_draw_samples(self, dtype, a_shape, a_is_samples, b_shape, b_is_samples, rv_samples_rt = var.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(rv_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, rv_samples_rt) + assert array_has_samples(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples rtol, atol = 1e-1, 1e-1 diff --git a/testing/distributions/categorical_test.py b/testing/distributions/categorical_test.py index 9c70088..31233e5 100644 --- a/testing/distributions/categorical_test.py +++ b/testing/distributions/categorical_test.py @@ -1,7 +1,7 @@ import pytest import mxnet as mx import numpy as np -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.components.distributions import Categorical from mxfusion.util.testutils import numpy_array_reshape from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -83,6 +83,6 @@ def test_draw_samples(self, dtype, log_prob, log_prob_isSamples, rv_shape, num_s rv_samples_rt = cat.draw_samples( F=mx.nd, variables=variables, num_samples=num_samples) - assert is_sampled_array(mx.nd, rv_samples_rt) + assert array_has_samples(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples assert np.allclose(rv_samples_np, rv_samples_rt.asnumpy()) diff --git a/testing/distributions/dirichlet_test.py b/testing/distributions/dirichlet_test.py index 42b477e..bc4bdab 100644 --- a/testing/distributions/dirichlet_test.py +++ b/testing/distributions/dirichlet_test.py @@ -4,7 +4,7 @@ import mxnet as mx from scipy.stats import dirichlet as scipy_dirichlet -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.components.distributions import Dirichlet from mxfusion.util.testutils import numpy_array_reshape from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -55,7 +55,7 @@ def test_log_pdf_with_broadcast(self, dtype, a, a_is_samples, rv, rv_is_samples, log_pdf_rt = dirichlet.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + assert array_has_samples(mx.nd, log_pdf_rt) == is_samples_any if is_samples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) @@ -100,7 +100,7 @@ def test_log_pdf_no_broadcast(self, dtype, a, a_is_samples, log_pdf_rt = dirichlet.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + assert array_has_samples(mx.nd, log_pdf_rt) == is_samples_any if is_samples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) diff --git a/testing/distributions/gamma_test.py b/testing/distributions/gamma_test.py index da1d3e2..4302277 100644 --- a/testing/distributions/gamma_test.py +++ b/testing/distributions/gamma_test.py @@ -1,7 +1,7 @@ import pytest import mxnet as mx import numpy as np -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.components.distributions import Gamma, GammaMeanVariance from mxfusion.util.testutils import numpy_array_reshape from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -46,7 +46,7 @@ def test_log_pdf_mean_variance(self, dtype, mean, mean_isSamples, variance, vari log_pdf_rt = gamma.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == isSamples_any + assert array_has_samples(mx.nd, log_pdf_rt) == isSamples_any if isSamples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples if np.issubdtype(dtype, np.float64): @@ -89,7 +89,7 @@ def test_draw_samples_mean_variance(self, dtype, mean, mean_isSamples, variance, rv_samples_mx = mx.nd.random.gamma(alpha=alpha_np, beta=beta_np, dtype=dtype) assert np.issubdtype(rv_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, rv_samples_rt) + assert array_has_samples(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples if np.issubdtype(dtype, np.float64): @@ -133,7 +133,7 @@ def test_log_pdf(self, dtype, alpha, alpha_isSamples, beta, beta_isSamples, log_pdf_rt = gamma.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == isSamples_any + assert array_has_samples(mx.nd, log_pdf_rt) == isSamples_any if isSamples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples if np.issubdtype(dtype, np.float64): @@ -175,7 +175,7 @@ def test_draw_samples(self, dtype, alpha, alpha_isSamples, beta, rv_samples_mx = mx.nd.random.gamma(alpha=alpha_np, beta=beta_np, dtype=dtype) assert np.issubdtype(rv_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, rv_samples_rt) + assert array_has_samples(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples if np.issubdtype(dtype, np.float64): diff --git a/testing/distributions/gp/cond_gp_test.py b/testing/distributions/gp/cond_gp_test.py index f74dd18..b2e62b8 100644 --- a/testing/distributions/gp/cond_gp_test.py +++ b/testing/distributions/gp/cond_gp_test.py @@ -2,7 +2,7 @@ import mxnet as mx import numpy as np from mxfusion.models import Model -from mxfusion.components.variables.runtime_variable import is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import array_has_samples, get_num_samples from mxfusion.components.distributions import ConditionalGaussianProcess from mxfusion.components.distributions.gp.kernels import RBF from mxfusion.components.variables import Variable @@ -68,7 +68,7 @@ def test_log_pdf(self, dtype, X, X_isSamples, X_cond, X_cond_isSamples, Y_cond, log_pdf_np = np.array(log_pdf_np) isSamples_any = any([X_isSamples, rbf_lengthscale_isSamples, rbf_variance_isSamples, rv_isSamples]) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == isSamples_any + assert array_has_samples(mx.nd, log_pdf_rt) == isSamples_any if isSamples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples assert np.allclose(log_pdf_np, log_pdf_rt) @@ -181,7 +181,7 @@ def test_clone_cond_gp(self, dtype, X, X_isSamples, X_cond, X_cond_isSamples, Y_ log_pdf_np = np.array(log_pdf_np) isSamples_any = any([X_isSamples, rbf_lengthscale_isSamples, rbf_variance_isSamples, rv_isSamples]) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == isSamples_any + assert array_has_samples(mx.nd, log_pdf_rt) == isSamples_any if isSamples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples assert np.allclose(log_pdf_np, log_pdf_rt) diff --git a/testing/distributions/gp/gp_test.py b/testing/distributions/gp/gp_test.py index 5f4b063..87af0c4 100644 --- a/testing/distributions/gp/gp_test.py +++ b/testing/distributions/gp/gp_test.py @@ -2,7 +2,7 @@ import mxnet as mx import numpy as np from mxfusion.models import Model -from mxfusion.components.variables.runtime_variable import is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import array_has_samples, get_num_samples from mxfusion.components.distributions import GaussianProcess from mxfusion.components.distributions.gp.kernels import RBF from mxfusion.components import Variable @@ -52,7 +52,7 @@ def test_log_pdf(self, dtype, X, X_isSamples, rbf_lengthscale, rbf_lengthscale_i log_pdf_np = np.array(log_pdf_np) isSamples_any = any([X_isSamples, rbf_lengthscale_isSamples, rbf_variance_isSamples, rv_isSamples]) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == isSamples_any + assert array_has_samples(mx.nd, log_pdf_rt) == isSamples_any if isSamples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples assert np.allclose(log_pdf_np, log_pdf_rt) @@ -135,7 +135,7 @@ def test_clone_gp(self, dtype, X, X_isSamples, rbf_lengthscale, rbf_lengthscale_ log_pdf_np = np.array(log_pdf_np) isSamples_any = any([X_isSamples, rbf_lengthscale_isSamples, rbf_variance_isSamples, rv_isSamples]) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == isSamples_any + assert array_has_samples(mx.nd, log_pdf_rt) == isSamples_any if isSamples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples assert np.allclose(log_pdf_np, log_pdf_rt) diff --git a/testing/distributions/gp/kernel_test.py b/testing/distributions/gp/kernel_test.py index 79ae49a..e456e4e 100644 --- a/testing/distributions/gp/kernel_test.py +++ b/testing/distributions/gp/kernel_test.py @@ -2,7 +2,7 @@ import mxnet as mx import numpy as np from mxfusion.components.variables import Variable -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.components.distributions.gp.kernels import RBF, Linear, Bias, White from mxfusion.util.testutils import numpy_array_reshape, prepare_mxnet_array diff --git a/testing/distributions/normal_test.py b/testing/distributions/normal_test.py index f172c91..39d9585 100644 --- a/testing/distributions/normal_test.py +++ b/testing/distributions/normal_test.py @@ -1,7 +1,7 @@ import pytest import mxnet as mx import numpy as np -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.components.distributions import Normal, MultivariateNormal from mxfusion.util.testutils import numpy_array_reshape from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -42,7 +42,7 @@ def test_log_pdf(self, dtype, mean, mean_isSamples, var, var_isSamples, log_pdf_rt = normal.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == isSamples_any + assert array_has_samples(mx.nd, log_pdf_rt) == isSamples_any if isSamples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples if np.issubdtype(dtype, np.float64): @@ -84,7 +84,7 @@ def test_draw_samples(self, dtype, mean, mean_isSamples, var, F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(rv_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, rv_samples_rt) + assert array_has_samples(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples if np.issubdtype(dtype, np.float64): @@ -151,7 +151,7 @@ def test_log_pdf_with_broadcast(self, dtype, mean, mean_isSamples, var, var_isSa log_pdf_rt = normal.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == isSamples_any + assert array_has_samples(mx.nd, log_pdf_rt) == isSamples_any if isSamples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) @@ -204,7 +204,7 @@ def test_log_pdf_no_broadcast(self, dtype, mean, mean_isSamples, var, var_isSamp log_pdf_rt = normal.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == isSamples_any + assert array_has_samples(mx.nd, log_pdf_rt) == isSamples_any if isSamples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) @@ -260,7 +260,7 @@ def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype, mean, me draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) is True + assert array_has_samples(mx.nd, draw_samples_rt) is True assert draw_samples_rt.shape == (5,) + rv_shape @pytest.mark.parametrize( @@ -293,5 +293,5 @@ def test_draw_samples_no_broadcast(self, dtype, mean, mean_isSamples, var, draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) == True + assert array_has_samples(mx.nd, draw_samples_rt) == True assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, (get_num_samples(mx.nd, draw_samples_rt), num_samples) diff --git a/testing/distributions/wishart_test.py b/testing/distributions/wishart_test.py index 20cc478..b1bd15b 100644 --- a/testing/distributions/wishart_test.py +++ b/testing/distributions/wishart_test.py @@ -5,7 +5,7 @@ from scipy.stats import wishart from mxfusion.components.distributions import Wishart -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.util.testutils import MockMXNetRandomGenerator, numpy_array_reshape @@ -79,7 +79,7 @@ def test_log_pdf(self, dtype_dof, dtype, degrees_of_freedom, random_state, log_pdf_rt = var.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + assert array_has_samples(mx.nd, log_pdf_rt) == is_samples_any if is_samples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) @@ -112,7 +112,7 @@ def test_draw_samples_with_broadcast(self, dtype_dof, dtype, degrees_of_freedom, draw_samples_rt = var.draw_samples(F=mx.nd, variables=variables) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) == scale_is_samples + assert array_has_samples(mx.nd, draw_samples_rt) == scale_is_samples if scale_is_samples: assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, (get_num_samples(mx.nd, draw_samples_rt), num_samples) @@ -143,7 +143,7 @@ def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype_dof, dtyp draw_samples_rt = var.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) + assert array_has_samples(mx.nd, draw_samples_rt) @pytest.mark.parametrize( "dtype_dof, dtype, degrees_of_freedom, scale, scale_is_samples, rv_shape, num_samples", [ @@ -171,6 +171,6 @@ def test_draw_samples_no_broadcast(self, dtype_dof, dtype, degrees_of_freedom, s draw_samples_rt = var.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) + assert array_has_samples(mx.nd, draw_samples_rt) assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, (get_num_samples(mx.nd, draw_samples_rt), num_samples) diff --git a/testing/functions/function_evaluation_test.py b/testing/functions/function_evaluation_test.py index dbba209..c13001d 100644 --- a/testing/functions/function_evaluation_test.py +++ b/testing/functions/function_evaluation_test.py @@ -1,7 +1,7 @@ import pytest import mxnet as mx import numpy as np -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples @pytest.mark.usefixtures("set_seed") @@ -62,5 +62,5 @@ def test_eval(self, dtype, A, A_isSamples, B, B_isSamples, num_samples, variables = {eval.A.uuid: A_mx, eval.B.uuid: B_mx} res_rt = eval.eval(F=mx.nd, variables=variables) - assert np_isSamples == is_sampled_array(mx.nd, res_rt) + assert np_isSamples == array_has_samples(mx.nd, res_rt) assert np.allclose(res_np, res_rt.asnumpy()) diff --git a/testing/functions/mxfusion_function_test.py b/testing/functions/mxfusion_function_test.py index bb2061e..12429c1 100644 --- a/testing/functions/mxfusion_function_test.py +++ b/testing/functions/mxfusion_function_test.py @@ -7,7 +7,7 @@ from mxnet.initializer import Zero from mxfusion.components.functions.mxfusion_function import MXFusionFunction from mxfusion.components import Variable -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples class TestMXFusionFunctionTests(unittest.TestCase): diff --git a/testing/functions/mxfusion_gluon_function_test.py b/testing/functions/mxfusion_gluon_function_test.py index 8530637..81fe38e 100644 --- a/testing/functions/mxfusion_gluon_function_test.py +++ b/testing/functions/mxfusion_gluon_function_test.py @@ -6,7 +6,7 @@ from mxnet.initializer import Zero from mxfusion.components.functions.mxfusion_gluon_function import MXFusionGluonFunction from mxfusion.components import Variable -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples @pytest.mark.usefixtures("set_seed") @@ -69,7 +69,7 @@ def test_eval(self, dtype, A, A_isSamples, B, B_isSamples, num_samples, variables = {eval.dot_input_0.uuid: A_mx, eval.dot_input_1.uuid: B_mx} res_rt = eval.eval(F=mx.nd, variables=variables) - assert np_isSamples == is_sampled_array(mx.nd, res_rt) + assert np_isSamples == array_has_samples(mx.nd, res_rt) assert np.allclose(res_np, res_rt.asnumpy()) def _make_gluon_function_evaluation_rand_param(self, dtype, broadcastable): @@ -138,7 +138,7 @@ def test_eval_gluon_parameters(self, dtype, A, A_isSamples, B, variables = {eval.dot_input_0.uuid: A_mx, eval.dot_input_1.uuid: B_mx, eval.dot_const.uuid: C_mx} res_rt = eval.eval(F=mx.nd, variables=variables) - assert np_isSamples == is_sampled_array(mx.nd, res_rt) + assert np_isSamples == array_has_samples(mx.nd, res_rt) assert np.allclose(res_np, res_rt.asnumpy()) def test_success(self): From 96834d4986cd4d776a42391127c7419b3f759e12 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 19 Oct 2018 18:00:00 +0100 Subject: [PATCH 26/70] Fix the default dtype in InferenceParameter. Change the default mxnet device to be the default mxnet devie in mxnet setting. --- mxfusion/common/config.py | 14 ++++++++++++-- mxfusion/inference/inference_parameters.py | 4 ++-- mxfusion/inference/meanfield.py | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/mxfusion/common/config.py b/mxfusion/common/config.py index cb90f03..e58fd80 100644 --- a/mxfusion/common/config.py +++ b/mxfusion/common/config.py @@ -2,9 +2,16 @@ MXNET_DEFAULT_DTYPE = 'float32' MXNET_DEFAULT_MODE = mxnet.ndarray -MXNET_DEFAULT_DEVICE = mxnet.cpu() +MXNET_DEFAULT_DEVICE = None + def get_default_dtype(): + """ + Return the default dtype. The default dtype is float32. + + :returns: the default dtype + :rtypes: str + """ return MXNET_DEFAULT_DTYPE @@ -24,4 +31,7 @@ def get_default_device(): :returns: an MXNet cpu or gpu, indicating the default device. """ - return MXNET_DEFAULT_DEVICE + if MXNET_DEFAULT_DEVICE: + return MXNET_DEFAULT_DEVICE + else: + return mxnet.context.Context.default_ctx diff --git a/mxfusion/inference/inference_parameters.py b/mxfusion/inference/inference_parameters.py index a790d2b..8a61824 100644 --- a/mxfusion/inference/inference_parameters.py +++ b/mxfusion/inference/inference_parameters.py @@ -7,7 +7,7 @@ from ..components.variables import VariableType, Variable from ..components import ModelComponent from ..util.inference import realize_shape -from ..common.config import get_default_device +from ..common.config import get_default_device, get_default_dtype from ..components.functions.gluon_func_eval import GluonFunctionEvaluation @@ -26,7 +26,7 @@ class InferenceParameters(object): :type context: {mxnet.cpu or mxnet.gpu} """ def __init__(self, constants=None, dtype=None, context=None): - self.dtype = dtype if dtype is not None else np.float32 + self.dtype = dtype if dtype is not None else get_default_dtype() self.mxnet_context = context if context is not None else get_default_device() self._constants = {} self._var_ties = {} diff --git a/mxfusion/inference/meanfield.py b/mxfusion/inference/meanfield.py index bd70222..89bd87a 100644 --- a/mxfusion/inference/meanfield.py +++ b/mxfusion/inference/meanfield.py @@ -6,7 +6,7 @@ from ..common.config import get_default_dtype -def create_Gaussian_meanfield(model, observed, dtype): +def create_Gaussian_meanfield(model, observed, dtype=None): """ Create the Meanfield posterior for Variational Inference. From 69311da66b8344148efa2a6e092311b7d9ac85c0 Mon Sep 17 00:00:00 2001 From: Mark Pullin Date: Fri, 19 Oct 2018 18:26:41 +0100 Subject: [PATCH 27/70] Fix typos --- mxfusion/components/distributions/categorical.py | 2 +- .../components/distributions/gp/kernels/add_kernel.py | 2 +- .../components/distributions/gp/kernels/stationary.py | 4 ++-- mxfusion/components/distributions/wishart.py | 2 +- mxfusion/components/functions/mxfusion_function.py | 4 ++-- mxfusion/components/functions/mxfusion_gluon_function.py | 6 +++--- mxfusion/components/variables/variable.py | 8 ++++---- mxfusion/inference/grad_based_inference.py | 2 +- mxfusion/inference/inference.py | 5 ++--- mxfusion/inference/inference_alg.py | 6 +++--- mxfusion/models/factor_graph.py | 6 +++--- mxfusion/modules/gp_modules/gp_regression.py | 2 +- mxfusion/modules/gp_modules/sparsegp_regression.py | 2 +- mxfusion/modules/gp_modules/svgp_regression.py | 2 +- mxfusion/modules/module.py | 6 +++--- mxfusion/util/__init__.py | 2 +- mxfusion/util/util.py | 4 ++-- 17 files changed, 32 insertions(+), 33 deletions(-) diff --git a/mxfusion/components/distributions/categorical.py b/mxfusion/components/distributions/categorical.py index eb8dfe4..a0760c2 100644 --- a/mxfusion/components/distributions/categorical.py +++ b/mxfusion/components/distributions/categorical.py @@ -154,7 +154,7 @@ def log_pdf(self, log_prob, random_variable, F=None): :param F: MXNet computation type . :param log_prob: the logarithm of the probability being in each of the classes. :type log_prob: MXNet NDArray or MXNet Symbol - :param random_variable: the point to compute the logpdf for. + :param random_variable: the point to compute the log pdf for. :type random_variable: MXNet NDArray or MXNet Symbol :returns: log pdf of the distribution. :rtypes: MXNet NDArray or MXNet Symbol diff --git a/mxfusion/components/distributions/gp/kernels/add_kernel.py b/mxfusion/components/distributions/gp/kernels/add_kernel.py index a82b400..bcd8770 100644 --- a/mxfusion/components/distributions/gp/kernels/add_kernel.py +++ b/mxfusion/components/distributions/gp/kernels/add_kernel.py @@ -3,7 +3,7 @@ class AddKernel(CombinationKernel): """ - The add kernel that computes a covariance matrix by suming the covariance + The add kernel that computes a covariance matrix by summing the covariance matrices of a list of kernels. :param sub_kernels: a list of kernels that are combined to compute a covariance matrix. diff --git a/mxfusion/components/distributions/gp/kernels/stationary.py b/mxfusion/components/distributions/gp/kernels/stationary.py index 614f679..70a41d8 100644 --- a/mxfusion/components/distributions/gp/kernels/stationary.py +++ b/mxfusion/components/distributions/gp/kernels/stationary.py @@ -9,7 +9,7 @@ class StationaryKernel(NativeKernel): The base class for Stationary kernels (covariance functions). Stationary kernels (covariance functions). - Stationary covariance fucntion depend only on r^2, where r^2 is defined as + Stationary covariance function depend only on r^2, where r^2 is defined as .. math:: r2(x, x') = \\sum_{q=1}^Q (x_q - x'_q)^2 The covariance function k(x, x' can then be written k(r). @@ -17,7 +17,7 @@ class StationaryKernel(NativeKernel): In this implementation, r is scaled by the lengthscales parameter(s): .. math:: r2(x, x') = \\sum_{q=1}^Q \\frac{(x_q - x'_q)^2}{\\ell_q^2}. - By default, there's only one lengthscale: seaprate lengthscales for each dimension can be enables by setting ARD=True. + By default, there's only one lengthscale: separate lengthscales for each dimension can be enables by setting ARD=True. :param input_dim: the number of dimensions of the kernel. (The total number of active dimensions). diff --git a/mxfusion/components/distributions/wishart.py b/mxfusion/components/distributions/wishart.py index 1af4537..6abcacf 100644 --- a/mxfusion/components/distributions/wishart.py +++ b/mxfusion/components/distributions/wishart.py @@ -82,7 +82,7 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, num_samples_inferred = max([get_num_samples(F, v) for v in variables.values()]) if num_samples_inferred != num_samples: - raise InferenceError("The number of samples in the num_amples argument of draw_samples of " + raise InferenceError("The number of samples in the num_samples argument of draw_samples of " "the Wishart distribution must be the same as the number of samples " "given to the inputs. num_samples: {}, the inferred number of samples " "from inputs: {}.".format(num_samples, num_samples_inferred)) diff --git a/mxfusion/components/functions/mxfusion_function.py b/mxfusion/components/functions/mxfusion_function.py index f1676f4..fa299c4 100644 --- a/mxfusion/components/functions/mxfusion_function.py +++ b/mxfusion/components/functions/mxfusion_function.py @@ -39,7 +39,7 @@ def eval(self, F, **input_kws): def __call__(self, *args, **kwargs): """ - The evaluation of the function in a model defition. It takes a list of + The evaluation of the function in a model definition. It takes a list of arguments in the type of MXFusion Variable and returns the output variables. @@ -124,7 +124,7 @@ def _parse_arguments(self, args, kwargs): def replicate_self(self, attribute_map=None): """ - The copy constructor for the fuction. + The copy constructor for the function. """ replicant = self.__class__.__new__(self.__class__) diff --git a/mxfusion/components/functions/mxfusion_gluon_function.py b/mxfusion/components/functions/mxfusion_gluon_function.py index 75746ab..5d41577 100644 --- a/mxfusion/components/functions/mxfusion_gluon_function.py +++ b/mxfusion/components/functions/mxfusion_gluon_function.py @@ -13,7 +13,7 @@ class MXFusionGluonFunction(MXFusionFunction): wrapper is called in Model definition, it returns a factor corresponding to the function evaluation. :param block: The MXNet Gluon block to be wrapped. - :type block: mxnet.gluon.Blockk or mxnet.gluon.HybridBlock + :type block: mxnet.gluon.Block or mxnet.gluon.HybridBlock :param num_outputs: The number of output variables of the Gluon block. :type num_outputs: int :param dtype: the data type of float point numbers used in the Gluon block. @@ -170,7 +170,7 @@ def _override_block_parameters(self, input_kws): because otherwise these parameters will be directly exposed to a gradient optimizer as free parameters. For each parameters of the Gluon bock with probabilistic distribution, this method dynamically sets its values as the outcome of - upstream computation and ensure the correct gradient can be estimated via automatic differenciation. + upstream computation and ensure the correct gradient can be estimated via automatic differentiation. :param **input_kws: the dict of inputs to the functions. The key in the dict should match with the name of inputs specified in the inputs of FunctionEvaluation. @@ -193,7 +193,7 @@ def _override_block_parameters(self, input_kws): def replicate_self(self, attribute_map=None): """ - The copy constructor for the fuction. + The copy constructor for the function. """ replicant = super( MXFusionGluonFunction, self).replicate_self(attribute_map) diff --git a/mxfusion/components/variables/variable.py b/mxfusion/components/variables/variable.py index a949247..0bff0fe 100644 --- a/mxfusion/components/variables/variable.py +++ b/mxfusion/components/variables/variable.py @@ -28,7 +28,7 @@ class Variable(ModelComponent): following a probabilistic distribution. :param value: The value of variable. If it is a numpy or MXNet array, the variable is considered as a constant. If it is a function evaluation, the - varaible is considered as the outcome of a function evaluation. If it is a probabilitistic distribution, the variable is considered as a random + variable is considered as the outcome of a function evaluation. If it is a probabilistic distribution, the variable is considered as a random variable. If it is None, the variable is considered as a parameter. :type value: (optional) None or numpy array or MXNet array or float or int or FunctionEvaluation or Distribution. :param shape: The expected shape of the Variable. @@ -84,7 +84,7 @@ def replicate_self(self, attribute_map=None): Replicates this Factor, using new inputs, outputs, and a new uuid. Used during model replication to functionally replicate a factor into a new graph. - :param attribute_map: A mapping from attributes of this object that were Variables to thier replicants. + :param attribute_map: A mapping from attributes of this object that were Variables to their replicants. :type attribute_map: {Variable: replicated Variable} """ if attribute_map is not None: @@ -145,14 +145,14 @@ def _initialize_as_param(self, value, shape, transformation): def _initialize_as_randvar(self, value, shape, transformation): if transformation is not None: - raise NotImplementedError('Contraints on random variables are not supported!') + raise NotImplementedError('Constraints on random variables are not supported!') def _initialize_as_funcvar(self, value, shape, transformation): self._inputs = [value] if shape is None: raise ModelSpecificationError("The shape argument was not given when defining a variable as the outcome of a function evaluation.") if transformation is not None: - raise NotImplementedError('Contraints on function outputs are not supported!') + raise NotImplementedError('Constraints on function outputs are not supported!') def set_prior(self, distribution): """ diff --git a/mxfusion/inference/grad_based_inference.py b/mxfusion/inference/grad_based_inference.py index 39821b6..5add859 100644 --- a/mxfusion/inference/grad_based_inference.py +++ b/mxfusion/inference/grad_based_inference.py @@ -14,7 +14,7 @@ class GradBasedInference(Inference): :type graphs: [FactorGraph] :param observed: A list of observed variables :type observed: [Variable] - :param grad_loop: The reference to the main loop of gradient optmization + :param grad_loop: The reference to the main loop of gradient optimization :type grad_loop: GradLoop :param constants: Specify a list of model variables as constants :type constants: {Variable: mxnet.ndarray} diff --git a/mxfusion/inference/inference.py b/mxfusion/inference/inference.py index 79217f7..1ec3a69 100644 --- a/mxfusion/inference/inference.py +++ b/mxfusion/inference/inference.py @@ -117,10 +117,9 @@ def run(self, **kwargs): """ Run the inference method. - :param kwargs: The keyword arguments specify the data for inference. The key of each argument - is the name of the corresponding + :param **kwargs: The keyword arguments specify the data for inference self. The key of each argument is the name of the corresponding variable in model definition and the value of the argument is the data in numpy array format. - :returns: the samples of target variables (if not spcified, the samples of all the latent variables) + :returns: the samples of target variables (if not specified, the samples of all the latent variables) :rtype: {UUID: samples} """ data = [kwargs[v] for v in self.observed_variable_names] diff --git a/mxfusion/inference/inference_alg.py b/mxfusion/inference/inference_alg.py index 049aefe..43b9d25 100644 --- a/mxfusion/inference/inference_alg.py +++ b/mxfusion/inference/inference_alg.py @@ -51,7 +51,7 @@ def hybrid_forward(self, F, x, *args, **kw): :type x: MXNet NDArray or MXNet Symbol :param *arg: all the positional arguments, which correspond to the data provided to the InferenceAlgorithm. :type *arg: list of MXNet NDArray or MXNet Symbol - :parma **kw: all the keyword arguments, which correspond to the parameters that may require gradients. + :param **kw: all the keyword arguments, which correspond to the parameters that may require gradients. :type kw: {str(UUID): MXNet NDArray or MXNet Symbol} :returns: the outcome of the InferenceAlgorithm that are determined by the inference algorithm. :rtypes: {str: MXNet NDArray or MXNet Symbol} @@ -139,7 +139,7 @@ def prepare_executor(self, rv_scaling=None): :param rv_scaling: The scaling of log_pdf of the random variables that are set by users for data sub-sampling or mini-batch learning. :type rv_scaling: {UUID: float} - :returns: the list of the variable transformations and the list of the variables that are excluded from being setted as Gluon block parameters (see the excluded argument of __init__ of ObjectiveBlock). + :returns: the list of the variable transformations and the list of the variables that are excluded from being set as Gluon block parameters (see the excluded argument of __init__ of ObjectiveBlock). :rtypes: {str(UUID): Transformation}, set(str(UUID)) """ excluded = set() @@ -210,7 +210,7 @@ def set_parameter(self, variables, target_variable, target_value): :type variables: {str(UUID): MXNet NDArray or MXNet Symbol} :param target_variable: the variable that a value is set to :type target_variable: Variable - :param target_value: the value to be setted + :param target_value: the value to be set :type target_value: MXNet NDArray or float """ variables[target_variable.uuid] = target_value diff --git a/mxfusion/models/factor_graph.py b/mxfusion/models/factor_graph.py index cb34451..34a0b31 100644 --- a/mxfusion/models/factor_graph.py +++ b/mxfusion/models/factor_graph.py @@ -410,7 +410,7 @@ def get_parameters(self, excluded=None, include_inherited=False): :type excluded: set(UUID) or [UUID] :param include_inherited: whether inherited variables are included. :type include_inherited: boolean - :returns: the list of contant variables. + :returns: the list of constant variables. :rtype: [Variable] """ if include_inherited: @@ -420,7 +420,7 @@ def get_parameters(self, excluded=None, include_inherited=False): def get_constants(self): """ - Get all the contants in the factor graph. + Get all the constants in the factor graph. :returns: the list of constant variables. :rtype: [Variable] @@ -432,7 +432,7 @@ def reconcile_graphs(current_graphs, primary_previous_graph, secondary_previous_ """ Reconciles two sets of graphs, matching the model components in the previous graph to the current graph. This is primarily used when loading back a graph from a file and matching it to an existing in-memory graph in order to load the previous - graph's paramters correctly. + graph's parameters correctly. :param current_graphs: A list of the graphs we are reconciling a loaded factor graph against. This must be a fully built set of graphs generated through the model definition process. diff --git a/mxfusion/modules/gp_modules/gp_regression.py b/mxfusion/modules/gp_modules/gp_regression.py index a8ad78c..c7fcbd2 100644 --- a/mxfusion/modules/gp_modules/gp_regression.py +++ b/mxfusion/modules/gp_modules/gp_regression.py @@ -325,7 +325,7 @@ def define_variable(X, kernel, noise_var, shape=None, mean_func=None, def replicate_self(self, attribute_map=None): """ - The copy constructor for the fuction. + The copy constructor for the function. """ rep = super(GPRegression, self).replicate_self(attribute_map) diff --git a/mxfusion/modules/gp_modules/sparsegp_regression.py b/mxfusion/modules/gp_modules/sparsegp_regression.py index bc79aea..55cd48a 100644 --- a/mxfusion/modules/gp_modules/sparsegp_regression.py +++ b/mxfusion/modules/gp_modules/sparsegp_regression.py @@ -339,7 +339,7 @@ def define_variable(X, kernel, noise_var, shape=None, inducing_inputs=None, def replicate_self(self, attribute_map=None): """ - The copy constructor for the fuction. + The copy constructor for the function. """ rep = super(SparseGPRegression, self).replicate_self(attribute_map) diff --git a/mxfusion/modules/gp_modules/svgp_regression.py b/mxfusion/modules/gp_modules/svgp_regression.py index 463e63c..58486a0 100644 --- a/mxfusion/modules/gp_modules/svgp_regression.py +++ b/mxfusion/modules/gp_modules/svgp_regression.py @@ -370,7 +370,7 @@ def define_variable(X, kernel, noise_var, shape=None, inducing_inputs=None, def replicate_self(self, attribute_map=None): """ - The copy constructor for the fuction. + The copy constructor for the function. """ rep = super(SVGPRegression, self).replicate_self(attribute_map) diff --git a/mxfusion/modules/module.py b/mxfusion/modules/module.py index 1d5d674..4e3f8c6 100644 --- a/mxfusion/modules/module.py +++ b/mxfusion/modules/module.py @@ -13,7 +13,7 @@ class Module(Factor): """ The base class for a probabilistic module. - A probabilistic module is a combination of model definition and Inference algorithms. + A probabilistic module is a combination of model dentition and Inference algorithms. It acts as a factor and are defined as such during model definition, producing random variables like a plain probabilistic distribution. It differs from a plain distribution in that to compute it's log_pdf @@ -367,7 +367,7 @@ def prepare_executor(self, rv_scaling=None): :param rv_scaling: The scaling of log_pdf of the random variables that are set by users for data sub-sampling or mini-batch learning. :type rv_scaling: {UUID: float} - :returns: the list of the variable transformations and the list of the variables that are excluded from being setted as Gluon block parameters (see the excluded argument of __init__ of ObjectiveBlock). + :returns: the list of the variable transformations and the list of the variables that are excluded from being set as Gluon block parameters (see the excluded argument of __init__ of ObjectiveBlock). :rtypes: {str(UUID): Transformation}, set(str(UUID)) """ excluded = set() @@ -388,7 +388,7 @@ def prepare_executor(self, rv_scaling=None): def replicate_self(self, attribute_map=None): """ - The copy constructor for the fuction. + The copy constructor for the function. """ rep = super(Module, self).replicate_self(attribute_map) diff --git a/mxfusion/util/__init__.py b/mxfusion/util/__init__.py index de2c737..abf69ef 100644 --- a/mxfusion/util/__init__.py +++ b/mxfusion/util/__init__.py @@ -1,4 +1,4 @@ -"""This module contains utlity functions used throughout MXFusion. +"""This module contains utility functions used throughout MXFusion. Submodules ========== diff --git a/mxfusion/util/util.py b/mxfusion/util/util.py index 908e547..0b3441c 100644 --- a/mxfusion/util/util.py +++ b/mxfusion/util/util.py @@ -16,7 +16,7 @@ def slice_axis(F, array, axis, indices): :param indices: the indices used in slicing :type indices: list or MXNet Array """ - assert F == mx.nd, "The slice_axis helper funcion only works on imperative mode, because fancy indexing only exists in NDArray API." + assert F == mx.nd, "The slice_axis helper function only works on imperative mode, because fancy indexing only exists in NDArray API." if isinstance(indices, (list, tuple)): num_indices = len(indices) elif isinstance(indices, (NDArray, Symbol)): @@ -129,7 +129,7 @@ def create_variables_with_names(names): def create_constant_from_values(var): """ - Utility function to createa a constant variable from a raw value. + Utility function to create a constant variable from a raw value. :param: var the value of the constant """ From 767e06b95f13dfa1f910e4da7a29522c030173b7 Mon Sep 17 00:00:00 2001 From: Keerthana Elango Date: Fri, 19 Oct 2018 18:46:42 +0100 Subject: [PATCH 28/70] Validate shape of array variables --- mxfusion/components/variables/variable.py | 35 ++++++++++++++--------- testing/core/variable_test.py | 25 ++++++++++++++++ 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/mxfusion/components/variables/variable.py b/mxfusion/components/variables/variable.py index a949247..0acb248 100644 --- a/mxfusion/components/variables/variable.py +++ b/mxfusion/components/variables/variable.py @@ -40,10 +40,12 @@ class Variable(ModelComponent): """ def __init__(self, value=None, shape=None, transformation=None, isInherited=False, initial_value=None): super(Variable, self).__init__() - - # TODO If no shape we assume a scalar but this could be incorrect if we really just mean the shape is unknown. - self.shape = shape if shape is not None else (1,) - self.attributes = [s for s in self.shape if isinstance(s, Variable)] + self.shape = shape # For constants, if shape is None then it is inferred from the value + if self.shape is not None: + assert isinstance(self.shape, tuple), "Shape is expected to be a tuple or None" + self.attributes = [s for s in self.shape if isinstance(s, Variable)] + else: + self.attributes = [] # whether the variable is inherited from a Gluon block. self.isInherited = isInherited self._transformation = transformation @@ -56,11 +58,11 @@ def __init__(self, value=None, shape=None, transformation=None, isInherited=Fals from ...modules.module import Module from ..functions.function_evaluation import FunctionEvaluation if isinstance(value, (Distribution, Module)): - self._initialize_as_randvar(value, shape, transformation) + self._initialize_as_randvar(value, self.shape, transformation) elif isinstance(value, FunctionEvaluation): - self._initialize_as_funcvar(value, shape, transformation) + self._initialize_as_funcvar(value, self.shape, transformation) else: - self._initialize_as_param(value, shape, transformation) + self._initialize_as_param(value, self.shape, transformation) @property def type(self): @@ -128,20 +130,27 @@ def _initialize_as_param(self, value, shape, transformation): if value is None: # Initialize as VariableType.PARAMETER if shape is None: - self.shape = (1,) + shape = (1,) else: # Initialize as VariableType.CONSTANT self.isConstant = True if isinstance(value, np.ndarray): - if shape is not None and shape != value.shape: - raise ModelSpecificationError("Shape mismatch in Variable creation. The numpy array shape " + str(value.shape) + " does not no match with the shape argument " + str(shape) + ".") + if shape is None: + shape = value.shape + if shape != value.shape: + raise ModelSpecificationError("Shape mismatch in Variable creation. The numpy array shape " + str(value.shape) + " does not match with the shape argument " + str(shape) + ".") value = mx.nd.array(value) elif isinstance(value, mx.nd.NDArray): - if shape is not None and shape != value.shape: - raise ModelSpecificationError("Shape mismatch in Variable creation. The MXNet array shape " + str(value.shape) + " does not no match with the shape argument " + str(shape) + ".") + if shape is None: + shape = value.shape + if shape != value.shape: + raise ModelSpecificationError("Shape mismatch in Variable creation. The MXNet array shape " + str(value.shape) + " does not match with the shape argument " + str(shape) + ".") elif isinstance(value, (float, int)): - self.shape = (1,) + shape = (1,) + else: + raise ModelSpecificationError("Variable type {} not supported".format(type(value))) self._value = value + self.shape = shape # Update self.shape with the latest shape def _initialize_as_randvar(self, value, shape, transformation): if transformation is not None: diff --git a/testing/core/variable_test.py b/testing/core/variable_test.py index c46a9a3..5b52a91 100644 --- a/testing/core/variable_test.py +++ b/testing/core/variable_test.py @@ -1,7 +1,9 @@ import unittest import mxnet as mx +import numpy as np import mxfusion.components as mfc import mxfusion as mf +import mxfusion.common.exceptions as mf_exception class VariableTests(unittest.TestCase): @@ -40,3 +42,26 @@ def func(component): self.assertTrue(x2.uuid == m.x.uuid) self.assertTrue(x2.shape == m.x.shape, (x2.shape, m.x.shape)) self.assertTrue(y in m) + + def test_array_variable_shape(self): + mxnet_array_shape = (3, 2) + numpy_array_shape = (10, ) + mxnet_array = mx.nd.zeros(shape=mxnet_array_shape) + numpy_array = np.zeros(shape=numpy_array_shape) + + # Test Case 1: Shape param not explicitly passed to Variable class + variable = mf.Variable(value=mxnet_array) + self.assertTrue(variable.shape == mxnet_array_shape) + variable = mf.Variable(value=numpy_array) + self.assertTrue(variable.shape == numpy_array_shape) + + # Test Case 2: Correct shape passed to Variable class + variable = mf.Variable(value=mxnet_array, shape=mxnet_array_shape) + self.assertTrue(variable.shape == mxnet_array_shape) + variable = mf.Variable(value=numpy_array, shape=numpy_array_shape) + self.assertTrue(variable.shape == numpy_array_shape) + + # Test Case 3: Incorrect shape passed to Variable class + incorrect_shape = (1234, 1234) + self.assertRaises(mf_exception.ModelSpecificationError, mf.Variable, value=mxnet_array, shape=incorrect_shape) + self.assertRaises(mf_exception.ModelSpecificationError, mf.Variable, value=numpy_array, shape=incorrect_shape) From 64136ca53d7d1625fdfe0632a6a120b86a30e901 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Mon, 22 Oct 2018 11:15:49 +0100 Subject: [PATCH 29/70] Improve variable shape debug message --- mxfusion/util/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mxfusion/util/inference.py b/mxfusion/util/inference.py index 3096c77..aa8a6b7 100644 --- a/mxfusion/util/inference.py +++ b/mxfusion/util/inference.py @@ -39,7 +39,7 @@ def discover_shape_constants(data_shapes, graphs): for s1, s2 in zip(def_shape, shape): if isinstance(s1, int): if s1 != s2: - raise ModelSpecificationError("Variable shape mismatch! s1 : {} s2 : {}".format(str(s1), str(s2))) + raise ModelSpecificationError("Variable ({}) shape mismatch between expected and found! s1 : {} s2 : {}".format(str(variables[var_id]),str(s1), str(s2))) elif isinstance(s1, Variable): shape_constants[s1] = s2 else: From 14b9c99278f38305ed07a8904ec3b884c9cdb405 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Mon, 22 Oct 2018 13:01:46 +0100 Subject: [PATCH 30/70] Add Expectation Inference algorithms. --- mxfusion/inference/__init__.py | 1 + mxfusion/inference/expectation.py | 93 +++++++++++++++++++++++++++ testing/inference/expectation_test.py | 49 ++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 mxfusion/inference/expectation.py create mode 100644 testing/inference/expectation_test.py diff --git a/mxfusion/inference/__init__.py b/mxfusion/inference/__init__.py index 63c089d..5a04002 100644 --- a/mxfusion/inference/__init__.py +++ b/mxfusion/inference/__init__.py @@ -30,4 +30,5 @@ from .variational import StochasticVariationalInference from .inference_parameters import InferenceParameters from .score_function import ScoreFunctionInference, ScoreFunctionRBInference +from .expectation import ExpectationAlgorithm, ExpectationScoreFunctionAlgorithm from .prediction import ModulePredictionAlgorithm diff --git a/mxfusion/inference/expectation.py b/mxfusion/inference/expectation.py new file mode 100644 index 0000000..4dce67e --- /dev/null +++ b/mxfusion/inference/expectation.py @@ -0,0 +1,93 @@ +from ..common.exceptions import InferenceError +from ..components.variables import Variable, VariableType +from .variational import StochasticVariationalInference +from .inference_alg import SamplingAlgorithm +from .inference import TransferInference +from .map import MAP +from ..components.variables.runtime_variable import expectation + + +class ExpectationAlgorithm(SamplingAlgorithm): + """ + Sampling inference algorithm that returns the expectation of each variable in the model. + + :param model: the definition of the probabilistic model + :type model: Model + :param observed: A list of observed variables + :type observed: [Variable] + :param num_samples: the number of samples used in estimating the variational lower bound + :type num_samples: int + :param target_variables: (optional) the target variables to sample + :type target_variables: [UUID] + :param extra_graphs: a list of extra FactorGraph used in the inference + algorithm. + :type extra_graphs: [FactorGraph] + """ + def compute(self, F, variables): + """ + Compute the inference algorithm + + :param F: the execution context (mxnet.ndarray or mxnet.symbol) + :type F: Python module + :param variables: the set of MXNet arrays that holds the values of + variables at runtime. + :type variables: {str(UUID): MXNet NDArray or MXNet Symbol} + :returns: the outcome of the inference algorithm + :rtype: mxnet.ndarray.ndarray.NDArray or mxnet.symbol.symbol.Symbol + """ + samples = self.model.draw_samples( + F=F, variables=variables, + num_samples=self.num_samples) + samples = {k: expectation(F,v) for k, v in samples.items()} + + if self.target_variables: + return tuple(samples[v] for v in self.target_variables) + else: + return samples + + +class ExpectationScoreFunctionAlgorithm(SamplingAlgorithm): + """ + Sampling inference algorithm that computes the expectation of the model w.r.t. some loss function in that model, specified as the target variable. It does so via the score function trick sampling the necessary inputs to the function and using them to compute a Monte Carlo estimate of the loss function's gradient. + + :param model: the definition of the probabilistic model + :type model: Model + :param observed: A list of observed variables + :type observed: [Variable] + :param num_samples: the number of samples used in estimating the variational lower bound + :type num_samples: int + :param target_variables: the target function in the model to optimize. should only be one for this. + :type target_variables: [UUID] + :param extra_graphs: a list of extra FactorGraph used in the inference + algorithm. + :type extra_graphs: [FactorGraph] + """ + def compute(self, F, variables): + """ + Compute the inference algorithm + + :param F: the execution context (mxnet.ndarray or mxnet.symbol) + :type F: Python module + :param variables: the set of MXNet arrays that holds the values of + variables at runtime. + :type variables: {str(UUID): MXNet NDArray or MXNet Symbol} + :returns: the outcome of the inference algorithm + :rtype: mxnet.ndarray.ndarray.NDArray or mxnet.symbol.symbol.Symbol + """ + samples = self.model.draw_samples( + F=F, variables=variables, + num_samples=self.num_samples) + variables.update(samples) + targets = [v for v in self.model.get_latent_variables(self.observed_variables) if v.type == VariableType.RANDVAR] + + q_z_lambda = self.model.log_pdf(F=F, variables=variables, targets=targets) + + p_x_z = variables[self.target_variables[0]] + + gradient_lambda = F.mean(q_z_lambda * F.stop_gradient(p_x_z), axis=0) + + gradient_theta = F.mean(p_x_z, axis=0) + + gradient_log_L = gradient_lambda + gradient_theta + + return gradient_theta, gradient_log_L diff --git a/testing/inference/expectation_test.py b/testing/inference/expectation_test.py new file mode 100644 index 0000000..84d99b3 --- /dev/null +++ b/testing/inference/expectation_test.py @@ -0,0 +1,49 @@ +import mxnet as mx +import numpy as np +import pytest +import mxfusion as mf +from mxfusion import Model, Variable +from mxfusion.inference import GradBasedInference, TransferInference, ExpectationScoreFunctionAlgorithm, ExpectationAlgorithm + + +@pytest.mark.usefixtures("set_seed") +class TestExpectationInference(object): + """ + Test class that tests the MXFusion.inference.expectation classes. + """ + + def make_model(self): + class Func(mx.gluon.HybridBlock): + def hybrid_forward(self, F, v2, v3, v4, v1): + return - (F.sum(v2 * F.minimum(v4, v1) - v3 * v1)) + + m = Model() + N = 1 + m.v1 = Variable(shape=(N,)) + m.v2 = Variable(shape=(N,)) + m.v3 = Variable(shape=(N,)) + m.v4 = mf.components.distributions.Gamma.define_variable(alpha=mx.nd.array([1]), + beta=mx.nd.array([0.1]), + shape=(N,)) + v5 = mf.components.functions.MXFusionGluonFunction(Func(), num_outputs=1) + m.v5 = v5(m.v2, m.v3, m.v4, m.v1) + return m + + @pytest.mark.parametrize("v2, v3", [ + (mx.nd.random.uniform(1,100) * 2, mx.nd.random.uniform(1,100) * 0.5), + ]) + def test_score_function_gradient(self, v2, v3): + + m = self.make_model() + observed = [m.v2, m.v3] + target_variables = [m.v5] + + infr = GradBasedInference( + ExpectationScoreFunctionAlgorithm(m, observed, num_samples=10, target_variables=target_variables)) + + infr.run(max_iter=1, v2=v2, v3=v3, verbose=True) + + infr2 = TransferInference( + ExpectationAlgorithm(m, observed, num_samples=10, target_variables=target_variables), infr_params=infr.params) + + infr2.run(max_iter=1, v2=v2, v3=v3, verbose=True) From 2ed2e7b4c94339cdb697f1c00fac9809eef969df Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Mon, 22 Oct 2018 13:44:25 +0100 Subject: [PATCH 31/70] Add TODOs around limitations and test inconsistency. --- mxfusion/inference/expectation.py | 2 +- testing/inference/expectation_test.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mxfusion/inference/expectation.py b/mxfusion/inference/expectation.py index 4dce67e..5f346ed 100644 --- a/mxfusion/inference/expectation.py +++ b/mxfusion/inference/expectation.py @@ -86,7 +86,7 @@ def compute(self, F, variables): gradient_lambda = F.mean(q_z_lambda * F.stop_gradient(p_x_z), axis=0) - gradient_theta = F.mean(p_x_z, axis=0) + gradient_theta = F.mean(p_x_z, axis=0) # TODO known issue. This will double count the gradient of any distribution using the reparameterization trick (i.e. Normal). Issue #91 gradient_log_L = gradient_lambda + gradient_theta diff --git a/testing/inference/expectation_test.py b/testing/inference/expectation_test.py index 84d99b3..0064083 100644 --- a/testing/inference/expectation_test.py +++ b/testing/inference/expectation_test.py @@ -32,7 +32,8 @@ def hybrid_forward(self, F, v2, v3, v4, v1): @pytest.mark.parametrize("v2, v3", [ (mx.nd.random.uniform(1,100) * 2, mx.nd.random.uniform(1,100) * 0.5), ]) - def test_score_function_gradient(self, v2, v3): + def test_inference_basic_run(self, v2, v3): + # TODO test correctness m = self.make_model() observed = [m.v2, m.v3] From 5823171aa6dc68ba4223dedbd333fc16fd2cf09a Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 22 Oct 2018 13:45:11 +0100 Subject: [PATCH 32/70] Update mxfusion/inference/expectation.py Co-Authored-By: meissnereric --- mxfusion/inference/expectation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mxfusion/inference/expectation.py b/mxfusion/inference/expectation.py index 5f346ed..5da5c72 100644 --- a/mxfusion/inference/expectation.py +++ b/mxfusion/inference/expectation.py @@ -48,7 +48,7 @@ def compute(self, F, variables): class ExpectationScoreFunctionAlgorithm(SamplingAlgorithm): """ - Sampling inference algorithm that computes the expectation of the model w.r.t. some loss function in that model, specified as the target variable. It does so via the score function trick sampling the necessary inputs to the function and using them to compute a Monte Carlo estimate of the loss function's gradient. + Sampling-based inference algorithm that computes the expectation of the model w.r.t. some loss function in that model, specified as the target variable. It does so via the score function trick sampling the necessary inputs to the function and using them to compute a Monte Carlo estimate of the loss function's gradient. :param model: the definition of the probabilistic model :type model: Model From bf60f43429df88e52cad00de105bdb4f5b0077de Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 22 Oct 2018 13:45:18 +0100 Subject: [PATCH 33/70] Update mxfusion/inference/expectation.py Co-Authored-By: meissnereric --- mxfusion/inference/expectation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mxfusion/inference/expectation.py b/mxfusion/inference/expectation.py index 5da5c72..c61f2f5 100644 --- a/mxfusion/inference/expectation.py +++ b/mxfusion/inference/expectation.py @@ -9,7 +9,7 @@ class ExpectationAlgorithm(SamplingAlgorithm): """ - Sampling inference algorithm that returns the expectation of each variable in the model. + Sampling-based inference algorithm that returns the expectation of each variable in the model. :param model: the definition of the probabilistic model :type model: Model From 293253c2376637637363195d09242e920a9df372 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Thu, 11 Oct 2018 17:06:33 +0100 Subject: [PATCH 34/70] Refactor testing folder structure to match source folder. --- testing/{ => components}/distributions/categorical_test.py | 0 testing/{ => components}/distributions/gp/cond_gp_test.py | 0 testing/{ => components}/distributions/gp/gp_test.py | 0 testing/{ => components}/distributions/gp/kernel_test.py | 0 testing/{ => components}/distributions/normal_test.py | 0 testing/{core => components}/factor_test.py | 0 testing/{ => components}/functions/function_evaluation_test.py | 0 .../{ => components}/functions/mxfusion_gluon_function_test.py | 0 testing/{core => components}/model_component_test.py | 0 testing/{core => components/variables}/var_trans_test.py | 0 testing/{core => components/variables}/variable_test.py | 0 testing/{core => models}/factor_graph_test.py | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename testing/{ => components}/distributions/categorical_test.py (100%) rename testing/{ => components}/distributions/gp/cond_gp_test.py (100%) rename testing/{ => components}/distributions/gp/gp_test.py (100%) rename testing/{ => components}/distributions/gp/kernel_test.py (100%) rename testing/{ => components}/distributions/normal_test.py (100%) rename testing/{core => components}/factor_test.py (100%) rename testing/{ => components}/functions/function_evaluation_test.py (100%) rename testing/{ => components}/functions/mxfusion_gluon_function_test.py (100%) rename testing/{core => components}/model_component_test.py (100%) rename testing/{core => components/variables}/var_trans_test.py (100%) rename testing/{core => components/variables}/variable_test.py (100%) rename testing/{core => models}/factor_graph_test.py (100%) diff --git a/testing/distributions/categorical_test.py b/testing/components/distributions/categorical_test.py similarity index 100% rename from testing/distributions/categorical_test.py rename to testing/components/distributions/categorical_test.py diff --git a/testing/distributions/gp/cond_gp_test.py b/testing/components/distributions/gp/cond_gp_test.py similarity index 100% rename from testing/distributions/gp/cond_gp_test.py rename to testing/components/distributions/gp/cond_gp_test.py diff --git a/testing/distributions/gp/gp_test.py b/testing/components/distributions/gp/gp_test.py similarity index 100% rename from testing/distributions/gp/gp_test.py rename to testing/components/distributions/gp/gp_test.py diff --git a/testing/distributions/gp/kernel_test.py b/testing/components/distributions/gp/kernel_test.py similarity index 100% rename from testing/distributions/gp/kernel_test.py rename to testing/components/distributions/gp/kernel_test.py diff --git a/testing/distributions/normal_test.py b/testing/components/distributions/normal_test.py similarity index 100% rename from testing/distributions/normal_test.py rename to testing/components/distributions/normal_test.py diff --git a/testing/core/factor_test.py b/testing/components/factor_test.py similarity index 100% rename from testing/core/factor_test.py rename to testing/components/factor_test.py diff --git a/testing/functions/function_evaluation_test.py b/testing/components/functions/function_evaluation_test.py similarity index 100% rename from testing/functions/function_evaluation_test.py rename to testing/components/functions/function_evaluation_test.py diff --git a/testing/functions/mxfusion_gluon_function_test.py b/testing/components/functions/mxfusion_gluon_function_test.py similarity index 100% rename from testing/functions/mxfusion_gluon_function_test.py rename to testing/components/functions/mxfusion_gluon_function_test.py diff --git a/testing/core/model_component_test.py b/testing/components/model_component_test.py similarity index 100% rename from testing/core/model_component_test.py rename to testing/components/model_component_test.py diff --git a/testing/core/var_trans_test.py b/testing/components/variables/var_trans_test.py similarity index 100% rename from testing/core/var_trans_test.py rename to testing/components/variables/var_trans_test.py diff --git a/testing/core/variable_test.py b/testing/components/variables/variable_test.py similarity index 100% rename from testing/core/variable_test.py rename to testing/components/variables/variable_test.py diff --git a/testing/core/factor_graph_test.py b/testing/models/factor_graph_test.py similarity index 100% rename from testing/core/factor_graph_test.py rename to testing/models/factor_graph_test.py From 08016c3d69d2f5b83b4296e02af5a1356f8e1098 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Mon, 22 Oct 2018 14:01:48 +0100 Subject: [PATCH 35/70] Merge latest develop and move recent tests. --- testing/{ => components}/distributions/beta_test.py | 0 testing/{ => components}/distributions/gamma_test.py | 0 testing/{ => components}/distributions/wishart_test.py | 0 testing/{ => components}/functions/mxfusion_function_test.py | 0 testing/{ => components}/functions/operators_test.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename testing/{ => components}/distributions/beta_test.py (100%) rename testing/{ => components}/distributions/gamma_test.py (100%) rename testing/{ => components}/distributions/wishart_test.py (100%) rename testing/{ => components}/functions/mxfusion_function_test.py (100%) rename testing/{ => components}/functions/operators_test.py (100%) diff --git a/testing/distributions/beta_test.py b/testing/components/distributions/beta_test.py similarity index 100% rename from testing/distributions/beta_test.py rename to testing/components/distributions/beta_test.py diff --git a/testing/distributions/gamma_test.py b/testing/components/distributions/gamma_test.py similarity index 100% rename from testing/distributions/gamma_test.py rename to testing/components/distributions/gamma_test.py diff --git a/testing/distributions/wishart_test.py b/testing/components/distributions/wishart_test.py similarity index 100% rename from testing/distributions/wishart_test.py rename to testing/components/distributions/wishart_test.py diff --git a/testing/functions/mxfusion_function_test.py b/testing/components/functions/mxfusion_function_test.py similarity index 100% rename from testing/functions/mxfusion_function_test.py rename to testing/components/functions/mxfusion_function_test.py diff --git a/testing/functions/operators_test.py b/testing/components/functions/operators_test.py similarity index 100% rename from testing/functions/operators_test.py rename to testing/components/functions/operators_test.py From 2d41981c79ac25336c19c1cdf91fea5d3ac755b5 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 22 Oct 2018 14:10:03 +0100 Subject: [PATCH 36/70] Fix the bug when setting the default dtype to float64. --- mxfusion/components/variables/variable.py | 4 +- mxfusion/inference/inference.py | 4 +- testing/inference/forward_sampling_test.py | 20 ++++++---- .../inference/inference_serialization_test.py | 19 +++++---- testing/inference/map_test.py | 23 ++++++----- testing/inference/meanfield_test.py | 25 +++++++----- testing/inference/score_function_test.py | 40 ++++++++++++------- 7 files changed, 83 insertions(+), 52 deletions(-) diff --git a/mxfusion/components/variables/variable.py b/mxfusion/components/variables/variable.py index a949247..9076a75 100644 --- a/mxfusion/components/variables/variable.py +++ b/mxfusion/components/variables/variable.py @@ -3,6 +3,7 @@ import numpy as np from ...common.exceptions import ModelSpecificationError from ..model_component import ModelComponent +from ...common.config import get_default_dtype class VariableType(Enum): @@ -49,7 +50,8 @@ def __init__(self, value=None, shape=None, transformation=None, isInherited=Fals self._transformation = transformation self._value = None if isinstance(initial_value, (int, float)): - initial_value = mx.nd.array([initial_value]) + initial_value = mx.nd.array([initial_value], + dtype=get_default_dtype()) self._initial_value = initial_value self.isConstant = False from ..distributions import Distribution diff --git a/mxfusion/inference/inference.py b/mxfusion/inference/inference.py index 79217f7..90901e2 100644 --- a/mxfusion/inference/inference.py +++ b/mxfusion/inference/inference.py @@ -2,7 +2,7 @@ import numpy as np import mxnet as mx from .inference_parameters import InferenceParameters -from ..common.config import get_default_device +from ..common.config import get_default_device, get_default_dtype from ..common.exceptions import InferenceError from ..util.inference import discover_shape_constants, init_outcomes from ..models.factor_graph import FactorGraph @@ -30,7 +30,7 @@ class Inference(object): def __init__(self, inference_algorithm, constants=None, hybridize=False, dtype=None, context=None): - self.dtype = dtype if dtype is not None else np.float32 + self.dtype = dtype if dtype is not None else get_default_dtype() self.mxnet_context = context if context is not None else get_default_device() self._hybridize = hybridize self._graphs = inference_algorithm.graphs diff --git a/testing/inference/forward_sampling_test.py b/testing/inference/forward_sampling_test.py index 3d66145..87ec64e 100644 --- a/testing/inference/forward_sampling_test.py +++ b/testing/inference/forward_sampling_test.py @@ -5,14 +5,16 @@ import mxfusion as mf from mxfusion.inference.forward_sampling import VariationalPosteriorForwardSampling from mxfusion.components.functions import MXFusionGluonFunction +from mxfusion.common.config import get_default_dtype -class InferenceTests(unittest.TestCase): +class ForwardSamplingTests(unittest.TestCase): """ Test class that tests the MXFusion.utils methods. """ def make_model(self, net): + dtype = get_default_dtype() m = mf.models.Model(verbose=False) m.N = mf.components.Variable() m.f = MXFusionGluonFunction(net, num_outputs=1) @@ -20,25 +22,27 @@ def make_model(self, net): m.r = m.f(m.x) for k, v in m.r.factor.parameters.items(): if k.endswith('_weight') or k.endswith('_bias'): - v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0]), variance=mx.nd.array([1e6]))) - m.y = mf.components.distributions.Categorical.define_variable(log_prob=m.r, num_classes=2, normalization=True, one_hot_encoding=False, shape=(m.N, 1)) + v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0], dtype=dtype), variance=mx.nd.array([1e6], dtype=dtype))) + m.y = mf.components.distributions.Categorical.define_variable(log_prob=m.r, num_classes=2, normalization=True, one_hot_encoding=False, shape=(m.N, 1), dtype=dtype) return m def make_net(self): D = 100 + dtype = get_default_dtype() net = nn.HybridSequential(prefix='hybrid0_') with net.name_scope(): - net.add(nn.Dense(D, activation="tanh")) - net.add(nn.Dense(D, activation="tanh")) - net.add(nn.Dense(2, flatten=True)) + net.add(nn.Dense(D, activation="tanh", dtype=dtype)) + net.add(nn.Dense(D, activation="tanh", dtype=dtype)) + net.add(nn.Dense(2, flatten=True, dtype=dtype)) net.initialize(mx.init.Xavier(magnitude=3)) return net def test_forward_sampling(self): + dtype = get_default_dtype() x = np.random.rand(1000, 1) - y = np.random.rand(1000, 1)>0.5 - x_nd, y_nd = mx.nd.array(y), mx.nd.array(x) + y = np.random.rand(1000, 1) > 0.5 + x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) diff --git a/testing/inference/inference_serialization_test.py b/testing/inference/inference_serialization_test.py index 355f084..2415fe3 100644 --- a/testing/inference/inference_serialization_test.py +++ b/testing/inference/inference_serialization_test.py @@ -6,6 +6,7 @@ import mxfusion as mf from mxfusion.components.variables.var_trans import PositiveTransformation from mxfusion.components.functions import MXFusionGluonFunction +from mxfusion.common.config import get_default_dtype class InferenceSerializationTests(unittest.TestCase): @@ -22,33 +23,36 @@ def setUp(self): self.PREFIX = 'test_' + str(uuid.uuid4()) def make_model(self, net): + dtype = get_default_dtype() m = mf.models.Model(verbose=True) m.N = mf.components.Variable() m.f = MXFusionGluonFunction(net, num_outputs=1) m.x = mf.components.Variable(shape=(m.N,1)) - m.v = mf.components.Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=mx.nd.array([0.01])) + m.v = mf.components.Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=0.01) m.prior_variance = mf.components.Variable(shape=(1,), transformation=PositiveTransformation()) m.r = m.f(m.x) for _, v in m.r.factor.parameters.items(): - v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0]),variance=m.prior_variance)) + v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0], dtype=dtype), variance=m.prior_variance)) m.y = mf.components.distributions.Normal.define_variable(mean=m.r, variance=m.v, shape=(m.N,1)) return m def make_net(self): D = 100 + dtype = get_default_dtype() net = nn.HybridSequential(prefix='hybrid0_') with net.name_scope(): - net.add(nn.Dense(D, activation="tanh")) - net.add(nn.Dense(D, activation="tanh")) - net.add(nn.Dense(1, flatten=True)) + net.add(nn.Dense(D, activation="tanh", dtype=dtype)) + net.add(nn.Dense(D, activation="tanh", dtype=dtype)) + net.add(nn.Dense(1, flatten=True, dtype=dtype)) net.initialize(mx.init.Xavier(magnitude=3)) return net def test_meanfield_saving(self): + dtype = get_default_dtype() x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) - x_nd, y_nd = mx.nd.array(y), mx.nd.array(x) + x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) @@ -71,6 +75,7 @@ def test_meanfield_saving(self): self.remove_saved_files(self.PREFIX) def test_meanfield_save_and_load(self): + dtype = get_default_dtype() from mxfusion.inference.meanfield import create_Gaussian_meanfield from mxfusion.inference import StochasticVariationalInference from mxfusion.inference.grad_based_inference import GradBasedInference @@ -78,7 +83,7 @@ def test_meanfield_save_and_load(self): x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) - x_nd, y_nd = mx.nd.array(y), mx.nd.array(x) + x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) net = self.make_net() net(x_nd) diff --git a/testing/inference/map_test.py b/testing/inference/map_test.py index f291349..53c9d36 100644 --- a/testing/inference/map_test.py +++ b/testing/inference/map_test.py @@ -6,6 +6,7 @@ from mxfusion.inference.map import MAP from mxfusion.components.variables.var_trans import PositiveTransformation from mxfusion.inference import VariationalPosteriorForwardSampling, GradBasedInference +from mxfusion.common.config import get_default_dtype class MAPTests(unittest.TestCase): @@ -14,11 +15,12 @@ class MAPTests(unittest.TestCase): """ def setUp(self): + dtype = get_default_dtype() self.D = 10 self.net = nn.HybridSequential() with self.net.name_scope(): - self.net.add(nn.Dense(self.D, activation="relu")) - self.net.add(nn.Dense(1, activation="relu")) + self.net.add(nn.Dense(self.D, activation="relu", dtype=dtype)) + self.net.add(nn.Dense(1, activation="relu", dtype=dtype)) self.net.initialize() from mxnet.gluon import HybridBlock @@ -32,7 +34,7 @@ def hybrid_forward(self, F, x, *args, **kwargs): m.var = mf.components.Variable(transformation=PositiveTransformation()) m.N = mf.components.Variable() m.x = mf.components.distributions.Normal.define_variable(mean=m.mean, variance=m.var, shape=(m.N,)) - m.y = mf.components.distributions.Normal.define_variable(mean=m.x, variance=mx.nd.array([1]), shape=(m.N,)) + m.y = mf.components.distributions.Normal.define_variable(mean=m.x, variance=mx.nd.array([1], dtype=dtype), shape=(m.N,)) self.m = m q = mf.models.posterior.Posterior(m) @@ -46,7 +48,7 @@ def hybrid_forward(self, F, x, *args, **kwargs): m.var = mf.components.Variable(transformation=PositiveTransformation()) m.N = mf.components.Variable() m.x = mf.components.distributions.Normal.define_variable(mean=m.mean, variance=m.var, shape=(m.N,)) - m.y = mf.components.distributions.Normal.define_variable(mean=m.x, variance=mx.nd.array([1]), shape=(m.N,)) + m.y = mf.components.distributions.Normal.define_variable(mean=m.x, variance=mx.nd.array([1], dtype=dtype), shape=(m.N,)) self.m2 = m q = mf.models.posterior.Posterior(m) @@ -61,10 +63,11 @@ def test_one_map_example(self): from mxfusion.inference.map import MAP from mxfusion.inference.grad_based_inference import GradBasedInference from mxfusion.inference import BatchInferenceLoop + dtype = get_default_dtype() observed = [self.m.y] alg = MAP(model=self.m, observed=observed) infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop()) - infr.run(y=mx.nd.array(np.random.rand(10)), max_iter=10) + infr.run(y=mx.nd.array(np.random.rand(10), dtype=dtype), max_iter=10) def test_function_map_example(self): """ @@ -73,20 +76,22 @@ def test_function_map_example(self): from mxfusion.inference.map import MAP from mxfusion.inference.grad_based_inference import GradBasedInference from mxfusion.inference import BatchInferenceLoop + dtype = get_default_dtype() observed = [self.m.y, self.m.x] alg = MAP(model=self.m, observed=observed) infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop()) - infr.run(y=mx.nd.array(np.random.rand(self.D)), x=mx.nd.array(np.random.rand(self.D)), max_iter=10) + infr.run(y=mx.nd.array(np.random.rand(self.D), dtype=dtype), x=mx.nd.array(np.random.rand(self.D), dtype=dtype), max_iter=10) def test_inference_outcome_passing_success(self): + dtype = get_default_dtype() observed = [self.m.y, self.m.x] alg = MAP(model=self.m, observed=observed) infr = GradBasedInference(inference_algorithm=alg) - infr.run(y=mx.nd.array(np.random.rand(self.D)), - x=mx.nd.array(np.random.rand(self.D)), max_iter=1) + infr.run(y=mx.nd.array(np.random.rand(self.D), dtype=dtype), + x=mx.nd.array(np.random.rand(self.D), dtype=dtype), max_iter=1) infr2 = VariationalPosteriorForwardSampling(10, [self.m.x], infr, [self.m.y]) - infr2.run(x=mx.nd.array(np.random.rand(self.D))) + infr2.run(x=mx.nd.array(np.random.rand(self.D), dtype=dtype)) # infr2 = mf.inference.MAPInference(model_graph=self.m2, post_graph=self.q2, observed=[self.m2.y, self.m2.x], hybridize=False) # infr2.run(y=mx.nd.array(np.random.rand(1)), diff --git a/testing/inference/meanfield_test.py b/testing/inference/meanfield_test.py index 675fe2c..91e17fb 100644 --- a/testing/inference/meanfield_test.py +++ b/testing/inference/meanfield_test.py @@ -6,41 +6,45 @@ from mxfusion.components.variables.var_trans import PositiveTransformation from mxfusion.components.functions import MXFusionGluonFunction from mxfusion.util.testutils import make_basic_model +from mxfusion.common.config import get_default_dtype -class InferenceTests(unittest.TestCase): +class MeanFieldInferenceTests(unittest.TestCase): """ Test class that tests the MXFusion.utils methods. """ def make_model(self, net): + dtype = get_default_dtype() m = mf.models.Model(verbose=True) m.N = mf.components.Variable() m.f = MXFusionGluonFunction(net, num_outputs=1) - m.x = mf.components.Variable(shape=(m.N,1)) - m.v = mf.components.Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=mx.nd.array([0.01])) + m.x = mf.components.Variable(shape=(m.N, 1)) + m.v = mf.components.Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=0.01) m.prior_variance = mf.components.Variable(shape=(1,), transformation=PositiveTransformation()) m.r = m.f(m.x) for _, v in m.r.factor.parameters.items(): - v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0]),variance=m.prior_variance)) - m.y = mf.components.distributions.Normal.define_variable(mean=m.r, variance=m.v, shape=(m.N,1)) + v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0], dtype=dtype), variance=m.prior_variance)) + m.y = mf.components.distributions.Normal.define_variable(mean=m.r, variance=m.v, shape=(m.N, 1)) return m def make_net(self): + dtype = get_default_dtype() D = 100 net = nn.HybridSequential(prefix='hybrid0_') with net.name_scope(): - net.add(nn.Dense(D, activation="tanh")) - net.add(nn.Dense(D, activation="tanh")) - net.add(nn.Dense(1, flatten=True)) + net.add(nn.Dense(D, activation="tanh", dtype=dtype)) + net.add(nn.Dense(D, activation="tanh", dtype=dtype)) + net.add(nn.Dense(1, dtype=dtype)) net.initialize(mx.init.Xavier(magnitude=3)) return net def test_meanfield_batch(self): + dtype = get_default_dtype() x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) - x_nd, y_nd = mx.nd.array(y), mx.nd.array(x) + x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) @@ -59,9 +63,10 @@ def test_meanfield_batch(self): infr.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd) def test_meanfield_minibatch(self): + dtype = get_default_dtype() x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) - x_nd, y_nd = mx.nd.array(y), mx.nd.array(x) + x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) diff --git a/testing/inference/score_function_test.py b/testing/inference/score_function_test.py index d4eea4f..b34db4e 100644 --- a/testing/inference/score_function_test.py +++ b/testing/inference/score_function_test.py @@ -8,6 +8,7 @@ from mxfusion.components.functions import MXFusionGluonFunction from mxfusion.util.testutils import make_basic_model from mxfusion.inference import ScoreFunctionInference, ScoreFunctionRBInference, StochasticVariationalInference +from mxfusion.common.config import get_default_dtype @pytest.mark.usefixtures("set_seed") @@ -17,25 +18,27 @@ class TestScoreFunction(object): """ def make_bnn_model(self, net): + dtype = get_default_dtype() m = mf.models.Model(verbose=True) m.N = mf.components.Variable() m.f = MXFusionGluonFunction(net, num_outputs=1) m.x = mf.components.Variable(shape=(m.N,1)) - m.v = mf.components.Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=mx.nd.array([0.01])) + m.v = mf.components.Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=0.01) m.prior_variance = mf.components.Variable(shape=(1,), transformation=PositiveTransformation()) m.r = m.f(m.x) for _, v in m.r.factor.parameters.items(): - v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0]),variance=m.prior_variance)) + v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0], dtype=dtype),variance=m.prior_variance)) m.y = mf.components.distributions.Normal.define_variable(mean=m.r, variance=m.v, shape=(m.N,1)) return m def make_net(self): + dtype = get_default_dtype() D = 100 net = nn.HybridSequential(prefix='hybrid0_') with net.name_scope(): - net.add(nn.Dense(D, activation="tanh")) - net.add(nn.Dense(D, activation="tanh")) - net.add(nn.Dense(1, flatten=True)) + net.add(nn.Dense(D, activation="tanh", dtype=dtype)) + net.add(nn.Dense(D, activation="tanh", dtype=dtype)) + net.add(nn.Dense(1, flatten=True, dtype=dtype)) net.initialize(mx.init.Xavier(magnitude=3)) return net @@ -54,31 +57,34 @@ def log_spiral(a,b,t): return x_train def make_ppca_model(self): + dtype = get_default_dtype() m = Model() m.w = Variable(shape=(self.K,self.D), initial_value=mx.nd.array(np.random.randn(self.K,self.D))) dot = nn.HybridLambda(function='dot') m.dot = mf.functions.MXFusionGluonFunction(dot, num_outputs=1, broadcastable=False) - cov = mx.nd.broadcast_to(mx.nd.expand_dims(mx.nd.array(np.eye(self.K,self.K)), 0),shape=(self.N,self.K,self.K)) - m.z = mf.distributions.MultivariateNormal.define_variable(mean=mx.nd.zeros(shape=(self.N,self.K)), covariance=cov, shape=(self.N,self.K)) + cov = mx.nd.broadcast_to(mx.nd.expand_dims(mx.nd.array(np.eye(self.K,self.K), dtype=dtype), 0),shape=(self.N,self.K,self.K)) + m.z = mf.distributions.MultivariateNormal.define_variable(mean=mx.nd.zeros(shape=(self.N,self.K), dtype=dtype), covariance=cov, shape=(self.N,self.K)) sigma_2 = Variable(shape=(1,), transformation=PositiveTransformation()) m.x = mf.distributions.Normal.define_variable(mean=m.dot(m.z, m.w), variance=sigma_2, shape=(self.N,self.D)) return m def make_ppca_post(self, m): from mxfusion.inference import BatchInferenceLoop, GradBasedInference + dtype = get_default_dtype() class SymmetricMatrix(mx.gluon.HybridBlock): def hybrid_forward(self, F, x, *args, **kwargs): return F.sum((F.expand_dims(x, 3)*F.expand_dims(x, 2)), axis=-3) q = mf.models.Posterior(m) sym = mf.components.functions.MXFusionGluonFunction(SymmetricMatrix(), num_outputs=1, broadcastable=False) - cov = Variable(shape=(self.N,self.K,self.K), initial_value=mx.nd.broadcast_to(mx.nd.expand_dims(mx.nd.array(np.eye(self.K,self.K) * 1e-2), 0),shape=(self.N,self.K,self.K))) + cov = Variable(shape=(self.N,self.K,self.K), initial_value=mx.nd.broadcast_to(mx.nd.expand_dims(mx.nd.array(np.eye(self.K,self.K) * 1e-2, dtype=dtype), 0),shape=(self.N,self.K,self.K))) q.post_cov = sym(cov) - q.post_mean = Variable(shape=(self.N,self.K), initial_value=mx.nd.array(np.random.randn(self.N,self.K))) + q.post_mean = Variable(shape=(self.N,self.K), initial_value=mx.nd.array(np.random.randn(self.N,self.K), dtype=dtype)) q.z.set_prior(mf.distributions.MultivariateNormal(mean=q.post_mean, covariance=q.post_cov)) return q def get_ppca_grad(self, x_train, inf_type, num_samples=100): import random + dtype = get_default_dtype() random.seed(0) np.random.seed(0) mx.random.seed(0) @@ -91,14 +97,15 @@ def get_ppca_grad(self, x_train, inf_type, num_samples=100): from mxfusion.inference import BatchInferenceLoop infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop()) - infr.initialize(x=mx.nd.array(x_train)) - infr.run(max_iter=1, learning_rate=1e-2, x=mx.nd.array(x_train), verbose=False) + infr.initialize(x=mx.nd.array(x_train, dtype=dtype)) + infr.run(max_iter=1, learning_rate=1e-2, x=mx.nd.array(x_train, dtype=dtype), verbose=False) return infr, q.post_mean def test_score_function_batch(self): + dtype = get_default_dtype() x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) - x_nd, y_nd = mx.nd.array(y), mx.nd.array(x) + x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) @@ -117,9 +124,10 @@ def test_score_function_batch(self): def test_score_function_minibatch(self): + dtype = get_default_dtype() x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) - x_nd, y_nd = mx.nd.array(y), mx.nd.array(x) + x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) @@ -139,9 +147,10 @@ def test_score_function_minibatch(self): def test_score_function_rb_batch(self): + dtype = get_default_dtype() x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) - x_nd, y_nd = mx.nd.array(y), mx.nd.array(x) + x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) @@ -159,9 +168,10 @@ def test_score_function_rb_batch(self): infr.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd) def test_score_function_rb_minibatch(self): + dtype = get_default_dtype() x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) - x_nd, y_nd = mx.nd.array(y), mx.nd.array(x) + x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) From 722b023e36c5557b3746aacc461af8a062b762c6 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 22 Oct 2018 14:19:32 +0100 Subject: [PATCH 37/70] Add the test case for switching the default dtype. --- testing/inference/inference_alg_test.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/testing/inference/inference_alg_test.py b/testing/inference/inference_alg_test.py index 89c132f..54563e1 100644 --- a/testing/inference/inference_alg_test.py +++ b/testing/inference/inference_alg_test.py @@ -4,9 +4,12 @@ from mxfusion import Model, Variable from mxfusion.inference import Inference from mxfusion.inference.inference_alg import InferenceAlgorithm +from mxfusion.components.distributions import Normal +from mxfusion.components.variables import PositiveTransformation +from mxfusion.inference import GradBasedInference, MAP -class InferenceTests(unittest.TestCase): +class InferenceAlgorithmTests(unittest.TestCase): """ Test class that tests the MXFusion.utils methods. """ @@ -43,3 +46,23 @@ def compute(self, F, variables): assert np.allclose(x_res.asnumpy(), x_np) assert np.allclose(y_res.asnumpy(), y_np) + + def test_chagne_default_dtype(self): + from mxfusion.common import config + config.MXNET_DEFAULT_DTYPE = 'float64' + + np.random.seed(0) + mean_groundtruth = 3. + variance_groundtruth = 5. + N = 100 + data = np.random.randn(N)*np.sqrt(variance_groundtruth) + mean_groundtruth + + m = Model() + m.mu = Variable() + m.s = Variable(transformation=PositiveTransformation()) + m.Y = Normal.define_variable(mean=m.mu, variance=m.s, shape=(100,)) + + infr = GradBasedInference(inference_algorithm=MAP(model=m, observed=[m.Y])) + infr.run(Y=mx.nd.array(data, dtype='float64'), learning_rate=0.1, max_iters=1) + + config.MXNET_DEFAULT_DTYPE = 'float32' From f19f911776dcd66374563ec68ec7c06c1e126278 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 22 Oct 2018 15:20:04 +0100 Subject: [PATCH 38/70] Rename MXNET_DEFAULT_DTYPE to DEFAULT_DTYPE. --- mxfusion/common/config.py | 4 ++-- testing/inference/inference_alg_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mxfusion/common/config.py b/mxfusion/common/config.py index e58fd80..a4f89c6 100644 --- a/mxfusion/common/config.py +++ b/mxfusion/common/config.py @@ -1,6 +1,6 @@ import mxnet -MXNET_DEFAULT_DTYPE = 'float32' +DEFAULT_DTYPE = 'float32' MXNET_DEFAULT_MODE = mxnet.ndarray MXNET_DEFAULT_DEVICE = None @@ -12,7 +12,7 @@ def get_default_dtype(): :returns: the default dtype :rtypes: str """ - return MXNET_DEFAULT_DTYPE + return DEFAULT_DTYPE def get_default_MXNet_mode(): diff --git a/testing/inference/inference_alg_test.py b/testing/inference/inference_alg_test.py index 54563e1..4a11968 100644 --- a/testing/inference/inference_alg_test.py +++ b/testing/inference/inference_alg_test.py @@ -49,7 +49,7 @@ def compute(self, F, variables): def test_chagne_default_dtype(self): from mxfusion.common import config - config.MXNET_DEFAULT_DTYPE = 'float64' + config.DEFAULT_DTYPE = 'float64' np.random.seed(0) mean_groundtruth = 3. @@ -65,4 +65,4 @@ def test_chagne_default_dtype(self): infr = GradBasedInference(inference_algorithm=MAP(model=m, observed=[m.Y])) infr.run(Y=mx.nd.array(data, dtype='float64'), learning_rate=0.1, max_iters=1) - config.MXNET_DEFAULT_DTYPE = 'float32' + config.DEFAULT_DTYPE = 'float32' From f5a089d73f6b24acda6b3755f1b3011029d92d77 Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Mon, 22 Oct 2018 15:20:05 +0100 Subject: [PATCH 39/70] Docstring correction Co-Authored-By: jnkm --- mxfusion/components/distributions/random_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mxfusion/components/distributions/random_gen.py b/mxfusion/components/distributions/random_gen.py index 96a0dd7..b2f53ea 100644 --- a/mxfusion/components/distributions/random_gen.py +++ b/mxfusion/components/distributions/random_gen.py @@ -39,7 +39,7 @@ def _sample_univariate(func, shape=None, dtype=None, out=None, ctx=None, F=None, dtype = get_default_dtype() if dtype is None else dtype if F is mx.ndarray: - # This is required MXNet uses _Null instead of None as shape default + # This is required because MXNet uses _Null instead of None as shape default if shape is None: return func(dtype=dtype, ctx=ctx, out=out, **kwargs) else: From ed8c1806d67e5311b1286ce1aacd88293a696503 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 22 Oct 2018 17:38:56 +0100 Subject: [PATCH 40/70] Add the getting started notebook. --- docs/tutorials.md | 7 +- examples/notebooks/getting_started.ipynb | 434 +++++++++++++++++++++++ 2 files changed, 436 insertions(+), 5 deletions(-) create mode 100644 examples/notebooks/getting_started.ipynb diff --git a/docs/tutorials.md b/docs/tutorials.md index e7e3dcc..db4a478 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -2,11 +2,8 @@ Below is a list of tutorial / example notebooks demonstrating MXFusion's functionality. -## Getting Started - -* [Introduction (through Probabilistic PCA)](examples/notebooks/ppca_tutorial.ipynb) - -## Example Models +* [Getting Started](examples/notebooks/getting_started.ipynb) +* [Probabilistic PCA](examples/notebooks/ppca_tutorial.ipynb) * [Bayesian Neural Network Classification](examples/notebooks/bnn_classification.ipynb) * [Bayesian Neural Network Regression](examples/notebooks/bnn_regression.ipynb) * [Variational Auto-Encoder](examples/notebooks/variational_auto_encoder.ipynb) diff --git a/examples/notebooks/getting_started.ipynb b/examples/notebooks/getting_started.ipynb new file mode 100644 index 0000000..82918ed --- /dev/null +++ b/examples/notebooks/getting_started.ipynb @@ -0,0 +1,434 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Getting Started\n", + "\n", + "**Zhenwen Dai (2018.10.22)**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "MXFusion is a probabilistic programming language. It provides a convenient interface for designing probabilistic models and applying them to real world problems.\n", + "\n", + "Probabilistic models describe the relation in data through the probabilistic distribution of random variables. Probabilistic modeling is typically done through stating the prior believe in terms of a probabilistic model and performing inference with the observation of some of the random variables." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings('ignore')\n", + "import os\n", + "os.environ['MXNET_ENGINE_TYPE'] = 'NaiveEngine'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A Simple Example\n", + "\n", + "Let's start with a toy example about estimating the mean and variance of a set of data. For simplicity, we generate 100 data points with a given mean and variance following a normal distribution." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "np.random.seed(0)\n", + "mean_groundtruth = 3.\n", + "variance_groundtruth = 5.\n", + "N = 100\n", + "data = np.random.randn(N)*np.sqrt(variance_groundtruth) + mean_groundtruth" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's visualize our data by building a histogram." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEB5JREFUeJzt3X+MZWV9x/H3p4BaEQu6I8qPdWxLaJEIksmqJTVYlMJCoG1sy6a1VGlWDbbamNRVE23sPzRW7Q+MdAtbtFI0RbGkuyBbNUETQQdcYBEQSlcZl7KLKEixMavf/jFnk+lwZ3d6z525u/O8X8nNPec5zz3P92R3P3vmmXPOTVUhSWrHz4y7AEnS8jL4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY05dNwFDLJq1aqanJwcdxmSdNC47bbbHq2qicX0PSCDf3Jykunp6XGXIUkHjSTfXmxfp3okqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxB+Sdu9KBanLD5rGMu+PSc8cyrlYmz/glqTH7PeNPsgk4D9hVVSd3bZ8GTuy6HAn8oKpOHfDZHcAPgZ8Ae6pqakR1S5KGtJipnquAy4BP7G2oqt/du5zkQ8Dj+/j8a6rq0WELlCSN1n6Dv6puTjI5aFuSAL8D/Npoy5IkLZW+c/y/CjxSVfcvsL2Am5LclmT9vnaUZH2S6STTu3fv7lmWJGkhfYN/HXDNPrafXlWnAecAlyR59UIdq2pjVU1V1dTExKK+S0CSNIShgz/JocBvAZ9eqE9V7ezedwHXAWuGHU+SNBp9zvhfC9xbVTODNiY5PMkRe5eBs4DtPcaTJI3AfoM/yTXAV4ETk8wkubjbdCHzpnmSHJNkS7d6NPCVJHcAXwM2V9WNoytdkjSMxVzVs26B9j8c0LYTWNstPwic0rM+SdKI+cgGHZTG9egEaSXwkQ2S1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjfFaPdBAY57OJdlx67tjG1tLwjF+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMbsN/iTbEqyK8n2OW1/nuS7SbZ1r7ULfPbsJPcleSDJhlEWLkkazmLO+K8Czh7Q/pGqOrV7bZm/MckhwEeBc4CTgHVJTupTrCSpv/0Gf1XdDDw2xL7XAA9U1YNV9WPgU8AFQ+xHkjRCfeb435bkzm4q6KgB248FHpqzPtO1DZRkfZLpJNO7d+/uUZYkaV+GDf6PAb8AnAo8DHxoQJ8MaKuFdlhVG6tqqqqmJiYmhixLkrQ/QwV/VT1SVT+pqp8C/8DstM58M8Dxc9aPA3YOM54kaXSGCv4kL5qz+pvA9gHdvg6ckOQlSZ4BXAhcP8x4kqTR2e9jmZNcA5wBrEoyA7wfOCPJqcxO3ewA3tz1PQa4oqrWVtWeJG8DPg8cAmyqqruX5CgkSYu23+CvqnUDmq9coO9OYO2c9S3A0y71lCSNj3fuSlJjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmP2+w1c0kImN2wedwmShuAZvyQ1Zr/Bn2RTkl1Jts9p+2CSe5PcmeS6JEcu8NkdSe5Ksi3J9CgLlyQNZzFn/FcBZ89r2wqcXFUvA74FvHsfn39NVZ1aVVPDlShJGqX9Bn9V3Qw8Nq/tpqra063eAhy3BLVJkpbAKOb43wTcsMC2Am5KcluS9SMYS5LUU6+repK8F9gDXL1Al9OrameSFwBbk9zb/QQxaF/rgfUAq1ev7lOWJGkfhj7jT3IRcB7we1VVg/pU1c7ufRdwHbBmof1V1caqmqqqqYmJiWHLkiTtx1DBn+Rs4F3A+VX11AJ9Dk9yxN5l4Cxg+6C+kqTls5jLOa8BvgqcmGQmycXAZcARzE7fbEtyedf3mCRbuo8eDXwlyR3A14DNVXXjkhyFJGnR9jvHX1XrBjRfuUDfncDabvlB4JRe1UmSRs47dyWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUmF5fxCJp5ZvcsHks4+649NyxjNsCz/glqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWrMooI/yaYku5Jsn9P2vCRbk9zfvR+1wGcv6vrcn+SiURUuSRrOYs/4rwLOnte2AfhCVZ0AfKFb/z+SPA94P/AKYA3w/oX+g5AkLY9FBX9V3Qw8Nq/5AuDj3fLHgd8Y8NFfB7ZW1WNV9X1gK0//D0SStIz6zPEfXVUPA3TvLxjQ51jgoTnrM12bJGlMlvqXuxnQVgM7JuuTTCeZ3r179xKXJUnt6hP8jyR5EUD3vmtAnxng+DnrxwE7B+2sqjZW1VRVTU1MTPQoS5K0L32C/3pg71U6FwH/OqDP54GzkhzV/VL3rK5NkjQmi72c8xrgq8CJSWaSXAxcCrwuyf3A67p1kkwluQKgqh4D/gL4evf6QNcmSRqTRT2Pv6rWLbDpzAF9p4E/mrO+Cdg0VHWSpJHzzl1JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDVm6OBPcmKSbXNeTyR5x7w+ZyR5fE6f9/UvWZLUx6HDfrCq7gNOBUhyCPBd4LoBXb9cVecNO44kabRGNdVzJvAfVfXtEe1PkrRERhX8FwLXLLDtVUnuSHJDkpeOaDxJ0pB6B3+SZwDnA/8yYPPtwIur6hTg74DP7WM/65NMJ5nevXt337IkSQsYxRn/OcDtVfXI/A1V9URVPdktbwEOS7Jq0E6qamNVTVXV1MTExAjKkiQNMorgX8cC0zxJXpgk3fKabrzvjWBMSdKQhr6qByDJs4HXAW+e0/YWgKq6HHg98NYke4AfARdWVfUZU5LUT6/gr6qngOfPa7t8zvJlwGV9xpAkjVav4NeBYXLD5nGXII3cOP9e77j03LGNvRx8ZIMkNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUmN7Bn2RHkruSbEsyPWB7kvxtkgeS3JnktL5jSpKGN6ovW39NVT26wLZzgBO61yuAj3XvkqQxWI6pnguAT9SsW4Ajk7xoGcaVJA0wijP+Am5KUsDfV9XGeduPBR6asz7TtT08t1OS9cB6gNWrV4+grOU1uWHzuEuQNCLj+ve849Jzl2WcUZzxn15VpzE7pXNJklfP254Bn6mnNVRtrKqpqpqamJgYQVmSpEF6B39V7ezedwHXAWvmdZkBjp+zfhyws++4kqTh9Ar+JIcnOWLvMnAWsH1et+uBP+iu7nkl8HhVPYwkaSz6zvEfDVyXZO++/rmqbkzyFoCquhzYAqwFHgCeAt7Yc0xJUg+9gr+qHgROGdB++ZzlAi7pM44kaXS8c1eSGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqzNDBn+T4JF9Kck+Su5O8fUCfM5I8nmRb93pfv3IlSX31+bL1PcA7q+r2JEcAtyXZWlXfnNfvy1V1Xo9xJEkjNPQZf1U9XFW3d8s/BO4Bjh1VYZKkpTGSOf4kk8DLgVsHbH5VkjuS3JDkpaMYT5I0vD5TPQAkeQ7wGeAdVfXEvM23Ay+uqieTrAU+B5ywwH7WA+sBVq9e3bcsSdICep3xJzmM2dC/uqo+O397VT1RVU92y1uAw5KsGrSvqtpYVVNVNTUxMdGnLEnSPvS5qifAlcA9VfXhBfq8sOtHkjXdeN8bdkxJUn99pnpOB94A3JVkW9f2HmA1QFVdDrweeGuSPcCPgAurqnqMKUnqaejgr6qvANlPn8uAy4YdQ5I0et65K0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TG9P4ilgPN5IbN4y5Bkg5onvFLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktSYXsGf5Owk9yV5IMmGAdufmeTT3fZbk0z2GU+S1N/QwZ/kEOCjwDnAScC6JCfN63Yx8P2q+kXgI8BfDjueJGk0+pzxrwEeqKoHq+rHwKeAC+b1uQD4eLd8LXBmkvQYU5LUU5/gPxZ4aM76TNc2sE9V7QEeB57fY0xJUk99ntUz6My9hugz2zFZD6zvVp9Mcl+P2kZpFfDouItYQiv9+GDlH+NKPz5Y+ce4Cng0/SbDX7zYjn2CfwY4fs76ccDOBfrMJDkU+DngsUE7q6qNwMYe9SyJJNNVNTXuOpbKSj8+WPnHuNKPD1b+MS738fWZ6vk6cEKSlyR5BnAhcP28PtcDF3XLrwe+WFUDz/glSctj6DP+qtqT5G3A54FDgE1VdXeSDwDTVXU9cCXwT0keYPZM/8JRFC1JGl6v5/FX1RZgy7y2981Z/h/gt/uMcQA44KafRmylHx+s/GNc6ccHK/8Yl/X44syLJLXFRzZIUmMM/kVI8sEk9ya5M8l1SY4cd02jsL9HbhzMkhyf5EtJ7klyd5K3j7umpZDkkCTfSPJv465lKSQ5Msm13b+/e5K8atw1jVqSP+3+jm5Pck2SZy31mAb/4mwFTq6qlwHfAt495np6W+QjNw5me4B3VtUvA68ELllhx7fX24F7xl3EEvob4Maq+iXgFFbYsSY5FvgTYKqqTmb2QpklvwjG4F+Eqrqpu/MY4BZm71k42C3mkRsHrap6uKpu75Z/yGxgzL+z/KCW5DjgXOCKcdeyFJI8F3g1s1cHUlU/rqofjLeqJXEo8LPdvU7P5un3Q42cwf//9ybghnEXMQKLeeTGitA9FfblwK3jrWTk/hr4M+Cn4y5kifw8sBv4x24664okh4+7qFGqqu8CfwV8B3gYeLyqblrqcQ3+TpJ/7+bY5r8umNPnvcxOIVw9vkpHZtGP0ziYJXkO8BngHVX1xLjrGZUk5wG7quq2cdeyhA4FTgM+VlUvB/4bWGm/izqK2Z+0XwIcAxye5PeXetxe1/GvJFX12n1tT3IRcB5w5gq5+3gxj9w4qCU5jNnQv7qqPjvuekbsdOD8JGuBZwHPTfLJqlry0FhGM8BMVe39Se1aVljwA68F/rOqdgMk+SzwK8Anl3JQz/gXIcnZwLuA86vqqXHXMyKLeeTGQat7/PeVwD1V9eFx1zNqVfXuqjquqiaZ/bP74goLfarqv4CHkpzYNZ0JfHOMJS2F7wCvTPLs7u/smSzDL7A941+cy4BnAlu7rxO4pareMt6S+lnokRtjLmuUTgfeANyVZFvX9p7ubnMdPP4YuLo7OXkQeOOY6xmpqro1ybXA7cxOI3+DZbiL1zt3JakxTvVIUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGvO/DZ47ZQlqyawAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%matplotlib inline\n", + "from pylab import *\n", + "_=hist(data, 10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, let's pretend that we do not know the mean and variance that are used to generate the above data. \n", + "\n", + "We still believe that the data come from a normal distribution, which is our model. It is formulated as\n", + "$$y_n \\sim \\mathcal{N}(\\mu, s), \\quad Y=(y_1, \\ldots, y_{100})$$\n", + "where $\\mu$ is the mean, $s$ is the variance and $Y$ is the vector representing the data.\n", + "\n", + "\n", + "In MXFusion, the above model can be defined as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from mxfusion import Variable, Model\n", + "from mxfusion.components.variables import PositiveTransformation\n", + "from mxfusion.components.distributions import Normal\n", + "from mxfusion.common import config\n", + "config.DEFAULT_DTYPE = 'float64'\n", + "\n", + "m = Model()\n", + "m.mu = Variable()\n", + "m.s = Variable(transformation=PositiveTransformation())\n", + "m.Y = Normal.define_variable(mean=m.mu, variance=m.s, shape=(N,))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the above definition, we start with defining a model by instantiated from the class Model. The variable $\\mu$ and $s$ are created from the class Variable. Both of them are assigned as members of the model instance m. This is the way in which variables are organized in MXFusion. The variable s is created by assigning a PositiveTransformation instance to the transforamtion argument. This constrains the value of the variable s to be positive through a \"soft-plus\" transformation. The variable Y is created from a normal distribution by specifying the mean and variance and its shape. \n", + "\n", + "Note that, in this example, the mean and variance variable are both scalar, with the shape (1,), while the random variable Y has the shape (100,). This indicates the mean and variance variable are broadcasted into the shape of the random variable, just like the broadcasting rule in numpy array operation. In this case, this means the individual entries of the random variable Y follows a scalar normal distribution with the same mean and variance.\n", + "\n", + "To list the content that is defined in the model instance, just print the model instance as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Y ~ Normal(mean=mu, variance=s)\n" + ] + } + ], + "source": [ + "print(m)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After defining the probabilistic model, we want to estimate the mean and variance of the normal distribution in our model conditioned on the data that we generated. In MXFusion, this is done by creating an inference algorithm and passing it into the creation of an Inference instance. An inference algorithm represents a specific algorithm for a probabilistic inference. In this example, we performs a maximum likelihood estimate by using the MAP class. The Inference class takes care of the initialization of parameters and the execution of inference.\n", + "\n", + "In the following code, we created a MAP inference algorithm by specifying the model and the set of observed variable. Then, we created a GradBasedInference instance from the instantiated MAP infernece algorithm.\n", + "\n", + "The execution of inference is done by calling the call function. The call function takes all observed data (specified when creating the inference algorithm) as the keyword arguments, where the keys are the names of the member variables of the model and the values are the corresponding MXNet NDArrays. In this example, we only observed the variable Y, then, we pass \"Y\" as the key and the generated data as the value. We also specify the configuration parameters for the gradient optimizer such as the learning rate, the maximum number of iterations and whether to print the optimization progress. The default optimizer is adam." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 201 loss: 226.00307424753382\n", + "Iteration 401 loss: 223.62512496838366\n", + "Iteration 601 loss: 223.23108001422028\n", + "Iteration 801 loss: 223.16242835266598\n", + "Iteration 1001 loss: 223.15215875874755\n", + "Iteration 1201 loss: 223.15098355232075\n", + "Iteration 1401 loss: 223.15088846215527\n", + "Iteration 1601 loss: 223.15088333213185\n", + "Iteration 1801 loss: 223.15088315658113\n", + "Iteration 2000 loss: 223.15088315295884" + ] + } + ], + "source": [ + "from mxfusion.inference import GradBasedInference, MAP\n", + "import mxnet as mx\n", + "\n", + "infr = GradBasedInference(inference_algorithm=MAP(model=m, observed=[m.Y]))\n", + "infr.run(Y=mx.nd.array(data, dtype='float64'), learning_rate=0.1, max_iter=2000, verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After optimiztaion, the estimated parameters are stored in an instance of the class InferenceParameters, which can be access from the Inference instance such as infr.params.\n", + "\n", + "We collect the estimated mean and variance and compared with the generating parameters." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The estimated mean and variance: 3.133735, 5.079126.\n", + "The true mean and variance: 3.000000, 5.000000.\n" + ] + } + ], + "source": [ + "mean_estimated = infr.params[m.mu].asnumpy()\n", + "variance_estimated = infr.params[m.s].asnumpy()\n", + "\n", + "print('The estimated mean and variance: %f, %f.' % (mean_estimated, variance_estimated))\n", + "print('The true mean and variance: %f, %f.' % (mean_groundtruth, variance_groundtruth))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The estimated parameters are close to the generating parameters with a small difference. This difference is due to the small number of existing data, which is also known as *over-fitting*." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A Bayesian model\n", + "\n", + "From the above example, we have done a maximum likelihood estimate from the observed data. Due to the limited number of data, the estimated parameters are not the same as the true parameters. An interesting question here is that whether we can have an estimate about how big the difference is. One approach to provide such an estimate is via Bayesian inference. \n", + "\n", + "Following the above example, we need to assume prior distributions for the mean and variance of the normal distribution. We assume the mean to be a normal distribution with a relative big variance, indicating that we do not have much knowledge about the parameter." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "m = Model()\n", + "m.mu = Normal.define_variable(mean=mx.nd.array([0], dtype='float64'), \n", + " variance=mx.nd.array([100], dtype='float64'), shape=(1,))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, we need to specify a prior distribution for the variance. This is a bit more complicated as the variance needs to be positive. In principle, one can use a distribution of positive values such as the Gamma distribution. To enable inference with the reparameterization trick, we, instead, assume a random variable $\\hat{s}$ with a normal distribution and the variance $s$ is a function of $\\hat{s}$,\n", + "$$\n", + "\\hat{s} \\sim \\mathcal{N}(5, 100), \\quad s = \\log(1+e^{\\hat{s}}).\n", + "$$\n", + "The above function is often referred to as the \"soft-plus\" function, which transforms a real number to a positive number. By applying the transformation, we indirectly specifies the prior distribution for the variance. \n", + "\n", + "To implement the above prior in MXFusion, we first create the variable s_hat with a normal distribution. Then, we defines a function in the MXNet Gluon syntax, which is also called a Gluon block, for the \"soft-plus\" transformation. The MXNet function is brought into the MXFusion environment by applying a wrapper called MXFusionGluonFunction, in which we specify the number of outputs. We pass the variable s_hat as the input to the function and get the variable s as the return value." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from mxfusion.components.functions import MXFusionGluonFunction\n", + "\n", + "m.s_hat = Normal.define_variable(mean=mx.nd.array([5], dtype='float64'), \n", + " variance=mx.nd.array([100], dtype='float64'),\n", + " shape=(1,), dtype=dtype)\n", + "trans_mxnet = mx.gluon.nn.HybridLambda(lambda F, x: F.Activation(x, act_type='softrelu'))\n", + "m.trans = MXFusionGluonFunction(trans_mxnet, num_outputs=1, broadcastable=True)\n", + "m.s = m.trans(m.s_hat)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We define the variable Y following a normal distribution with the mean mu and the variance s. " + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "s_hat ~ Normal(mean=Variable(66591), variance=Variable(582f6))\n", + "s = GluonFunctionEvaluation(hybridlambda0_input_0=s_hat)\n", + "mu ~ Normal(mean=Variable(230f5), variance=Variable(e5c1f))\n", + "Y ~ Normal(mean=mu, variance=s)\n" + ] + } + ], + "source": [ + "m.Y = Normal.define_variable(mean=m.mu, variance=m.s, shape=(N,), dtype=dtype)\n", + "print(m)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Inference for the above model is more complex, as the exact inference is intractable. We use variational inference with a Gaussian mean field posterior. \n", + "\n", + "We construct the variational posterior by calling the function create_Gaussian_meanfield, which defines a Gaussian distribution for both the mean and the variance as the variational posterior. The content in the generated posterior can be listed by printing the posterior." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "s_hat ~ Normal(mean=Variable(987b5), variance=Variable(bf47c))\n", + "mu ~ Normal(mean=Variable(9c88b), variance=Variable(3ce74))\n" + ] + } + ], + "source": [ + "from mxfusion.inference import create_Gaussian_meanfield\n", + "\n", + "q = create_Gaussian_meanfield(model=m, observed=[m.Y])\n", + "print(q)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, we created an instance of StochasticVariationalInference with both the model and the variational posterior. We also need to specify the number of samples used in inference, as it uses the Monte Carlo method for approximating the integral in the variational lower bound. The execution of inference follows the same interface." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 201 loss: 234.52798823187217\n", + "Iteration 401 loss: 231.30663180752714\n", + "Iteration 601 loss: 230.14185436095775\n", + "Iteration 801 loss: 230.02993763459781\n", + "Iteration 1001 loss: 229.6937452192292\n", + "Iteration 1201 loss: 229.72574317072662\n", + "Iteration 1401 loss: 229.65264175269712\n", + "Iteration 1601 loss: 229.67285188868293\n", + "Iteration 1801 loss: 229.52906307607037\n", + "Iteration 2000 loss: 229.64091981034755" + ] + } + ], + "source": [ + "from mxfusion.inference import StochasticVariationalInference\n", + "\n", + "infr = GradBasedInference(inference_algorithm=StochasticVariationalInference(\n", + " model=m, posterior=q, num_samples=10, observed=[m.Y]))\n", + "infr.run(Y=mx.nd.array(data, dtype='float64'), learning_rate=0.1, verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's check the resulting posterior distribution." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The mean and standard deviation of the mean parameter is 3.120117(0.221690). \n", + "The 15th, 50th and 85th percentile of the variance parameter is 4.604521, 5.309114 and 6.016289.\n" + ] + } + ], + "source": [ + "mu_mean = infr.params[q.mu.factor.mean].asscalar()\n", + "mu_std = np.sqrt(infr.params[q.mu.factor.variance].asscalar())\n", + "s_hat_mean = infr.params[q.s_hat.factor.mean].asscalar()\n", + "s_hat_std = np.sqrt(infr.params[q.s_hat.factor.variance].asscalar())\n", + "s_15 = np.log1p(np.exp(s_hat_mean - s_hat_std))\n", + "s_50 = np.log1p(np.exp(s_hat_mean))\n", + "s_85 = np.log1p(np.exp(s_hat_mean + s_hat_std))\n", + "print('The mean and standard deviation of the mean parameter is %f(%f). ' % (mu_mean, mu_std))\n", + "print('The 15th, 50th and 85th percentile of the variance parameter is %f, %f and %f.'%(s_15, s_50, s_85))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The true parameter sits within one standard deviation of the estimated posterior distribution for both the mean and variance parameters. The above error gives a good indication about how much we could trust the parameters that we estimate." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 5e3ee917e55f5db58007335a8907f0be4803fda8 Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Mon, 22 Oct 2018 18:11:56 +0100 Subject: [PATCH 41/70] Update testing/inference/inference_alg_test.py Co-Authored-By: zhenwendai --- testing/inference/inference_alg_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/inference/inference_alg_test.py b/testing/inference/inference_alg_test.py index 4a11968..9f3db1f 100644 --- a/testing/inference/inference_alg_test.py +++ b/testing/inference/inference_alg_test.py @@ -47,7 +47,7 @@ def compute(self, F, variables): assert np.allclose(x_res.asnumpy(), x_np) assert np.allclose(y_res.asnumpy(), y_np) - def test_chagne_default_dtype(self): + def test_change_default_dtype(self): from mxfusion.common import config config.DEFAULT_DTYPE = 'float64' From 8c75085dea9c6ace948a97ba98d35390e59f47b7 Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Mon, 22 Oct 2018 18:11:58 +0100 Subject: [PATCH 42/70] Update testing/inference/inference_alg_test.py Co-Authored-By: zhenwendai --- testing/inference/inference_alg_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/inference/inference_alg_test.py b/testing/inference/inference_alg_test.py index 9f3db1f..00bce3a 100644 --- a/testing/inference/inference_alg_test.py +++ b/testing/inference/inference_alg_test.py @@ -63,6 +63,6 @@ def test_change_default_dtype(self): m.Y = Normal.define_variable(mean=m.mu, variance=m.s, shape=(100,)) infr = GradBasedInference(inference_algorithm=MAP(model=m, observed=[m.Y])) - infr.run(Y=mx.nd.array(data, dtype='float64'), learning_rate=0.1, max_iters=1) + infr.run(Y=mx.nd.array(data, dtype='float64'), learning_rate=0.1, max_iters=2) config.DEFAULT_DTYPE = 'float32' From 455996e39c4716386d308c476ffa5957411b88e8 Mon Sep 17 00:00:00 2001 From: Jordan Date: Tue, 23 Oct 2018 09:15:16 +0100 Subject: [PATCH 43/70] Better explain required use of float64 --- testing/distributions/dirichlet_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/distributions/dirichlet_test.py b/testing/distributions/dirichlet_test.py index bc4bdab..b4618fd 100644 --- a/testing/distributions/dirichlet_test.py +++ b/testing/distributions/dirichlet_test.py @@ -12,7 +12,8 @@ @pytest.mark.usefixtures("set_seed") class TestDirichletDistribution(object): - # dtype must be float64 to ensure that sum of x_i is always 1 + # scipy implementation of dirichlet throws an error if x_i does not sum to one and float32 is + # not precise enough to have sum close enough to 1 after normalisation so using float64 @pytest.mark.parametrize("dtype, a, a_is_samples, rv, rv_is_samples, num_samples", [ (np.float64, np.random.rand(2), False, np.random.rand(5, 3, 2), True, 5), (np.float64, np.random.rand(2), False, np.random.rand(10, 3, 2), True, 10), From 6798e4ee63931ecbebad1c3d7c0d55c1fdf35774 Mon Sep 17 00:00:00 2001 From: Massiah Date: Tue, 23 Oct 2018 14:53:59 +0100 Subject: [PATCH 44/70] Move dirichlet tests --- testing/{ => components}/distributions/dirichlet_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename testing/{ => components}/distributions/dirichlet_test.py (100%) diff --git a/testing/distributions/dirichlet_test.py b/testing/components/distributions/dirichlet_test.py similarity index 100% rename from testing/distributions/dirichlet_test.py rename to testing/components/distributions/dirichlet_test.py From fdebb5d3a5539d7b25a5a5e6dd6ed61ff8efd396 Mon Sep 17 00:00:00 2001 From: Mark Pullin Date: Thu, 18 Oct 2018 15:11:30 +0100 Subject: [PATCH 45/70] Add logistic variable transformation --- mxfusion/components/variables/__init__.py | 2 +- mxfusion/components/variables/var_trans.py | 46 +++++++++++++++++++ .../components/variables/var_trans_test.py | 15 +++++- 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/mxfusion/components/variables/__init__.py b/mxfusion/components/variables/__init__.py index 73e89b3..5bb6e35 100644 --- a/mxfusion/components/variables/__init__.py +++ b/mxfusion/components/variables/__init__.py @@ -14,5 +14,5 @@ __all__ = ['runtime_variable', 'var_trans', 'variable'] from .runtime_variable import add_sample_dimension, add_sample_dimension_to_arrays, expectation, is_sampled_array, get_num_samples, as_samples -from .var_trans import Softplus, PositiveTransformation +from .var_trans import Softplus, PositiveTransformation, Logistic from .variable import Variable, VariableType diff --git a/mxfusion/components/variables/var_trans.py b/mxfusion/components/variables/var_trans.py index 0163998..3fe3d08 100644 --- a/mxfusion/components/variables/var_trans.py +++ b/mxfusion/components/variables/var_trans.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +import numpy as np from ...common.config import get_default_MXNet_mode @@ -84,3 +85,48 @@ def __init__(self): Initializes as Softplus transformation with 0 offset. """ super(PositiveTransformation, self).__init__(offset=0.) + + +class Logistic(VariableTransformation): + """ + Transformation to constraint a variable to lie between two values. + """ + def __init__(self, lower, upper): + """ + :param lower: Lower bound + :param upper: Upper bound + """ + if lower >= upper: + raise ValueError('The lower bound is above the upper bound') + self._lower, self._upper = lower, upper + self._difference = self._upper - self._lower + + def transform(self, var, F=None, dtype=None): + """ + Forward transformation. + + :param var: Variable to be transformed. + :type var: mx.ndarray or mx.sym + :param F: Mode to run MxNet in. + :type F: mxnet.ndarray or mxnet.symbol + :param dtype: data type. + :type dtype: e.g. np.float32 + """ + F = get_default_MXNet_mode() if F is None else F + return self._lower + self._difference * F.Activation(var, act_type='sigmoid') + + def inverseTransform(self, out_var, F=None, dtype=None): + """ + Inverse transformation. + + :param out_var: Variable to be transformed. + :type out_var: mx.ndarray or mx.sym + :param F: Mode to run MxNet in. + :type F: mxnet.ndarray or mxnet.symbol + :param dtype: data type. + :type dtype: e.g. np.float32 + """ + F = get_default_MXNet_mode() if F is None else F + # Clip out_var to be within bounds to avoid taking log of zero or infinity + clipped_out_var = F.clip(out_var, self._lower + 1e-10, self._upper - 1e-10) + return F.log((clipped_out_var - self._lower) / (self._upper - clipped_out_var)) diff --git a/testing/components/variables/var_trans_test.py b/testing/components/variables/var_trans_test.py index 7df5e52..e7f1946 100644 --- a/testing/components/variables/var_trans_test.py +++ b/testing/components/variables/var_trans_test.py @@ -2,7 +2,7 @@ import mxnet as mx import numpy as np import numpy.testing as npt -from mxfusion.components.variables.var_trans import PositiveTransformation +from mxfusion.components.variables.var_trans import PositiveTransformation, Logistic @pytest.mark.usefixtures("set_seed") @@ -38,3 +38,16 @@ def test_softplus_numerical(self, x, rtol, atol): npt.assert_allclose(mf_pos.asnumpy(), np_pos, rtol=rtol, atol=atol) npt.assert_allclose(mf_inv.asnumpy(), np_inv, rtol=rtol, atol=atol) npt.assert_allclose(mf_inv.asnumpy(), x.asnumpy(), rtol=rtol, atol=atol) + + @pytest.mark.parametrize("x, upper, lower, rtol, atol", [ + (mx.nd.array([10], dtype=np.float64), 2, 20, 1e-7, 1e-10), + (mx.nd.array([1e-3], dtype=np.float64), 1e-6, 1e-2, 1e-7, 1e-10), + (mx.nd.array([1], dtype=np.float32), 1, 200000, 1e-4, 1e-5), + (mx.nd.array([5], dtype=np.float32), 2, 10000, 1e-4, 1e-5) + ]) + def test_logistic(self, x, upper, lower, rtol, atol): + transform = Logistic(upper, lower) + x_trans = transform.transform(x) + x_inversed = transform.inverseTransform(x_trans) + assert x_inversed.dtype == x.dtype + assert np.isclose(x.asnumpy(), x_inversed.asnumpy(), rtol=rtol, atol=atol) \ No newline at end of file From 82d6f26ddf37f3e916b2c814f91ca38c8742e758 Mon Sep 17 00:00:00 2001 From: Massiah Date: Tue, 23 Oct 2018 15:04:04 +0100 Subject: [PATCH 46/70] is_sampled_array->array_has_samples --- mxfusion/components/distributions/bernoulli.py | 4 ++-- mxfusion/components/distributions/normal.py | 2 +- testing/distributions/bernoulli_test.py | 6 +++--- .../distributions/normal_mean_precision_test.py | 16 ++++++++-------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/mxfusion/components/distributions/bernoulli.py b/mxfusion/components/distributions/bernoulli.py index 4036396..df83505 100644 --- a/mxfusion/components/distributions/bernoulli.py +++ b/mxfusion/components/distributions/bernoulli.py @@ -2,7 +2,7 @@ from .univariate import UnivariateDistribution from .distribution import LogPDFDecorator, DrawSamplesDecorator from ...util.customop import broadcast_to_w_samples -from ..variables import get_num_samples, is_sampled_array +from ..variables import get_num_samples, array_has_samples from ...common.config import get_default_MXNet_mode from ...common.exceptions import InferenceError @@ -57,7 +57,7 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, rv_shape = list(rv_shape.values())[0] variables = {name: kw[name] for name, _ in self.inputs} - is_samples = any([is_sampled_array(F, v) for v in variables.values()]) + is_samples = any([array_has_samples(F, v) for v in variables.values()]) if is_samples: num_samples_inferred = max([get_num_samples(F, v) for v in variables.values()]) if num_samples_inferred != num_samples: diff --git a/mxfusion/components/distributions/normal.py b/mxfusion/components/distributions/normal.py index 335062d..ee7d4a0 100644 --- a/mxfusion/components/distributions/normal.py +++ b/mxfusion/components/distributions/normal.py @@ -488,7 +488,7 @@ def draw_samples_broadcast(self, F, rv_shape, num_samples=1, rv_shape = list(rv_shape.values())[0] variables = {name: kw[name] for name, _ in self.inputs} - isSamples = any([is_sampled_array(F, v) for v in variables.values()]) + isSamples = any([array_has_samples(F, v) for v in variables.values()]) if isSamples: num_samples_inferred = max([get_num_samples(F, v) for v in variables.values()]) diff --git a/testing/distributions/bernoulli_test.py b/testing/distributions/bernoulli_test.py index 3f7cab2..bb76d2e 100644 --- a/testing/distributions/bernoulli_test.py +++ b/testing/distributions/bernoulli_test.py @@ -3,7 +3,7 @@ import numpy as np from scipy.stats import bernoulli -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.components.distributions import Bernoulli from mxfusion.util.testutils import numpy_array_reshape from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -68,7 +68,7 @@ def test_draw_samples(self, dtype, prob_true, prob_true_is_samples, rv_shape, nu rv_samples_rt = var.draw_samples( F=mx.nd, variables=variables, num_samples=num_samples) - assert is_sampled_array(mx.nd, rv_samples_rt) + assert array_has_samples(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples assert np.array_equal(rv_samples_np, rv_samples_rt.asnumpy().astype(bool)) @@ -82,6 +82,6 @@ def test_draw_samples(self, dtype, prob_true, prob_true_is_samples, rv_shape, nu rv_samples_rt = var.draw_samples( F=mx.nd, variables=variables, num_samples=num_samples) - assert is_sampled_array(mx.nd, rv_samples_rt) + assert array_has_samples(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples assert rv_samples_rt.dtype == dtype diff --git a/testing/distributions/normal_mean_precision_test.py b/testing/distributions/normal_mean_precision_test.py index 54bc81c..0276bdf 100644 --- a/testing/distributions/normal_mean_precision_test.py +++ b/testing/distributions/normal_mean_precision_test.py @@ -3,7 +3,7 @@ import numpy as np from scipy.stats import norm, multivariate_normal -from mxfusion.components.variables.runtime_variable import add_sample_dimension, is_sampled_array, get_num_samples +from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.components.distributions import NormalMeanPrecision, MultivariateNormalMeanPrecision from mxfusion.util.testutils import numpy_array_reshape from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -45,7 +45,7 @@ def test_log_pdf(self, dtype, mean, mean_is_samples, precision, precision_is_sam log_pdf_rt = var.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + assert array_has_samples(mx.nd, log_pdf_rt) == is_samples_any if is_samples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples if np.issubdtype(dtype, np.float64): @@ -86,7 +86,7 @@ def test_draw_samples(self, dtype, mean, mean_is_samples, precision, F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(rv_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, rv_samples_rt) + assert array_has_samples(mx.nd, rv_samples_rt) assert get_num_samples(mx.nd, rv_samples_rt) == num_samples if np.issubdtype(dtype, np.float64): @@ -154,7 +154,7 @@ def test_log_pdf_with_broadcast(self, dtype, mean, mean_is_samples, precision, p log_pdf_rt = normal.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + assert array_has_samples(mx.nd, log_pdf_rt) == is_samples_any if is_samples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) @@ -206,7 +206,7 @@ def test_log_pdf_no_broadcast(self, dtype, mean, mean_is_samples, precision, pre log_pdf_rt = normal.log_pdf(F=mx.nd, variables=variables) assert np.issubdtype(log_pdf_rt.dtype, dtype) - assert is_sampled_array(mx.nd, log_pdf_rt) == is_samples_any + assert array_has_samples(mx.nd, log_pdf_rt) == is_samples_any if is_samples_any: assert get_num_samples(mx.nd, log_pdf_rt) == num_samples, (get_num_samples(mx.nd, log_pdf_rt), num_samples) assert np.allclose(log_pdf_np, log_pdf_rt.asnumpy()) @@ -236,7 +236,7 @@ def test_draw_samples_with_broadcast(self, dtype, mean, mean_is_samples, precisi draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) == is_samples_any + assert array_has_samples(mx.nd, draw_samples_rt) == is_samples_any if is_samples_any: assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, \ (get_num_samples(mx.nd, draw_samples_rt), num_samples) @@ -267,7 +267,7 @@ def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype, mean, me draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) + assert array_has_samples(mx.nd, draw_samples_rt) @pytest.mark.parametrize( "dtype, mean, mean_is_samples, precision, precision_is_samples, rv_shape, num_samples", [ @@ -300,6 +300,6 @@ def test_draw_samples_no_broadcast(self, dtype, mean, mean_is_samples, precision draw_samples_rt = normal.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) assert np.issubdtype(draw_samples_rt.dtype, dtype) - assert is_sampled_array(mx.nd, draw_samples_rt) + assert array_has_samples(mx.nd, draw_samples_rt) assert get_num_samples(mx.nd, draw_samples_rt) == num_samples, \ (get_num_samples(mx.nd, draw_samples_rt), num_samples) From 2da047aaa859cb5e37ae1a599183216f196127c5 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Tue, 2 Oct 2018 12:01:19 +0100 Subject: [PATCH 47/70] Rename CIP to design document. No template yet. --- CONTRIBUTING.md | 4 ++-- .../CIPs.md => design_documents/design_doc_guidelines.md} | 0 docs/design/overview.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename docs/design/{CIPs/CIPs.md => design_documents/design_doc_guidelines.md} (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f54f978..60e81c1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,7 @@ If you're wishing for a feature that doesn't exist yet in MXFusion, there are pr If you're thinking about adding code to MXFusion, here are some guidelines to get you started. -* If the change is a major feature, create a [CIP](link to CIP definition) in the docs/CIPs folder and post it as a PR, optionally with a prototype implementation of your proposed changes. This is to get community feedback on the changes and document the design reasoning of MXFusion for future reference. +* If the change is a major feature, create a [design document](design/design_documents/design_doc_guidelines) in the docs/design/design_documents folder and post it as a PR, optionally with a prototype implementation of your proposed changes. This is to get community feedback on the changes and document the design reasoning of MXFusion for future reference. * Keep pull requests small, preferably one feature per pull request. This lowers the bar to entry for a reviewer, and keeps feedback focused for each feature. @@ -73,7 +73,7 @@ Before submitting the pull request, please go through this checklist to make the * Do all public functions have docstrings including examples? If you added a new module, did you add it to the Sphinx docstring in the ```__init__.py``` file of the module's folder? * Is the code style correct (PEP8)? * Is the commit message formatted correctly? -* If this is a large addition, is there a tutorial or more extensive module-level description? Did you discuss the addition in a [CIP](CIP)? Is there an issue related to the change? If so, please link the issue or CIP. +* If this is a large addition, is there a tutorial or more extensive module-level description? Did you discuss the addition in a [design document](design/design_documents/design_doc_guidelines)? Is there an issue related to the change? If so, please link the issue or design doc. ## Setting up a development environment diff --git a/docs/design/CIPs/CIPs.md b/docs/design/design_documents/design_doc_guidelines.md similarity index 100% rename from docs/design/CIPs/CIPs.md rename to docs/design/design_documents/design_doc_guidelines.md diff --git a/docs/design/overview.md b/docs/design/overview.md index 36b3843..3afea5a 100644 --- a/docs/design/overview.md +++ b/docs/design/overview.md @@ -8,4 +8,4 @@ Working in MXFusion breaks up into two primary phases. Model definition involves * [Inference](inference.md) ## Design Choices -* [CIPs](CIPs/CIPs.md) +* [Feature Submission](design_documents/design_doc_guidelines.md) From 41ecce8eb2c2df6cdb50a6fd3f54b2db81db17ae Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Mon, 22 Oct 2018 17:17:54 +0100 Subject: [PATCH 48/70] Update to inference documentation --- .../design_documents/design_doc_guidelines.md | 3 +- docs/design/inference.md | 103 +++++++++++++++++- mxfusion/inference/inference.py | 2 +- 3 files changed, 99 insertions(+), 9 deletions(-) diff --git a/docs/design/design_documents/design_doc_guidelines.md b/docs/design/design_documents/design_doc_guidelines.md index 4904139..3077130 100644 --- a/docs/design/design_documents/design_doc_guidelines.md +++ b/docs/design/design_documents/design_doc_guidelines.md @@ -1,6 +1,5 @@ -# CIPs +# Design Documents -CIPs are a design proposal mechanism from the Apache Software Foundation. ```eval_rst .. toctree:: diff --git a/docs/design/inference.md b/docs/design/inference.md index 00594f0..6b82d2d 100644 --- a/docs/design/inference.md +++ b/docs/design/inference.md @@ -1,15 +1,52 @@ # Inference -Notes about inference in MXFusion. +## Overview + +Inference in MXFusion is broken down into a few logical pieces that can be combined together as necessary. + +The highest level object you'll deal with will be derived from the ```mxfusion.inference.Inference``` class. This is the outer loop that drives the inference algorithm, holds the relevant parameters and models for training, and handles serialization after training. At a minimum, ```Inference``` objects take as input the ```InferenceAlgorithm``` to run and the model it will be run on. On creation, an ```InferenceParameters``` object is created and attached to the ```Inference``` method which will store and manage (MXNet) parameters during inference. + +Currently there are two main Inference subclasses: ```GradBasedInference``` and ```TransferInference```. An obvious third choice would be some kind of MCMC sampling Inference method. + +The first primary class of Inference methods is ```GradBasedInference```, which is for those methods use the gradient of some loss funtion in an optimization loop. We only use MXNet based optimizers for now. When using gradient based inference methods (```GradBasedInference```), the Inference class takes in a ```GradLoop``` in addition to the ```InferenceAlgorithm```. The ```GradLoop``` determines whether to use full or mini-batches of the data for the ```InferenceAlgorithm```. + +The second type of Inference method is ```TransferInference```. These are methods that take as an additional parameter the ```InferenceParameters``` object from a previous Inference method. An example of a ```TransferInference``` method is the ```VariationalPosteriorForwardSampling``` method, which takes as input a VariationalInference method that has already been trained and performs forward sampling through the variational posterior. + +A basic example to run variational inference with a meanfield posterior over some model looks like the following. See the next section for mathematical details on VariationalInference. + +### First Example + +First we create the model. The model creation function is dummy here, but this applies to almost any model. See the [Model Definiton](model_definition.md) file for details on model creation. Then we define the observed variables in our model, and apply the convenience method for creating a factorized Gaussian posterior to that model, and get the posterior ```q```. + +```py +m = make_model() +observed = [m.y, m.x] +q = create_Gaussian_meanfield(model=m, observed=observed) +``` + +Then we define what ```InferenceAlgorithm``` we want to run, and initialize it with the model, posterior, and observation pattern we defined above. This is used to initialize the ```GradBasedInference``` object, which creates a data structure to manage parameters of the model at this stage. + +```py +alg = StochasticVariationalInference(model=m, observed=observed, posterior=q) +infr = GradBasedInference(inference_algorithm=alg) +``` + +Then, we run the Inference method, passing in the data as keyword arguments, matching the observation pattern we defined previously. This will create and initialize parameters for the variational posterior and any model parameters, and optimize the standard KL-divergence loss function to match the variational posterior to the model's posterior. We run it for 1000 iterations. + +``` +infr.run(max_iter=1000, y=y, x=x) + +``` ## Inference Algorithms -MXFusion currently supports stochastic variational inference. +MXFusion currently supports stochastic variational inference. We provide a convenience method to generate a Gaussian meanfield posterior for your model, but the interface is flexible enough to allow defining a specialized posterior over your model as required. See the ```mxfusion.inference``` module of the documentation for a full list of supported inference methods. ### Variational Inference Variational inference is an approximate inference method that can serve as the inference method over generic models built in MXFusion. The main idea of variational inference is to approximate the (often intractable) posterior distribution of our model with a simpler parametric approximation, referred to as a variational posterior distribution. The goal is then to optimize the parameters of this variational posterior distribution to best approximate our true posterior distribution. This is typically done by minimizing the lower bound of the logarithm of the marginal distribution: + \begin{equation} \log p(y|z) = \log \int_x p(y|x) p(x|z) \geq \int_x q(x|y,z) \log \frac{p(y|x) p(x|z)}{q(x|y,z)} = \mathcal{L}(y,z), \label{eqn:lower_bound_1} \end{equation} @@ -28,10 +65,64 @@ where $y_i|z \sim q(y|z)$, $x_j|y_i,z \sim q(x|y_i,z)$ and $N$ is the number of Let's look at a simple model and then see how we apply stochastic variational inference to it in practice using MXFusion. ### Creating a Posterior - TODO +Variational inference is based around the idea that you can approximate your true model's, possibly complex, posterior distribution with an approximate variational posterior that is easy to compute. A common choice of approximate posterior is the Gaussian meanfield, which factorizes each variable as being drawn from a Normal distribution independent from the rest. + +This can be done easily for a given model by calling the ```mxfusion.inference.create_Gaussian_meanfield``` function and passing in your model. + +You can also define more complex posterior distributions to perform inference over if you know something more about your problem. See the [../../examples/notebooks/ppca_tutorial.ipynb](PPCA tutorial) for a detailed example of this process. + + +## Saving and Loading Inference Results + Saving and reloading inference results is managed at the ```Inference``` level in MXFusion. Once you have an ```Inference``` object that has been trained, you save the whole thing by running: + + ```py + inference.save('my_inference_prefix') + ``` + + This will save down all relevent pieces of the inference algorithm to files beginning with the prefix passed in at save time. These files include: MXNet parameter files, json files containing the model's topology, and any Inference configuration such as the number of samples it was run with. + +When reloading a saved inference method, you must re-run the code used to generate the original models and Inference method, and then load the saved parameters back into the new objects. An example is shown below: + +In process 1: +```py + +x = np.random.rand(1000, 1) +y = np.random.rand(1000, 1) + +m = make_model() + +observed = [m.y, m.x] +q = create_Gaussian_meanfield(model=m, observed=observed) +alg = StochasticVariationalInference(num_samples=3, model=m, observed=observed, posterior=q) +infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop()) +infr.initialize(y=y, x=x) +infr.run(max_iter=1, learning_rate=1e-2, y=y, x=x) + +infr.save(prefix=PREFIX) + +``` + +At some future time, in another process: +```py +x = np.random.rand(1000, 1) +y = np.random.rand(1000, 1) + +m2 = make_model() + +observed2 = [m2.y, m2.x] +q2 = create_Gaussian_meanfield(model=m2, observed=observed2) +alg2 = StochasticVariationalInference(num_samples=3, model=m2, observed=observed2, posterior=q2) +infr2 = GradBasedInference(inference_algorithm=alg2, grad_loop=BatchInferenceLoop()) +infr2.initialize(y=y, x=x) + +# Load previous parameters +infr2.load(primary_model_file=PREFIX+'_graph_0.json', + secondary_graph_files=[PREFIX+'_graph_1.json'], + parameters_file=PREFIX+'_params.json', + inference_configuration_file=PREFIX+'_configuration.json', + mxnet_constants_file=PREFIX+'_mxnet_constants.json', + variable_constants_file=PREFIX+'_variable_constants.json') -## Examples -* [PPCA](../../examples/notebooks/ppca_tutorial.ipynb) -## Saving Inference Results +``` diff --git a/mxfusion/inference/inference.py b/mxfusion/inference/inference.py index 9f9070a..85f388d 100644 --- a/mxfusion/inference/inference.py +++ b/mxfusion/inference/inference.py @@ -13,7 +13,7 @@ class Inference(object): Abstract class defining an inference method that can be applied to a model. An inference method consists of a few components: the applied inference algorithm, the model definition (optionally a definition of posterior - approximation), the inference parameters. + approximation), and the inference parameters. :param inference_algorithm: The applied inference algorithm :type inference_algorithm: InferenceAlgorithm From 38f166d2c52ba413d8925020e98eb5da2204c898 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Tue, 23 Oct 2018 14:42:21 +0100 Subject: [PATCH 49/70] Restructure design docs and add proposal guidelines --- .../design_documents/design_doc_guidelines.md | 10 ---- .../{design => design_documents}/inference.md | 0 .../model_definition.md | 0 docs/{design => design_documents}/overview.md | 0 .../design_proposal_guidelines.md | 47 +++++++++++++++++++ 5 files changed, 47 insertions(+), 10 deletions(-) delete mode 100644 docs/design/design_documents/design_doc_guidelines.md rename docs/{design => design_documents}/inference.md (100%) rename docs/{design => design_documents}/model_definition.md (100%) rename docs/{design => design_documents}/overview.md (100%) create mode 100644 docs/design_proposals/design_proposal_guidelines.md diff --git a/docs/design/design_documents/design_doc_guidelines.md b/docs/design/design_documents/design_doc_guidelines.md deleted file mode 100644 index 3077130..0000000 --- a/docs/design/design_documents/design_doc_guidelines.md +++ /dev/null @@ -1,10 +0,0 @@ -# Design Documents - - -```eval_rst -.. toctree:: - :glob: - :maxdepth: 1 - - * -``` diff --git a/docs/design/inference.md b/docs/design_documents/inference.md similarity index 100% rename from docs/design/inference.md rename to docs/design_documents/inference.md diff --git a/docs/design/model_definition.md b/docs/design_documents/model_definition.md similarity index 100% rename from docs/design/model_definition.md rename to docs/design_documents/model_definition.md diff --git a/docs/design/overview.md b/docs/design_documents/overview.md similarity index 100% rename from docs/design/overview.md rename to docs/design_documents/overview.md diff --git a/docs/design_proposals/design_proposal_guidelines.md b/docs/design_proposals/design_proposal_guidelines.md new file mode 100644 index 0000000..5b58c8e --- /dev/null +++ b/docs/design_proposals/design_proposal_guidelines.md @@ -0,0 +1,47 @@ +# Design Documents + + +```eval_rst +.. toctree:: + :glob: + :maxdepth: 1 + + * +``` + +## Overview +If you want to propose making a major change to the codebase rather than a simple feature addition, it's helpful to fill out and send around a design proposal document **before you go through all the work of implementing it**. This allows the community to better evaluate the idea, highight any potential downsides, or propose alternative solutions ahead of time and save unneeded effort. + +### What is considered a "major change" that needs a design proposal? + +Any of the following should be considered a major change: +* Anything that changes a public facing API such as model definition or inference. +* Any other major new feature, subsystem, or piece of functionality + +Example issues that might need design proposals include [#75](https://github.com/amzn/MXFusion/issues/75), +[#40](https://github.com/amzn/MXFusion/issues/40), +[#24](https://github.com/amzn/MXFusion/issues/24), or +[#23](https://github.com/amzn/MXFusion/issues/23). + +### Process to submit a design proposal +Fill out the template below, add it to this folder on your fork, and make a pull request against the main repo. If it is helpful, feel free to include some proof of concept or mockup code to demonstrate the idea more fully. The point isn't to have fully functioning or tested code of your idea, but to help communicate the idea and how it might look with the rest of the community. + +## Template + +The basic template should include the following things: + +### Motivation +Describe the problem to be solved. + +### Public Interfaces +Describe how this changes the public interfaces of the library (if at all). Also describe any backwards compatibility strategies here if this will break an existing API. + +### Proposed Changes +Describe the new thing you want to do. This may be fairly extensive and have large subsections of its own. Or it may be a few sentences, depending on the scope of the change. + +### Rejected Alternatives +What are the other alternatives you considered and why are they worse? The goal of this section is to help people understand why this is the best solution now, and also to prevent churn in the future when old alternatives are reconsidered. + +## Acknowledgements + +This process is heavily inspired and taken from the [Kafka Improvement Processes](https://cwiki.apache.org/confluence/display/KAFKA/Kafka+Improvement+Proposals). From a971b2d02e80caba52743f8676ef4d007656d5f5 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Tue, 23 Oct 2018 15:31:44 +0100 Subject: [PATCH 50/70] Update text from PR feedback --- CONTRIBUTING.md | 4 ++-- docs/design_documents/inference.md | 6 +++--- docs/design_documents/overview.md | 2 +- docs/design_proposals/design_proposal_guidelines.md | 2 ++ 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60e81c1..579a9f8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,13 +26,13 @@ If you're wishing for a feature that doesn't exist yet in MXFusion, there are pr If you're thinking about adding code to MXFusion, here are some guidelines to get you started. -* If the change is a major feature, create a [design document](design/design_documents/design_doc_guidelines) in the docs/design/design_documents folder and post it as a PR, optionally with a prototype implementation of your proposed changes. This is to get community feedback on the changes and document the design reasoning of MXFusion for future reference. +* If the change is a major feature, create a [design proposal](design_proposal/design_proposal_guidelines) in the design_proposals folder and post it as a PR, optionally with a prototype implementation of your proposed changes. This is to get community feedback on the changes and document the design reasoning of MXFusion for future reference. * Keep pull requests small, preferably one feature per pull request. This lowers the bar to entry for a reviewer, and keeps feedback focused for each feature. Some major areas where we appreciate contributions: * [Adding new Distributions/Functions/Modules](examples/notebooks/writing_a_new_distribution.ipynb) -* [Adding new Inference Algorithms](inference link TODO) +* [Adding new Inference Algorithms](design_documents/inference) * Example notebooks showing how to build/train a particular model. If you're still not sure where to begin, have a look at our [issues](issues TODO) page for open work. diff --git a/docs/design_documents/inference.md b/docs/design_documents/inference.md index 6b82d2d..505bfe0 100644 --- a/docs/design_documents/inference.md +++ b/docs/design_documents/inference.md @@ -4,15 +4,15 @@ Inference in MXFusion is broken down into a few logical pieces that can be combined together as necessary. -The highest level object you'll deal with will be derived from the ```mxfusion.inference.Inference``` class. This is the outer loop that drives the inference algorithm, holds the relevant parameters and models for training, and handles serialization after training. At a minimum, ```Inference``` objects take as input the ```InferenceAlgorithm``` to run and the model it will be run on. On creation, an ```InferenceParameters``` object is created and attached to the ```Inference``` method which will store and manage (MXNet) parameters during inference. +The highest level object you'll deal with will be derived from the ```mxfusion.inference.Inference``` class. This is the outer loop that drives the inference algorithm, holds the relevant parameters and models for training, and handles serialization after training. At a minimum, ```Inference``` objects take as input the ```InferenceAlgorithm``` to run. On creation, an ```InferenceParameters``` object is created and attached to the ```Inference``` method which will store and manage (MXNet) parameters during inference. Currently there are two main Inference subclasses: ```GradBasedInference``` and ```TransferInference```. An obvious third choice would be some kind of MCMC sampling Inference method. -The first primary class of Inference methods is ```GradBasedInference```, which is for those methods use the gradient of some loss funtion in an optimization loop. We only use MXNet based optimizers for now. When using gradient based inference methods (```GradBasedInference```), the Inference class takes in a ```GradLoop``` in addition to the ```InferenceAlgorithm```. The ```GradLoop``` determines whether to use full or mini-batches of the data for the ```InferenceAlgorithm```. +The first primary class of Inference methods is ```GradBasedInference```, which is for those methods that involve a gradient-based optimization. We only support the gradient optimizers that are available in MXNet for now. When using gradient-based inference methods (```GradBasedInference```), the Inference class takes in a ```GradLoop``` in addition to the ```InferenceAlgorithm```. The ```GradLoop``` determines how the gradient computed in the ```InferenceAlgorithm``` is used to update model parameters. The two available implementations of ```GradLoop``` are ```BatchInferenceLoop``` and ```MinibatchInferenceLoop```, which correspond to gradient-based optimization in batch or mini-batch mode. The second type of Inference method is ```TransferInference```. These are methods that take as an additional parameter the ```InferenceParameters``` object from a previous Inference method. An example of a ```TransferInference``` method is the ```VariationalPosteriorForwardSampling``` method, which takes as input a VariationalInference method that has already been trained and performs forward sampling through the variational posterior. -A basic example to run variational inference with a meanfield posterior over some model looks like the following. See the next section for mathematical details on VariationalInference. +A basic example to run variational inference with a meanfield posterior over some model looks like the following. See the next section for mathematical details on variational inference. ### First Example diff --git a/docs/design_documents/overview.md b/docs/design_documents/overview.md index 3afea5a..478da2f 100644 --- a/docs/design_documents/overview.md +++ b/docs/design_documents/overview.md @@ -8,4 +8,4 @@ Working in MXFusion breaks up into two primary phases. Model definition involves * [Inference](inference.md) ## Design Choices -* [Feature Submission](design_documents/design_doc_guidelines.md) +* [Design Proposal](../design_proposals/design_proposal_guidelines.md) diff --git a/docs/design_proposals/design_proposal_guidelines.md b/docs/design_proposals/design_proposal_guidelines.md index 5b58c8e..40e362d 100644 --- a/docs/design_proposals/design_proposal_guidelines.md +++ b/docs/design_proposals/design_proposal_guidelines.md @@ -12,6 +12,8 @@ ## Overview If you want to propose making a major change to the codebase rather than a simple feature addition, it's helpful to fill out and send around a design proposal document **before you go through all the work of implementing it**. This allows the community to better evaluate the idea, highight any potential downsides, or propose alternative solutions ahead of time and save unneeded effort. +For smaller feature requests just file an issue and fill out the feature request template. + ### What is considered a "major change" that needs a design proposal? Any of the following should be considered a major change: From dd85cc757fc3f90eff7dbba8ed55dbd82cc7217f Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Tue, 23 Oct 2018 15:39:30 +0100 Subject: [PATCH 51/70] Update link in CONTRIBUTING --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 579a9f8..8fd4a67 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,7 @@ Before submitting the pull request, please go through this checklist to make the * Do all public functions have docstrings including examples? If you added a new module, did you add it to the Sphinx docstring in the ```__init__.py``` file of the module's folder? * Is the code style correct (PEP8)? * Is the commit message formatted correctly? -* If this is a large addition, is there a tutorial or more extensive module-level description? Did you discuss the addition in a [design document](design/design_documents/design_doc_guidelines)? Is there an issue related to the change? If so, please link the issue or design doc. +* If this is a large addition, is there a tutorial or more extensive module-level description? Did you discuss the addition in a [design proposal](design_proposals/design_proposal_guidelines)? Is there an issue related to the change? If so, please link the issue or design doc. ## Setting up a development environment From 111cca0c29c13ae83b66d4e2edbf91e2e0b0378a Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Thu, 25 Oct 2018 17:20:46 +0100 Subject: [PATCH 52/70] Remove the dependency on future (#100) * Remove the dependency on future and mxnet. Resolve the potential issue of self-dependency in setup.py * Bump the version number. * Re-implement to be 3.4 and 3.5 compatible. * Fix import error. * Add numpy denpendency. * Applied the suggested changes. --- mxfusion/__version__.py | 2 +- mxfusion/models/factor_graph.py | 12 +++++++----- requirements/doc_requirements.txt | 3 +-- requirements/requirements.txt | 5 +++-- requirements/test_requirements.txt | 1 + setup.py | 11 +++++------ 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/mxfusion/__version__.py b/mxfusion/__version__.py index 7fd229a..fc79d63 100644 --- a/mxfusion/__version__.py +++ b/mxfusion/__version__.py @@ -1 +1 @@ -__version__ = '0.2.0' +__version__ = '0.2.1' diff --git a/mxfusion/models/factor_graph.py b/mxfusion/models/factor_graph.py index 34a0b31..4ecab0c 100644 --- a/mxfusion/models/factor_graph.py +++ b/mxfusion/models/factor_graph.py @@ -1,7 +1,7 @@ -from future.utils import raise_from from uuid import uuid4 import warnings import networkx as nx +from networkx.exception import NetworkXError import networkx.algorithms.dag from ..components import Distribution, Factor, ModelComponent, Variable, VariableType from ..modules.module import Module @@ -281,13 +281,15 @@ def remove_component(self, component): try: self.components_graph.remove_node(component) # implicitly removes edges - except Exception as e: - raise_from(ModelSpecificationError("Attempted to remove a node that isn't in the graph"), e) + except NetworkXError as e: + raise ModelSpecificationError("Attempted to remove a node "+str(component)+" that isn't in the graph.") if component.name is not None: + try: - self.__delattr__(component.name) - except Exception as e: + if getattr(self, component.name) is component: + delattr(self, component.name) + except AttributeError: pass component.graph = None diff --git a/requirements/doc_requirements.txt b/requirements/doc_requirements.txt index f64cf64..c6b72a9 100644 --- a/requirements/doc_requirements.txt +++ b/requirements/doc_requirements.txt @@ -4,5 +4,4 @@ sphinxcontrib-websupport>=1.1.0 recommonmark>=0.4.0 nbsphinx>=0.3.4 networkx>=2.1 -mxnet>=1.2 -future>=0.16.0 +mxnet>=1.3 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 576c19e..261f731 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,3 +1,4 @@ networkx>=2.1 -mxnet>=1.3 -future>=0.16.0 +numpy>=1.7 +# mxnet +# While MXFusion depends on MXNet, there are multiple versions in PyPi for GPU/MKL that are better managed by the user than in this requirements file. diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt index a92a371..d855858 100644 --- a/requirements/test_requirements.txt +++ b/requirements/test_requirements.txt @@ -7,3 +7,4 @@ scipy>=1.1.0 GPy>=1.8.5 matplotlib scikit-learn>=0.20.0 +mxnet>=1.3 diff --git a/setup.py b/setup.py index 89ffca5..c1f032a 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,12 @@ -import os from setuptools import setup, find_packages -from mxfusion.__version__ import __version__ +import re with open('README.md', 'r') as fh: long_description = fh.read() -with open('requirements/requirements.txt', 'r') as req: - requires = req.read().split("\n") +with open('mxfusion/__version__.py', 'r') as rv: + text = rv.read().split('=') + __version__ = re.search(r'\d+\.\d+\.\d+', text[1]).group() setup( name='MXFusion', # this is the name of the package as you will import it i.e import package-name @@ -19,10 +19,9 @@ url='https://github.com/amzn/MXFusion', packages=find_packages(exclude=['testing*']), include_package_data=True, - install_requires=requires, + install_requires=['networkx>=2.1', 'numpy>=1.7'], license='Apache License 2.0', classifiers=( - # https://pypi.org/pypi?%3Aaction=list_classifiers 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Intended Audience :: Education', From b5f82176ef06ad65860b2b5e65bfaf51086f4a7c Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Fri, 26 Oct 2018 07:40:06 +0100 Subject: [PATCH 53/70] Update examples/notebooks/getting_started.ipynb Co-Authored-By: zhenwendai --- examples/notebooks/getting_started.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/notebooks/getting_started.ipynb b/examples/notebooks/getting_started.ipynb index 82918ed..0d9eea4 100644 --- a/examples/notebooks/getting_started.ipynb +++ b/examples/notebooks/getting_started.ipynb @@ -17,7 +17,7 @@ "\n", "MXFusion is a probabilistic programming language. It provides a convenient interface for designing probabilistic models and applying them to real world problems.\n", "\n", - "Probabilistic models describe the relation in data through the probabilistic distribution of random variables. Probabilistic modeling is typically done through stating the prior believe in terms of a probabilistic model and performing inference with the observation of some of the random variables." + "Probabilistic models describe the relationships in data through probabilistic distributions of random variables. Probabilistic modeling is typically done by stating your prior belief about the data in terms of a probabilistic model and performing inference with the observations of some of the random variables." ] }, { From e966ad1e3ad8a9312882cdabee2cb6a1eecc5038 Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Fri, 26 Oct 2018 07:41:13 +0100 Subject: [PATCH 54/70] Update examples/notebooks/getting_started.ipynb Co-Authored-By: zhenwendai --- examples/notebooks/getting_started.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/notebooks/getting_started.ipynb b/examples/notebooks/getting_started.ipynb index 0d9eea4..96b16d2 100644 --- a/examples/notebooks/getting_started.ipynb +++ b/examples/notebooks/getting_started.ipynb @@ -120,7 +120,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In the above definition, we start with defining a model by instantiated from the class Model. The variable $\\mu$ and $s$ are created from the class Variable. Both of them are assigned as members of the model instance m. This is the way in which variables are organized in MXFusion. The variable s is created by assigning a PositiveTransformation instance to the transforamtion argument. This constrains the value of the variable s to be positive through a \"soft-plus\" transformation. The variable Y is created from a normal distribution by specifying the mean and variance and its shape. \n", + "In the above definition, we start with defining a model by instantiated from the class Model. The variable $\\mu$ and $s$ are created from the class Variable. Both of them are assigned as members of the model instance m. This is how variables are organized in MXFusion. The variable s is created by passing a PositiveTransformation instance to the transforamtion argument. This constrains the value of the variable s to be positive through a \"soft-plus\" transformation. The variable Y is created from a normal distribution by specifying the mean and variance and its shape. \n", "\n", "Note that, in this example, the mean and variance variable are both scalar, with the shape (1,), while the random variable Y has the shape (100,). This indicates the mean and variance variable are broadcasted into the shape of the random variable, just like the broadcasting rule in numpy array operation. In this case, this means the individual entries of the random variable Y follows a scalar normal distribution with the same mean and variance.\n", "\n", From 8e69d278e495e80d126a865e9050e513bebac94b Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Fri, 26 Oct 2018 07:41:40 +0100 Subject: [PATCH 55/70] Update examples/notebooks/getting_started.ipynb Co-Authored-By: zhenwendai --- examples/notebooks/getting_started.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/notebooks/getting_started.ipynb b/examples/notebooks/getting_started.ipynb index 96b16d2..5c1009c 100644 --- a/examples/notebooks/getting_started.ipynb +++ b/examples/notebooks/getting_started.ipynb @@ -189,7 +189,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "After optimiztaion, the estimated parameters are stored in an instance of the class InferenceParameters, which can be access from the Inference instance such as infr.params.\n", + "After optimization, the estimated parameters are stored in an instance of the class InferenceParameters, which can be access from an Inference instance by infr.params.\n", "\n", "We collect the estimated mean and variance and compared with the generating parameters." ] From 3cfc1b98a9a486e1a2c42ba82e0385021903eb5f Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Fri, 26 Oct 2018 07:42:16 +0100 Subject: [PATCH 56/70] Update examples/notebooks/getting_started.ipynb Co-Authored-By: zhenwendai --- examples/notebooks/getting_started.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/notebooks/getting_started.ipynb b/examples/notebooks/getting_started.ipynb index 5c1009c..0c45c7b 100644 --- a/examples/notebooks/getting_started.ipynb +++ b/examples/notebooks/getting_started.ipynb @@ -220,7 +220,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The estimated parameters are close to the generating parameters with a small difference. This difference is due to the small number of existing data, which is also known as *over-fitting*." + "The estimated parameters are close to the generating parameters, but still off by a small amount. This difference is due to the small size of dataset we used, a problem known as *over-fitting*." ] }, { From 5cc4ff03166aebbb9cb4edd501982ffc1e87a819 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 29 Oct 2018 10:24:31 +0000 Subject: [PATCH 57/70] The design doc for changing the broadcasting logic for distributions. (#102) --- .../design_proposals/design-1_broadcasting.md | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 docs/design_proposals/design-1_broadcasting.md diff --git a/docs/design_proposals/design-1_broadcasting.md b/docs/design_proposals/design-1_broadcasting.md new file mode 100644 index 0000000..150cf84 --- /dev/null +++ b/docs/design_proposals/design-1_broadcasting.md @@ -0,0 +1,57 @@ +# Changing the design of variable broadcasting for factors + +Zhenwen Dai (2018-10-26) + +## Motivation + +The current design of the interface for creating a probabilistic distribution in MXFusion supports automatic broadcasting if the shapes of one or all the input variables are smaller than the expected shape. For example, we can create a random variable of the size (3, 4) with the shape of its mean being (1,) and the shape of its variance being (4,) as follows: +```py +m = Model() +m.mean = Variable(shape=(1,)) +m.variance = Variable(shape=(4,)) +m.x = Normal.define_variable(mean=m.mean, variance=m.variance, shape=(3, 4)) +``` +Although it is a handy feature, it causes confusions for some users because they are not familiar with the common broadcasting rule. + +It also leads a big challenge when implementing new distributions. The current design requires users to write one function decorator for each of the two main APIs: log_pdf and draw_samples for any multi-variate distributions. Such a function decorator does two things: +1. If an input variable has a shape that is smaller than the expected one (computed from the shape of the random variable), it broadcasts the shape of the input variable to the expected shape. +2. For any the variables, the number of samples is not one. If the numbers of samples of more than one variables are more than one, the number of samples of these variables are assumed to be the same. It broadcast the size of the first dimension to the number of samples. + +This mechanism is complicated and makes it hard get contributions. + + +## Proposed Changes + +To simplify the logic of broadcasting, a natural choice is to let users take care of the first part of the broadcasting job. We still need to take care of the second part of the broadcasting as it is invisible from users. After the changes, the example with the new interface should be like +```py +m = Model() +m.mean = Variable(shape=(1,)) +m.variance = Variable(shape=(4,)) +m.x = Normal.define_variable(mean=broadcast_to(m.mean, (3, 4)), + variance=broadcast_to(m.variance, (3, 4)), shape=(3, 4)) +``` + +In the above example, the operator ```broadcast_to``` takes care of broadcasting a variable to another shape and the ```Normal``` instance expects that all the inputs have the same shape as the random variable, which is (3, 4) in this case. This simplifies the implementation of distributions. + +For broadcasting the number of samples, the mechanism for distributions and function evaluations can be unified. A boolean class attribute ```broadcastable``` will be added to ```Factor```. This attribute controls whether to expose the extra dimension of samples into internal computation logic. With this attribute being ```False```, a developer can implement the computation without the extra sample dimension, which is much straight-forward. + +With this simplification, the function decorator is not necessary anymore. The class structure of a distribution will look like +```py +class Distribution(Factor): + + def log_pdf(self, F, variables, targets=None): + # Take care of broadcasting of the number of samples. + # or looping through the number of samples + # call _log_pdf_implementation + + def _log_pdf_implementation(self, F, **kwargs): + # The inherited classes will implement the real computation here. +``` + +## Rejected Alternatives + +A possible solution to reduce the complexity of the broadcasting implementation without changing the interface is to add a shape inference function to each distribution. The shape inference will return the expected shapes of individual input variables given the shape of the random variable. Then, the ```broadcast_to``` operator just needs to be called inside the ```log_pdf``` function to hide the step of shape broadcasting. + +The drawbacks of this approach is +- The shape inference needs to be implemented for each multi-variate distribution. +- The broadcasting logic is not consistent with function evaluations and modules, which causes confusions to users. From 82421f1067a7ed784fcf977c77f8659bbf759001 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Mon, 29 Oct 2018 10:52:16 +0000 Subject: [PATCH 58/70] Add license headers to all files --- conftest.py | 15 + examples/notebooks/bnn_classification.ipynb | 23 +- examples/notebooks/bnn_regression.ipynb | 23 +- examples/notebooks/getting_started.ipynb | 23 +- .../ppca_tutorial-scorefunction-test.ipynb | 663 ------------------ examples/notebooks/ppca_tutorial.ipynb | 23 +- .../notebooks/variational_auto_encoder.ipynb | 23 +- .../writing_a_new_distribution.ipynb | 29 +- mxfusion/__init__.py | 15 + mxfusion/__version__.py | 15 + mxfusion/common/__init__.py | 15 + mxfusion/common/config.py | 15 + mxfusion/common/constants.py | 15 + mxfusion/common/exceptions.py | 15 + mxfusion/components/__init__.py | 15 + mxfusion/components/distributions/__init__.py | 15 + .../components/distributions/bernoulli.py | 17 +- mxfusion/components/distributions/beta.py | 15 + .../components/distributions/categorical.py | 15 + .../components/distributions/dirichlet.py | 15 + .../components/distributions/distribution.py | 15 + mxfusion/components/distributions/gamma.py | 15 + .../components/distributions/gp/__init__.py | 15 + .../components/distributions/gp/cond_gp.py | 15 + mxfusion/components/distributions/gp/gp.py | 15 + .../distributions/gp/kernels/__init__.py | 15 + .../distributions/gp/kernels/add_kernel.py | 15 + .../distributions/gp/kernels/kernel.py | 15 + .../distributions/gp/kernels/linear.py | 15 + .../gp/kernels/multiply_kernel.py | 15 + .../distributions/gp/kernels/rbf.py | 15 + .../distributions/gp/kernels/static.py | 15 + .../distributions/gp/kernels/stationary.py | 15 + mxfusion/components/distributions/normal.py | 15 + .../components/distributions/pointmass.py | 15 + .../components/distributions/random_gen.py | 15 + .../components/distributions/univariate.py | 15 + mxfusion/components/distributions/wishart.py | 15 + mxfusion/components/factor.py | 30 + mxfusion/components/functions/__init__.py | 15 + .../functions/function_evaluation.py | 15 + .../components/functions/gluon_func_eval.py | 15 + .../components/functions/mxfusion_function.py | 15 + .../functions/mxfusion_gluon_function.py | 15 + .../functions/operators/__init__.py | 14 + .../functions/operators/operator_impl.py | 15 + .../functions/operators/operators.py | 15 + mxfusion/components/model_component.py | 15 + mxfusion/components/variables/__init__.py | 15 + .../components/variables/runtime_variable.py | 15 + mxfusion/components/variables/var_trans.py | 15 + mxfusion/components/variables/variable.py | 15 + mxfusion/inference/__init__.py | 15 + mxfusion/inference/batch_loop.py | 30 + mxfusion/inference/expectation.py | 15 + mxfusion/inference/forward_sampling.py | 15 + mxfusion/inference/grad_based_inference.py | 15 + mxfusion/inference/grad_loop.py | 15 + mxfusion/inference/inference.py | 15 + mxfusion/inference/inference_alg.py | 15 + mxfusion/inference/inference_parameters.py | 15 + mxfusion/inference/map.py | 15 + mxfusion/inference/meanfield.py | 15 + mxfusion/inference/minibatch_loop.py | 15 + mxfusion/inference/prediction.py | 15 + mxfusion/inference/score_function.py | 15 + mxfusion/inference/variational.py | 15 + mxfusion/models/__init__.py | 15 + mxfusion/models/factor_graph.py | 15 + mxfusion/models/model.py | 15 + mxfusion/models/posterior.py | 15 + mxfusion/modules/__init__.py | 15 + mxfusion/modules/gp_modules/__init__.py | 15 + mxfusion/modules/gp_modules/gp_regression.py | 15 + .../modules/gp_modules/sparsegp_regression.py | 15 + .../modules/gp_modules/svgp_regression.py | 15 + mxfusion/modules/module.py | 15 + mxfusion/util/__init__.py | 15 + mxfusion/util/customop.py | 15 + mxfusion/util/graph_serialization.py | 15 + mxfusion/util/inference.py | 15 + mxfusion/util/special.py | 15 + mxfusion/util/testutils.py | 15 + mxfusion/util/util.py | 15 + testing/components/distributions/beta_test.py | 15 + .../distributions/categorical_test.py | 15 + .../distributions/dirichlet_test.py | 17 +- .../components/distributions/gamma_test.py | 15 + .../distributions/gp/cond_gp_test.py | 15 + .../components/distributions/gp/gp_test.py | 15 + .../distributions/gp/kernel_test.py | 15 + .../components/distributions/normal_test.py | 15 + .../components/distributions/wishart_test.py | 15 + testing/components/factor_test.py | 15 + .../functions/function_evaluation_test.py | 15 + .../functions/mxfusion_function_test.py | 15 + .../functions/mxfusion_gluon_function_test.py | 15 + .../components/functions/operators_test.py | 15 + testing/components/model_component_test.py | 15 + .../components/variables/var_trans_test.py | 17 +- testing/components/variables/variable_test.py | 15 + testing/distributions/bernoulli_test.py | 15 + .../normal_mean_precision_test.py | 17 +- testing/inference/expectation_test.py | 15 + testing/inference/forward_sampling_test.py | 15 + testing/inference/inference_alg_test.py | 15 + .../inference/inference_parameters_test.py | 15 + .../inference/inference_serialization_test.py | 15 + testing/inference/map_test.py | 15 + testing/inference/meanfield_test.py | 15 + testing/inference/score_function_test.py | 15 + testing/models/factor_graph_test.py | 15 + testing/modules/gpregression_test.py | 15 + testing/modules/sparsegpregression_test.py | 15 + testing/modules/svgpregression_test.py | 15 + testing/util/customop_test.py | 15 + testing/util/graph_serialization_test.py | 15 + testing/util/special_test.py | 15 + 118 files changed, 1836 insertions(+), 673 deletions(-) delete mode 100644 examples/notebooks/ppca_tutorial-scorefunction-test.ipynb diff --git a/conftest.py b/conftest.py index 0503461..d19bcb7 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import mxnet as mx import numpy as np import pytest diff --git a/examples/notebooks/bnn_classification.ipynb b/examples/notebooks/bnn_classification.ipynb index 94a3556..7db3d0b 100644 --- a/examples/notebooks/bnn_classification.ipynb +++ b/examples/notebooks/bnn_classification.ipynb @@ -7,6 +7,27 @@ "# Bayesian Neural Network (VI) for classification (under Development)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\").\n", + "# You may not use this file except in compliance with the License.\n", + "# A copy of the License is located at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# or in the \"license\" file accompanying this file. This file is distributed\n", + "# on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n", + "# express or implied. See the License for the specific language governing\n", + "# permissions and limitations under the License.\n", + "# ==============================================================================\n", + "```" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -411,7 +432,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.0" } }, "nbformat": 4, diff --git a/examples/notebooks/bnn_regression.ipynb b/examples/notebooks/bnn_regression.ipynb index 80b5395..e25eceb 100644 --- a/examples/notebooks/bnn_regression.ipynb +++ b/examples/notebooks/bnn_regression.ipynb @@ -9,6 +9,27 @@ "### Zhenwen Dai (2018-8-21)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\").\n", + "# You may not use this file except in compliance with the License.\n", + "# A copy of the License is located at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# or in the \"license\" file accompanying this file. This file is distributed\n", + "# on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n", + "# express or implied. See the License for the specific language governing\n", + "# permissions and limitations under the License.\n", + "# ==============================================================================\n", + "```" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -306,7 +327,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.0" } }, "nbformat": 4, diff --git a/examples/notebooks/getting_started.ipynb b/examples/notebooks/getting_started.ipynb index 0c45c7b..a9ecfd3 100644 --- a/examples/notebooks/getting_started.ipynb +++ b/examples/notebooks/getting_started.ipynb @@ -9,6 +9,27 @@ "**Zhenwen Dai (2018.10.22)**" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\").\n", + "# You may not use this file except in compliance with the License.\n", + "# A copy of the License is located at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# or in the \"license\" file accompanying this file. This file is distributed\n", + "# on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n", + "# express or implied. See the License for the specific language governing\n", + "# permissions and limitations under the License.\n", + "# ==============================================================================\n", + "```" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -426,7 +447,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.0" } }, "nbformat": 4, diff --git a/examples/notebooks/ppca_tutorial-scorefunction-test.ipynb b/examples/notebooks/ppca_tutorial-scorefunction-test.ipynb deleted file mode 100644 index fa9f14a..0000000 --- a/examples/notebooks/ppca_tutorial-scorefunction-test.ipynb +++ /dev/null @@ -1,663 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Probabilistic PCA Tutorial\n", - "This tutorial will demonstrate Probabilistic PCA, a factor analysis technique. \n", - "\n", - "Maths and notation following [Machine Learning: A Probabilistic Perspective](https://www.amazon.com/gp/product/0262018020).\n", - "\n", - "## Installation\n", - "Follow the instrallation instructions in the [README](../../README.md) file to get setup." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "## Probabalistic Modeling Introduction\n", - "\n", - "Probabilistic Models can be\n", - "categorized into directed graphical models (DGM, Bayes Net) and undirected\n", - "graphical models (UGM). Most popular probabilistic models\n", - "are DGMs, so MXFusion will only support the definition of\n", - "DGMs unless there is a strong customer need of UGMs in future.\n", - "\n", - "A DGM can be fully defined using 3 basic components: deterministic functions,\n", - "probabilistic distributions, and random variables. We show the interface for\n", - "defining a model using each of the three components below." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First lets import the basic libraries we'll need to train our model and visualize some data." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "os.environ['MXNET_ENGINE_TYPE'] = 'NaiveEngine'" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import warnings\n", - "warnings.filterwarnings('ignore')\n", - "import mxfusion as mf\n", - "import mxnet as mx\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Data Generation\n", - "We'll take as our function to learn components of the [log spiral function](https://en.wikipedia.org/wiki/Logarithmic_spiral) because it's 2-dimensional and easy to visualize." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "def log_spiral(a,b,t):\n", - " x = a * np.exp(b*t) * np.cos(t)\n", - " y = a * np.exp(b*t) * np.sin(t)\n", - " return np.vstack([x,y]).T" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We parameterize the function with 100 data points and plot the resulting function." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "N = 100\n", - "D = 100\n", - "K = 2\n", - "\n", - "a = 1\n", - "b = 0.1\n", - "t = np.linspace(0,6*np.pi,N)\n", - "r = log_spiral(a,b,t)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(100, 2)" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "r.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFEFJREFUeJzt3VuMXdddx/Hf33ZMlTalJm4VFHts\noqhFIW1pMw1GEZAoJUqpaR7ghaaR2iqyqNIoEQlRLgIhXkCq0otUS8hyg5CwVFDrElRS6gRcBA8O\nmTEJIQmOjJVpHVLl0qlaKVKdwX8ezkwzHp/r3mvvdft+njwX77P2OWd+a63/Wnsfc3cBAMqxKXYD\nAABhEewAUBiCHQAKQ7ADQGEIdgAoDMEOAIUh2AGgMAQ7ABSGYAeAwmyJ8aDbt2/33bt3x3hoAMjW\n4uLiq+7+zkm/FyXYd+/erYWFhRgPDQDZMrOlaX6PUgwAFIZgB4DCEOwAUBiCHQAKQ7ADQGEIdgAo\nDMGOXi0uLWv/0ZNaXFqO3RSgWMH2sZvZZkkLkl50972hjotyLC4t6+aDx3Rm5ay2btmkQ7fu0VW7\ntgU57rFTr2nPZRcHOR6Qu5AXKN0h6TlJbw94TEQUOjCPnXpNZ1bO6qxLb6yc1bFTr7U+bledBZCz\nIKUYM9sh6aOSDoY4HuJbC8wHj5zQzQePBSmd7LnsYm3dskmbTbpgyybtuezi1scc1lkAtQs1Yv+i\npHskXRToeIisi9H1Vbu26dCte4LOAtY6izdWzgbrLIDctQ52M9sr6WV3XzSza8f83j5J+yRpbm6u\n7cOiY10F5lW7tgUtlXTRWVCzR+7M3dsdwOzPJN0iaUXSWzSosR9290+M+j/z8/POTcDSV2PAUbNH\nysxs0d3nJ/1e6xG7u98n6b7VB71W0t3jQh35CD26zkEXJSigb+xjLwT7w8PoYoEX6FvQ+7G7+3ck\nfSfkMTEZ5YNwuqjZA32L8kEbCIvyQVg1lqBQFkoxBaB8kC5KZIiBEXsBKB+kiRIZYiHYC0H5ID2U\nyBALpRigI5TIEAsjdqAjlMgQC8EeUY1XdtaGEhliINgjYWENQFeosUfC7WYBdIVgj4SFNQBdoRQT\nCQtrmAbrMGiCYI+IhTWMwzoMmqIUAySKdRg0RbADiWIdBk1RigESxToMmiLYgYSxDoMmKMUAQGEI\n9oa4zzaAVFGKaYBtaABSxoi9AbahAUgZwd4A29AApIxSTANsQ0PKuA0BCPaG2IaGFLH+A4lSDFAU\n1n8gEexAUVj/gUQpBigK6z+QCHagOKz/gFIMABSGYAeAwhDsSAr34AHao8aOTs1yscyse7C5EAcY\nrupgJxi6NWtQD9uDPer3uRAHGK11KcbMdprZUTN71syeMbM7QjSsa2vB8OCRE7r54DGm/h2Y9WKZ\nWfZgz3JsyjuoTYgR+4qku9z9uJldJGnRzB5192cDHLszs4wOca5pZzprQf3GytmpLpaZZQ/2tMdm\nZD8aM9ZytQ52d39J0kur//6xmT0n6VJJSQf7rKGDgVmCssnFMtPuwZ722HTgw9HhlS1ojd3Mdkv6\ngKTHh/xsn6R9kjQ3NxfyYRvhCr1mZg3KLi+WmebYs3TgNY1g6fDKFizYzextkr4u6U53/9HGn7v7\nAUkHJGl+ft5DPW4bXKE3u9xmOtN24LWNYHN7HTGbIMFuZhdoEOqH3P1wiGMiTTnOdKbpwGsbweb4\nOmJ6rYPdzEzSVyQ95+6fb98kxDJtKaLEmU6NI9gSX0cMhBixXyPpFklPm9mTq9+7390fCXBs9KS2\nUsRGjGBRkhC7Yv5NkgVoCyKqrRQxzDQj2JoWWJGvqq88xZtqLEXMqvZZDfJBsEMSpYhpMKtBLgh2\n/BSLaeMxq0EuCPZKUBtuj1kNckGwV4DacDjjZjV0nkgFwV4BasPdK63zpJPKW3HBzhvyfNSGu1dS\n51laJ1WjooKdN+Rw1Ia7V1LnWVInVauigp035GjseOlWSZ1nSZ1UrYoKdt6Q5RlWWku13FZK51lS\nJ1Urc+//Drrz8/O+sLDQybFT/aPvWs7nPartw0prkkaW23J+DoBpmNmiu89P+r2iRuxSOaOmWeS0\ntrAxfMe1fdTnmg4rt6X8HNDhoG/FBXuNcllbGBa+49o+qrQ27HujjhM7VFPucFAugr0AKa4tDAvU\nYeE7ru2jar3DvjfsOCmEai6dLspCsBcgtcWuUYE6LHwntX1YaW3U9zYeZ//Rk9FDNcVOF+Uj2AuR\n0trCqFHqqBAP1faNxxkVqn2WZ1LrdFEHgh3BTSqv9BVuw0I1RnkmpU4XdSDY0drGEXBKo9SNoUrN\nGzUg2DMTe5fHsPYMGwGnOkodtcja13Oa2uuHMhHsGUlhl8dGuY2AN84mpNEXPIWW4us3LTqkvBDs\nGUkxRHPc9bF+NtHnzpkUX79p5Nwh1Ypgz0gqIbpx9JZKPb2Jjc/ptgu3av/Rk52cSyqv36xy7ZBq\nRrBnJIUQHVdTz9H653TbhVv1p998prORaQqvXxO5dkg1I9gzEztESxy9rT2nfZRlYr9+TeTaIdUs\n22BnMSeOkkdvJZ9bWzl2SDXL8ra9LOb0b31HKqnYTnXtPLdduFXLr5/p7BwZmKCJom/bW2I5IGXD\nOtLbrrs8drM6sfY+6nLgwMAEXdsUuwFNrE2ZN5uYMvdg1H3RS9X1+db2fKJ/WY7Ya1vMiT1tr632\n3PX51vZ8on9Z1thrksq0PXbn0rfFpWUdPn5aLul3Prgj+DnX9nwijKJr7DWJvZ6wPoBKrauP8vXj\np3Vm5awOHz/dyZ52Ah1dCVJjN7MbzeyEmZ00s3tDHBMDMdcT1mYLDx45oZsPHtPi0nJvjx0bdXDk\nrPWI3cw2S9ov6TclnZb0hJn9vbs/2/bYiLueEHu2EBN1cOQsRCnmakkn3f2UJJnZVyXdJIlgDyTW\ntL3mcLtq1zb98d5f0rf+6yV95Mqfr77Gnlt7U9XX8xgi2C+V9L11X5+W9CsBjovIatt9tN7i0vJP\n7xvzxAs/0HsuuSjY+aeyID6t3Nqbqj6fx972sZvZPjNbMLOFV155pa+HRQs1j9K6rLHnVr/Prb2p\n6vN5DDFif1HSznVf71j93jnc/YCkA9Jgu2OAx0WHah+ldVmGyq3ElVt7U9Xn89h6H7uZbZH0vKTr\nNQj0JyR93N2fGfV/2Meevv1HT+rBIyd01qXNJv3BDe+pbrtjlzOW3GZDubU3VW2fx972sbv7ipl9\nVtK3JW2W9NC4UMf0Yv4xMUrrdtE6t33subU3VX09j0EuUHL3RyQ9EuJYGIhdCql54RTIHVeeJiqF\nPeSM0sKjpIE+EOyJSqUUQhCFE3sWhnpkFew1hUwKpRCCKKwUZmGoQzbBXmPIxC6FEERhpTILQ/my\nCXZCpn8EUVgpzMJQh2yCnZDpH0EUXuxZGOqQ1Qdt1FRjTw3PfXM5Pnc5trkGRX7QBqOdOGpc3wgl\nx+cuxzbjXFl+mHVNFpeWtf/oyagfcsFNoJrL8bnLsc04V1Yj9tqkMnJifaO5HJ+7HNuMcxHsCUtl\nJxCLqM3l+Nzl2Gaci2BPWEojp/XrGyyszSbHtaEc24w3EewJS3HklEp5KFV0ekgBwZ641EZOqZSH\nUkSnh1SwKwYzWSsPbTZFLw+lht0kSAUjdsxkVHmIEkRaayKoW1ZXniJNlCDeRAeHLhV55SnSVGPd\nfVSAp7Ymgv6l0LkT7JlJ4U2zUW0liJJmKCm+n3KWynuDYM9IKm+ajcZtyywxOEqZoaT6fspZKu8N\ngj0jqbxphhlWgig1OEqZoaT8fspVKu+NLIO9xFHgNFJ500wr5+AY9x5L8cKxJnJ7P+UglfdGdrti\nSh0FTiunTm3ttVoLjo2vVarnUtN7LNXXAMMVuysm51FgCDntuphUe48dnqNCrab3WE7vJ0wvu2Bn\n+piXUcExKTy7HkmO61h4jyF32QV7KjUstDMuPKcZzU8K/kk/H9ex8B5D7rILdonpYwnGhec0o/lx\nwT9NxzBpVM57DDnLMtgxXG4LYaPCc1LoTgr+aWrkjMpRMoK9ECksRoYyKXQnBf+0NXJG5SgVwV6I\n0nZyjAvdScHPaDy/2RvCItgLUdtOjkmj7ZpH4yXN3tBMq2A3s89J+m1JZyT9j6RPufsPQzQMs2GU\nijWlzd4wu7afoPSopCvd/X2Snpd0X/smoamrdm3Tbdddzh9x5fiUK7Qasbv7kXVfHpP0u+2aA6At\nZm8IWWP/tKS/CXg8AA3VvMaAKYLdzB6TdMmQHz3g7g+v/s4DklYkHRpznH2S9knS3Nxco8aiHXZK\nAHWYGOzu/uFxPzezT0raK+l6H3OrSHc/IOmANLi742zNRFvslADq0Wrx1MxulHSPpI+5++thmoQu\nDNspAaBMbXfFfFnSRZIeNbMnzewvArQJHWCnRP4Wl5a1/+hJLS4tx24KEtd2V8zloRoSErXk87FT\nIm+U0jCL4q485Q9gNHZK5IuLjuLJcaBYXLDzB4AS1XbLiFTkOlAsLtj5A0CJKKXFketAsbhg5w8g\njBynn6WjlNa/XAeKNmbreWfm5+d9YWGh98fFdHKdfgJdSGmQY2aL7j4/6feKG7GjvVynn7lIKSgw\nWY4zJYId58l1+pkDZkPoA8GO87BO0R1mQ+gDwY6hmkw/KTFMxmwIfSDYEQQlhukwG0IfCHYEUWOJ\noekMJcfFOOSFYEcQtZUYmKEgZQQ7gmhTYsixNl/jDAX5INgRTNMF11gj3zYdSm0zFOSFYEdUbUe+\nTcO5bYfCIihSRrAjqjYj3zbhHKKUwiIoUkWwI6o2I9824UwpBSWrPthzXLgrTdORb5twppSCklV9\nd0e2rOWPjhk14e6OU2DLWv6ocwPn2xS7ATGtTeU3m6izAihG1SN26qxAuWou01Ud7BJTeaBEta+f\nVV2KAVCmYetnNSHYARSn9vWz6ksxAMpT+/oZwQ6gSDWvn1GKAYDCEOwAUBiCHQAKQ7ADQGEIdgAo\nTJBgN7O7zMzNbHuI4wEAmmsd7Ga2U9INkr7bvjl5Wlxa1v6jJ7W4tBy7KQAQZB/7FyTdI+nhAMfK\nTu33pACQnlYjdjO7SdKL7v5UoPZkp/Z7UgBIz8QRu5k9JumSIT96QNL9GpRhJjKzfZL2SdLc3NwM\nTUwbn50JIDWNPxrPzN4r6Z8kvb76rR2S/lfS1e7+/XH/N5WPxgul5vs+A+hP5x+N5+5PS3rXugd8\nQdK8u7/a9Ji5qvmeFEAIDI7C4iZgAKJiA0J4wS5QcvfdNY7WAbTDBoTwuPIUQFS1fyhGFyjFAIiq\n9g/F6ALBDiA6NiCERSkGAApDsANAYQj2BHFTMQBtUGNPDHt6AbTFiD0x7OkF0BbBnhj29AJoi1JM\nYtjTC6Atgj1B7OlFSrhBV34IdgAjsZifJ2rsAEZiMT9PBDuAkVjMzxOlGAAjsZifJ4IdwFgs5ueH\nUkyFuGUBUDZG7JVhlwNQPkbslWGXA1A+gr0y7HIAykcppjLscsgfV4JiEoK9QuxyyBdrJJgGpRgg\nI6yRYBoEOzrDtsrwWCPBNCjFoBOUDLrBGgmmQbCjE8NKBiWGUIyFTNZIMAnBjk6slQzeWDnbW8mg\n75BlVoJUEezoRN8lgxghW8usBPkh2NGZPksGMUI2xqwEmAbBjiLECFkWMpEqc/feH3R+ft4XFhZ6\nf1yUjSsyUTozW3T3+Um/13rEbma3S7pN0v9J+gd3v6ftMYEm2C0CDLQKdjO7TtJNkt7v7j8xs3eF\naRYAoKm2V55+RtKfu/tPJMndX27fJABAG22D/d2Sfs3MHjezfzGzD4VoFACguYmlGDN7TNIlQ370\nwOr//zlJeyR9SNLfmtllPmRF1sz2SdonSXNzc23aDAAYY2Kwu/uHR/3MzD4j6fBqkP+7mZ2VtF3S\nK0OOc0DSAWmwK6ZxiwEAY7UtxfydpOskyczeLWmrpFfbNgoA0FyrfexmtlXSQ5J+WdIZSXe7+z9P\n8f9ekbTU+IG7sV1ldkolnhfnlI8SzyvmOe1y93dO+qUoFyilyMwWptn4n5sSz4tzykeJ55XDOfFB\nGwBQGIIdAApDsL/pQOwGdKTE8+Kc8lHieSV/TtTYAaAwjNgBoDAE+xBmdpeZuZltj92Wtszsc2b2\n32b2n2b2DTN7R+w2tWFmN5rZCTM7aWb3xm5PW2a208yOmtmzZvaMmd0Ru02hmNlmM/sPM/tm7LaE\nYGbvMLOvrf49PWdmvxq7TaMQ7BuY2U5JN0j6buy2BPKopCvd/X2Snpd0X+T2NGZmmyXtl/QRSVdI\n+j0zuyJuq1pbkXSXu1+hwa05bivgnNbcIem52I0I6EuS/tHdf1HS+5XwuRHs5/uCpHskFbH44O5H\n3H1l9ctjknbEbE9LV0s66e6n3P2MpK9qcNvobLn7S+5+fPXfP9YgLC6N26r2zGyHpI9KOhi7LSGY\n2c9K+nVJX5Ekdz/j7j+M26rRCPZ1zOwmSS+6+1Ox29KRT0v6VuxGtHCppO+t+/q0CgjBNWa2W9IH\nJD0etyVBfFGDAdLZ2A0J5Bc0uAfWX66Wlw6a2VtjN2qU6j7zdMLdKu/XoAyTlXHn5O4Pr/7OAxpM\n+w/12TZMx8zeJunrku509x/Fbk8bZrZX0svuvmhm18ZuTyBbJH1Q0u3u/riZfUnSvZL+KG6zhqsu\n2EfdrdLM3qtBr/yUmUmDksVxM7va3b/fYxNnNu4OnJJkZp+UtFfS9cNuqZyRFyXtXPf1jtXvZc3M\nLtAg1A+5++HY7QngGkkfM7PfkvQWSW83s792909EblcbpyWddve12dTXNAj2JLGPfQQze0HSvLtn\nfQMjM7tR0ucl/Ya7n3c75ZyY2RYNFoCv1yDQn5D0cXd/JmrDWrDBKOKvJP3A3e+M3Z7QVkfsd7v7\n3thtacvM/lXSre5+wsz+RNJb3f0PIzdrqOpG7BX6sqSfkfTo6kzkmLv/ftwmNePuK2b2WUnflrRZ\n0kM5h/qqayTdIulpM3ty9Xv3u/sjEduE4W6XdGj1rranJH0qcntGYsQOAIVhVwwAFIZgB4DCEOwA\nUBiCHQAKQ7ADQGEIdgAoDMEOAIUh2AGgMP8PWeJUVeZqancAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(r[:,0], r[:,1],'.')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We now our project our $K$ dimensional ```r``` into a high-dimensional $D$ space using a random matrix of random weights $W$. Now that ```r``` is embedded in a $D$ dimensional space the goal of PPCA will be to recover ```r``` in it's original low-dimensional $K$ space." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "w = np.random.randn(K,N)\n", - "x_train = np.dot(r,w) + np.random.randn(N,N) * 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "# from sklearn.decomposition import PCA\n", - "# pca = PCA(n_components=2)\n", - "# new_r = pca.fit_transform(r_high)\n", - "# plt.plot(new_r[:,0], new_r[:,1],'.')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can explore the higher dimensional data manually by changing ```dim1``` and ```dim2``` in the following cell." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEICAYAAACj2qi6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XuQXOV95vHvo8tIYiQkBINA4m6D\nbJws2J4A3iWxEwgGyhXiXcfBm3Vw7C3FjtmKd5NKcKh1KHuTWufmrINtgi+xk/IFJw4xRbABJ3FI\nsoFYsBIGg0BgYWkM0iDQfSSh0W//eE8zPa3untOX03165vlUTU3POW+f8/bpnvd33msrIjAzM8tj\nXr8zYGZmg8NBw8zMcnPQMDOz3Bw0zMwsNwcNMzPLzUHDzMxyc9Cw0pD0C5LuKejYn5f0v4o4dp1z\nvUvSP7eQfouky4vMk1m3OGhYT0m6VNL/lbRb0guS/kXSjwFExBcj4ooS5PHbkv5rv/NRj6SQ9Moe\nn/Os7LwLenleKyd/CKxnJB0P3Am8D/gqMAT8OHCon/kys/xc07BeOg8gIr4cEZMRMRER90TEw3Bs\ns052d/srkp6UtFfSRyS9Iqup7JH0VUlD9Z5b9fxj7solnSDpTknjkl7MHp+W7fsdUiC7WdI+STdn\n218l6d6sdrRJ0turjneipDuyPP0b8IpmF0HSOyU9I2mnpBtr9l0k6V8l7ZL0rKSbq17jfVmyjVne\nfr7Za2lw7t+UNJZdz02SLsu2z5N0g6Snsnx9VdLK7GmV8+7KzvuGZq/PZjcHDeulJ4BJSV+QdJWk\nE3I8583A64FLgN8AbgX+C3A68CPAO9rIxzzgz4AzgTOACeBmgIi4Efgn4PqIWBoR10saBu4FvgSc\nDFwLfFLS+dnxPgEcBE4F3p391JU951PAO4HVwIlAdSE/Cfx34CTgDcBlwK9kefuJLM0FWd5ua/Za\n6px7LXA98GMRsYx0bbdku/8b8LPAG7N8vZi9LoDKeVdk5/3XRq/PZj8HDeuZiNgDXAoE8GlgPLtD\nX9Xkab8XEXsi4lHgEeCeiHg6InYD3wBe20Y+dkbE1yLiQETsBX6HVFg28hZgS0T8WUQciYj/B3wN\n+DlJ84H/BHwoIvZHxCPAF5oc623AnRFxX0QcAv4ncLQqbw9GxP3ZebYAf9osby2+lklgEXC+pIUR\nsSUinsr2vRe4MSK2Zfm6CXib+zGsloOG9VREPBYR74qI00g1hdXAHzd5yvaqxxN1/l7aah4kHSfp\nT7Mmoj2k5pcVWQCo50zg4qzJaJekXcAvAKcAI6S+wa1V6Z9pcvrV1WkjYj+wsypv52VNTM9leftd\nUq2j49cSEZuBD5ACwg5JX5G0uuo13l71+h4jBZlmAd3mIAcN65uIeBz4PCl4dGo/cFzlD0mnNEn7\na8Ba4OKIOJ6p5hdVslaTfivwjxGxoupnaUS8DxgHjpCayyrOaHLuZ6vTSjqO1ERV8SngceDcLG+/\nVZWvdl7LNBHxpYi4lBQkAvho1Wu8quY1Lo6IMY69HjaHOWhYz2Sdyb9W1el8OqlP4v4uHH4j8BpJ\nF0paTLqbbmQZqZayK+vs/e2a/duBc6r+vhM4L+vAXpj9/JikV0fEJPDXwE3ZXf/5wHVNzv1XwFuU\nhh4PAR9m+v/hMmAPsE/Sq0gjzZrlbabX8jJJayX9lKRFpD6YCaaaxm4BfkfSmVnaEUnXZPvGs3Tn\n1B7T5h4HDeulvcDFwAOS9pOCxSOku+WORMQTpAL4W8CTQLPJdX8MLAGez/LwzZr9/4fUnv+ipI9n\nfQVXkDrAfwg8R7pDX5Slv57UTPYcqeb0Z03y+SjwflKn+rOkDudtVUl+HfjPpGv1aeC2mkPcBHwh\na0Z6e47XUm0R8L+ztM+ROvU/WPWa7wDukbQ3O9bFWZ4PkPpK/iU77yVNzmGznPwlTGZmlpdrGmZm\nlltXgoakz0naIemRqm0rs8lQT2a/647Jl3RdluZJSc3ags3MrM+6VdP4PHBlzbYbgL+LiHOBv8v+\nnqaq4+5i4CLgt3NO+DIzsz7oStCIiPuAF2o2X8PUJKcvkGab1nozcG9EvBARL5Jm3dYGHzMzK4ki\nZ3uuiohns8fPUX+S0BqmT4ralm07hqR1wDqA4eHh17/qVa/qYlbNzGa/Bx988PmIGOnkGD1ZIiAi\nQlJHw7Qi4lbSukOMjo7G+vXru5I3M7O5QlKz1QpyKXL01HZJpwJkv3fUSTPG9Jm0p2XbzMyshIoM\nGncwNTP2OuDrddLcDVyRLe98AmkC1d0F5snMzDrQrSG3Xwb+FVgraZuk95Bmnv60pCeBy7O/kTQq\n6TMAEfEC8BHgO9nPh7NtZmZWQgM5I9x9GmZmrZP0YESMdnIMzwg3M7PcHDTMzCw3Bw0zM8vNQcPM\nzHJz0DAzs9wcNMzMLDcHDTMzy81Bw8zMcnPQMDOz3Bw0zMwsNwcNMzPLzUHDzMxyc9AwM7PcHDTM\nzCw3Bw0zM8vNQcPMzHJz0DAzs9wcNMzMLLdCg4aktZI2VP3skfSBmjRvkrS7Ks2HisyTmZm1b0GR\nB4+ITcCFAJLmA2PA7XWS/lNEvKXIvJiZWed62Tx1GfBURDzTw3OamVkX9TJoXAt8ucG+N0jaKOkb\nkl7TwzyZmVkLehI0JA0BPwP8ZZ3dDwFnRsQFwJ8Af9PgGOskrZe0fnx8vLjMmplZQ72qaVwFPBQR\n22t3RMSeiNiXPb4LWCjppDrpbo2I0YgYHRkZKT7HZmZ2jF4FjXfQoGlK0imSlD2+KMvTzh7ly8zM\nWlDo6CkAScPATwO/XLXtvQARcQvwNuB9ko4AE8C1ERFF58vMzFpXeNCIiP3AiTXbbql6fDNwc9H5\nMDOzznlGuJmZ5eagYWZmuTlomJlZbg4aZmaWm4OGmZnl5qBhZma5OWiYmVluDhpmZpabg4aZmeXm\noGFmZrk5aJiZWW4OGmZmlpuDhpmZ5eagYWZmuTlomJlZbg4aZmaWm4OGmZnl5qBhZma5OWiYmVlu\nhQcNSVskfVfSBknr6+yXpI9L2izpYUmvKzpPZmbWngU9Os9PRsTzDfZdBZyb/VwMfCr7bWZmJVOG\n5qlrgD+P5H5ghaRT+50pMzM7Vi+CRgD3SHpQ0ro6+9cAW6v+3pZtm0bSOknrJa0fHx8vKKtmZtZM\nL4LGpRHxOlIz1Psl/UQ7B4mIWyNiNCJGR0ZGuptDMzPLpfCgERFj2e8dwO3ARTVJxoDTq/4+Ldtm\nZmYlU2jQkDQsaVnlMXAF8EhNsjuAX8xGUV0C7I6IZ4vMl5mZtafo0VOrgNslVc71pYj4pqT3AkTE\nLcBdwNXAZuAA8EsF58nMzNpUaNCIiKeBC+psv6XqcQDvLzIfZmbWHWUYcmtmZgPCQcPMzHJz0DAz\ns9wcNMzMLDcHDTMzy81Bw8zMcnPQMDOz3Bw0zMwsNwcNMzPLrVdfwmTWFxMTMD4OBw/C4sUwMgJL\nlvTu+WazjWsaVnoTE/CDH8ATT6TfExP5n/fMMzA5CcPD6fczz/Tu+WazkYOGlVonBff4OCxalH6k\nqcd5v8Or0+dXv4Z2gp5ZGTloWE+1WoB2UnAfPAhDQ9O3DQ2l7Xl0+nxwbcVmHwcN65l2CtBOCu7F\ni+Hw4enbDh9O2/Po9PnQvdqKWVk4aFjPtFOAdlJwj4zAoUPpJ2Lqcd5vC+70+dCd2kqFm7msDBw0\nrGfaKUA7KbiXLIEzz4T582H//vT7zDPzj37q9PnQndoKuJnLysNDbq0r8gxNrRSgixZNbZupAK0U\n3OPjqeBevLj1gv+MM1p/Pd16/shIKtwhBcjDh1PQO/PM1o5TXUuDqd/j453lz6xVrmlYx/LeBbdb\na6gU3Oedl34P0jyJbtRWoLvNXGadKCxoSDpd0j9I+p6kRyX9ap00b5K0W9KG7OdDReXHipO3r6Jb\nBeig6UbQ61YzF7hvxDpTZPPUEeDXIuIhScuAByXdGxHfq0n3TxHxlgLzYQU7eDDVMKoNDaXAUKvT\n5p65qlvNXJVa4aJF6T07fDj9PReCt3VHYTWNiHg2Ih7KHu8FHgPWFHU+K1azu9Nu3gVbfd2qpXkI\nsHWqJ30aks4CXgs8UGf3GyRtlPQNSa9pcox1ktZLWj/uT3hPzdRn0Y2hqTazbjRzuW/EOlV40JC0\nFPga8IGI2FOz+yHgzIi4APgT4G8aHScibo2I0YgYHXFp1FMz3Z3O1b6KQeRaoXWq0CG3khaSAsYX\nI+Kva/dXB5GIuEvSJyWdFBHPF5kva02ePgv3VQyGbvSNeOXfua3I0VMCPgs8FhF/1CDNKVk6JF2U\n5WdnUXmy9vjudPbotFboSYZWZE3jPwDvBL4raUO27beAMwAi4hbgbcD7JB0BJoBrIyIKzJPlUHsn\nuXQp7NiR9nUycsfKoZNaoScZWmFBIyL+GdAMaW4Gbi4qD9a6ekMyd+yAk0+Gffvam5VdJjM1reRp\nepnLzTOtDK+22cnLiNg0je4k9+0b/DvJmeYo5JnDkHeew2wNLO0sBWOzi5cRsWlmw5DMRnNKZhoF\nlmcOQ540edv9B3FmtodXm4OGTTNond61Be8LLzQusGcKiHkCZp403QwsZePh1ebmKZumW8tV9EK9\npqKNG2HVqvodtTM1reRpesmTJk+7/yB3KHfSkT5bm+3mEtc07Bjz5qXC+Mkn4aWXynsnWe+O/ujR\n1P9SrVITmKlpJU/TS540eWprrTQDDmIzVj2DWruy6Rw07GWVf+qFC+Hcc1OwOHq037marroAffrp\nVPBUW7YM9u6dvq1SYM/UtJKn6SVPmm4FlsrrnS0Frde9mh3cPGUvK3uTSW1z1MKFsGULnH32VKG9\nbFkqzA8dqt+8NlPTSp6mlzzHmOmLo/I2A+Z9Twah2cfDdWcH1zTsZWUdOVWpXTzwAOzcmWo/Epxy\nStr/3HNTd/QSXHBB/ztqZ1pcMG+Hcp73ZFBqI4M2yMLqc03DXlbGMfjVtYt589LP2BisWZMK2LPO\ngm3bjr2jX7my8/PW3rnDsTPl9+1rnqbZHX+eWk2e96TsNcSKQRpkYY05aNjLyvhPXV0gLl6cahlD\nQ2lo7Zo16Q79nHO6UzhWAsWuXalGs2oVLF+ersOmTSnN8uXpbn7PHnjkkRS0GqWpTPyrzKZvp+ko\nz3syKM0+nX7fu5WDg4a9rPJPvXXrVEG1enV/87RrVyoUDx1Kf09MpH6LiYmpDuZOglq9QHHwYApG\nzz+fgtWSJVMF8Mknp99796aaxv79sGJFSlebZtGidKyNG1NwaSeQ5Clo89RGytLn0c5w3bLk3RL3\nadgxjh5NBdO556bO5n61j09MpIL84EE47riUFwkOHEh57KS/YmIijcD69rdTkNy1aypQ7N6dCvhK\njQZSP0H1SK1Dh1KeqvsWatNACi5Hj04fMRSRAknePoiZ+kdmGq01KH0e9Qxy3mcr1zRsmjK1j4+P\np87uHTvSfJGFC1N+Jifh4ovbDxZbt8JTT6Xgs3JlChZbtqTawPz5KWhUznfgQHre/PnTj7NoUdpX\nnYfaNJCCxrJlx26rBJLKsSDla/Hi1u+oZ6qNlOk9bdUg5322ck3DpinTCKqDB+H44+G001IH+IED\nqUA88cT2A8Yzz6QCZ8UKOHIk1SSOHk13sc8/nwLFkiXpTn3//lRIHTqU9g8PT93NL1uWmpeGh6fu\n7mvTHDqU8r106fR81Askk5MpkLV7R92sNlKm97RVg5z32co1DZumTCOoqodoKltk/6WXUoHfjvHx\nFCCefTYV5hMT6XXu2pXu6r///RQojj8+FerPPZfyMH8+rF07dYz9+1PBfskl05eLr02zeHEa/rtj\nx/R5I/UCyfbt6bzVd9QHD8KGDSlIdtKW3+w9LXt/QZk+j5Y4aNg0ZRpBNTKSRiQ9/3wqpBcsSIXx\ngQOpsGu1cNu1C158MR2nUnCPjaUaxwknpE7/yclUIFWCQu05aptE6g3trU2zZMnMgWTvXnjlK6ee\nMzGRAsnkZDpeoyXY82j0np58cr5l3vupTJ9HSxw0bJra9nFIBWylvb2Xd6JLlqTO5iVLUsG+aFGa\n/T1vXntt2pWZ4hGpIBoeTsNj9+9P/RjnnAOnn97911dvxFBtIDnnnOl9Ijt3ptc5PDzVgd5uzaNR\nn8cg9Bd4mG75OGjYMSqFXPXEuspdXj/uRM86KxWcExOpD2JiIjUztRrA5s9PTU4LF6bC5wc/SEHk\nta+FN76xt6+pNpBUrjWka713b6oRVWoyndY86gWuQZrfUZYgZj3oCJd0paRNkjZLuqHO/kWSbsv2\nPyDprKLzZPmUYYG5Spv2xERqSpqcTIV+O0OBJydTM9SePanmcs45cP75qebRb7XLihx33PSgWKl5\nHH98ei+OHk3bHnig/ZVvmy3rMYgr6w5ingdRoUFD0nzgE8BVwPnAOySdX5PsPcCLEfFK4GPAR4vM\nk+VXhpErlTkIlRoCpILtlFNaD2DDw6kgGRlJzVzHHTfVNLVhQ/8LmeoRUBdemIJEpTlt7970e+XK\nlM9t26aWVWl37kKj+R1Llw7e3AjP5+idomsaFwGbI+LpiDgMfAW4pibNNcAXssd/BVwmVcbKWD+V\nYYG5yh34Sy+ln3nz0hBcSAHjscfy31WuWJEKxMqIqZ07U5/G6tWpc71MhUyzmsfOnVO1Pyldh7Gx\n1gNfo0UT9+3rfw2zVWWoFc8VRfdprAG2Vv29Dbi4UZqIOCJpN3Ai8Hx1IknrgHUAZ7iBsydmGrnS\nq+GaS5akpqTJyVQQVJqqIDU3Ve4qZ2rfr+Sv0uE8NJTusCuT+MbG0uiqCy8sR0drdVt+5U760KF0\nvRcuTIV7Jf/Ll6caU6t9ToPc11FtEPM8qAZmcl9E3BoRoxExOuJvse+JZst397o5oLopZefOtC0i\njSTKe1e5ZEka7jo5mdIODaWax44daV7G8uUpaNx/Pzz8cLnaxavfi6NH08/ixamPY2go9dFU5nl0\nenddr4a5Z0+67mXtLyhDrXiuKDpojAGnV/19WratbhpJC4DlwM6C82U5NZpp3OvmgOpC88UXU2Fw\n2mlT+RkaSk1OM3WErlyZ5l+sXZuee/hwGmZ7/PHpzn3fvnSOgwfL1y5eeS8uvjgFy8OH0wirPXvS\n6921a2rxxU7U9nXs3p0mPlZW7y3bdYF835Zo3VF00PgOcK6ksyUNAdcCd9SkuQO4Lnv8NuDvI8ow\nnsWa6UcneaXQfPWrj20K27073QnnqfksWZKaoNasSX0FS5emAnh8PE14qywFUtZ28UoAPe64VEuq\n5HvlynT9d+7srECvrWHu2pWGPa9YUd7+grxfamWdK7RPI+ujuB64G5gPfC4iHpX0YWB9RNwBfBb4\nC0mbgRdIgcVKrp/LO9Tra9m+fWpEFcy8CGClkHnxxRRwli1L/SPLlqUO98rzy9ouXgl8//iP6c76\nhRdS4T48nIJhpxP0qvs6nnhiMPoLPJ+jNwqf3BcRdwF31Wz7UNXjg8DPFZ0P665+Lu9Qb5bwiSem\nJqZqk5Pw9NOpKareMhmVgrcygRHS8SKmRmiVvV28shRJ5StwI7pf46vcIFTmhlQWYhykpp+yr7E1\nSAamI9zKJU9zQJGTrWr7WlasOLYjdPv2VHNo1u9S/ToWL06B5uST0+Oyt4tv3Zo6wCtNaSMjKXBu\n397dQDcyMtWvMTmZ+lEmJqbWACs7z+HoLi8jYm1r1hxQvQRJLxbDq1fz2bMnfZFUtUqzSr07z8rS\nKYOwztHERFpKfeXK1DR16BD88Idw0knpNXUz0NWuAbZ4cWdrgPXaIKyxNUgcNKwQvf5Hrddk9YpX\nHPvFSJXaSLOAVvaCZGIiTeTbvTsFi8os8X37UvPRj/5oMYGusgZYRUT5+jXq8RyO7nLQsEL04x91\npkUAK/0u8+Y1D2hla/+uzg+kZqEDB1K/y9hYao46/fTUkb97d3rcbYP8vRaDnPcyctCwQpThH7XR\nstpbt9YfLlxptmpWC+lFQKkXJCpzJLZsmfryqAULUoAYH09NU6tXp5nzRTX/bdqUrtHkZKrBDQ9P\nffFUmfk7ObrLQcMK0co/apEFcb3mpmYBrVmzWuU1tRpQWtkO089RCRLLlk2tbjs8nPoWDh9O13bN\nmlTDOPHEYmoZg87fydFdGsR5dKOjo7F+/fp+Z8NmkCcY1PvOjkpwKeqfutk5t26d+uKjikrbfWV0\nVXWwOXQo3XVXB5Tab8fbsSP/9nnz0lpSlXM89VSqUcyfP9UcdeRI+lm9OnWC79mTOqqLXDPrBz9o\n/NrL3gdkUyQ9GBGjnRzDNQ0rTJ5O5X6MbGl259msFtKsn6bR69i0qf6kw0bbn3lm+oivxYunhtVC\n6vT+/ven8nrSSakWUvSd82zqTC5bn9Wg8TwN66t+fWdHozW1mq1h1GxRvEavY8+e1rZXjlmxcmUq\nmOfNS/mZNy8FipGR3i2XMTGRRmU9/nj6Ho/K/IZB7Ez2nI3OuaZhfdVOh3nRfSCNaiHN+mnGx+u/\njuOPb2376tVTtYqhoakgUfnCqMWLpxZb7IVKIbt8ecrfwYOpCW/VqtSEN2idyZ6z0TkHDeurVke2\n9GLSYKNmtXYCytq1qe8i7/bK664+Ry+DRK3qQnbRoqk+lF27yvO9I62YTc1s/eKgYX3V6siWft8p\nthNQlixpbTv07663tha3a1calVV5jWvWpNrQ/v2DFzCgHEPBB52DhvVdK7Owy3yn2CygtLK9X+rV\n4nbunPpmwIpBLmQ9Z6Nz7gi3gdLON7QVuXDibFLvi7VWrYLnnps9X27k793onGsaNlDK2AcyaBoN\nJKhXi1u+PA35rRSys2FiXNlqeIPGQcMGSq/6QGbrWP5mQbRRe/+KFbOrkJ2t722vOGjYwCm6D2TQ\nayfNCsU8y6TA7G3vH/T3tgzcpzGLuS2/vT6Qem37eb4Tu+jrnef4M01eazaZci6097f73tqUQoKG\npN+X9LikhyXdLmlFg3RbJH1X0gZJXkyqizzzNWk2w7uRdmapt3u98waavMefqVCcKYg2mik/W/Rr\nBYLZpKiaxr3Aj0TEvwOeAD7YJO1PRsSFnS6iZdP5jipp5+65V7WTVgJN3uPPVCi2E0Rnk3beW5uu\nkKAREfdExJHsz/uB04o4jzXmO6oprd4996p20kqgyXv8PDWJ2d4E1cxcD5rd0Is+jXcD32iwL4B7\nJD0oaV2zg0haJ2m9pPXjc+12uQ2+o2pfr2onrQSavMfPUyjO9iaoZuZ60OyGtkdPSfoWcEqdXTdG\nxNezNDcCR4AvNjjMpRExJulk4F5Jj0fEffUSRsStwK2Qvk+j3XzPFXNhJEyRWh3L3871bmVJi7zH\n9xcOTddoJNlsGkLca20HjYi4vNl+Se8C3gJcFg2+6SkixrLfOyTdDlwE1A0a1hoXHr3VzvVuJdC0\ncnwXiomH1xajkHkakq4EfgN4Y0QcaJBmGJgXEXuzx1cAHy4iP3OVC4/eavV6txpo/H62pt+LW85W\nRU3uuxlYRGpyArg/It4raTXwmYi4GlgF3J7tXwB8KSK+WVB+rEs8m7a7HAiKU+bFLQdZIUEjIl7Z\nYPsPgauzx08DFxRxfiuGq/s2SLwMejE8I9xy89wPGyQeXlsMrz1lubm6b2XTrLnUg0GK4aBhubm6\nb2WSp7nUfUbd5+Ypy83VfSsTN5f2h4OG5ebZtFYmXiqnP9w8ZS0purrvIb2Wl5tL+8M1DSsNL+du\nrXBzaX84aFhpuI16bmv1S6zcXNofbp6y0vCQ3rmr3YmjHh3Vew4aVhq9aKN2n0k5eZ2oweHmKSuN\notuo3WdSXh4JNTgcNKw0im6jdp9J8Vrtl6jwl4YNDjdPWakU2Ubdiz6Tudz81cmClv7SsMHhmobN\nGUXfzQ5q81e7tYNandTkPBJqcLimYXNG0XezRXfmFlGL6eZy953W5DwSajC4pmFzRtF3s0V25hZV\ni+lmP4/7JeYG1zRsTinybrbIIcNF1WK62c/jfom5wTUNsy4pcshwUbWYbtYO3C8xNxQWNCTdJGlM\n0obs5+oG6a6UtEnSZkk3FJUfs6IVWWgW1fTT7UBXqcmdd1767YAx+xTdPPWxiPiDRjslzQc+Afw0\nsA34jqQ7IuJ7BefLrBBFNX8V1fTjb7ezVvW7T+MiYHNEPA0g6SvANYCDhlmVIgt3j1qyVhTdp3G9\npIclfU7SCXX2rwG2Vv29Ldt2DEnrJK2XtH7cU3htDnLTj5VBR0FD0rckPVLn5xrgU8ArgAuBZ4E/\n7ORcEXFrRIxGxOiIF8w3M+uLjpqnIuLyPOkkfRq4s86uMeD0qr9Py7aZmVkJFTl66tSqP98KPFIn\n2XeAcyWdLWkIuBa4o6g8mZlZZ4rsCP89SRcCAWwBfhlA0mrgMxFxdUQckXQ9cDcwH/hcRDxaYJ7M\nzKwDhQWNiHhng+0/BK6u+vsu4K6i8mFmZt3jGeFmZpabg4aZmeXmoGFmZrk5aJiZWW4OGmZmlpuD\nhpmZ5eagYWZmuTlomJlZbg4aZmaWm4OGmZnl5qBhZma5OWiYmVluDhpmZpabg4aZmeXmoGFmZrk5\naJiZWW4OGmZmlpuDhpmZ5VbI171Kug1Ym/25AtgVERfWSbcF2AtMAkciYrSI/JiZWXcUEjQi4ucr\njyX9IbC7SfKfjIjni8iHmZl1VyFBo0KSgLcDP1XkeczMrDeK7tP4cWB7RDzZYH8A90h6UNK6gvNi\nZmYdarumIelbwCl1dt0YEV/PHr8D+HKTw1waEWOSTgbulfR4RNzX4HzrgHUAZ5xxRrvZNjOzDigi\nijmwtAAYA14fEdtypL8J2BcRfzBT2tHR0Vi/fn3nmTQzm0MkPdjpgKMim6cuBx5vFDAkDUtaVnkM\nXAE8UmB+zMysQ0UGjWupaZqStFrSXdmfq4B/lrQR+DfgbyPimwXmx8zMOlTY6KmIeFedbT8Ers4e\nPw1cUNT5zcys+zwj3MzMcnPQMDOz3Bw0zMwsNwcNMzPLzUHDzMxyc9AwM7PcHDTMzCw3Bw0zM8vN\nQcPMzHJz0DAzs9wcNMzMLDcHDTMzy81Bw8zMcnPQMDOz3Bw0zMwsNwcNMzPLzUHDzMxyc9AwM7Pc\nHDTMzCy3joKGpJ+T9Kiko5JE3MtFAAAGYUlEQVRGa/Z9UNJmSZskvbnB88+W9ECW7jZJQ53kx8zM\nitVpTeMR4D8C91VvlHQ+cC3wGuBK4JOS5td5/keBj0XEK4EXgfd0mB8zMytQR0EjIh6LiE11dl0D\nfCUiDkXE94HNwEXVCSQJ+Cngr7JNXwB+tpP8mJlZsRYUdNw1wP1Vf2/LtlU7EdgVEUeapHmZpHXA\nuuzPQ5Ie6VJei3QS8Hy/MzGDQcgjOJ/d5nx216Dkc22nB5gxaEj6FnBKnV03RsTXO81AXhFxK3Br\nlqf1ETE6w1P6bhDyOQh5BOez25zP7hqkfHZ6jBmDRkRc3sZxx4DTq/4+LdtWbSewQtKCrLZRL42Z\nmZVIUUNu7wCulbRI0tnAucC/VSeIiAD+AXhbtuk6oGc1FzMza12nQ27fKmkb8AbgbyXdDRARjwJf\nBb4HfBN4f0RMZs+5S9Lq7BC/CfwPSZtJfRyfzXnqWzvJdw8NQj4HIY/gfHab89ldcyafSjf8ZmZm\nM/OMcDMzy81Bw8zMcitt0Bi0JUqyc2zIfrZI2tAg3RZJ383SdTz8rY183iRprCqvVzdId2V2fTdL\nuqEP+fx9SY9LeljS7ZJWNEjXl+s50/XJBoHclu1/QNJZvcpbVR5Ol/QPkr6X/S/9ap00b5K0u+rz\n8KFe5zPLR9P3UcnHs+v5sKTX9Th/a6uu0QZJeyR9oCZN366lpM9J2lE9f03SSkn3Snoy+31Cg+de\nl6V5UtJ1M54sIkr5A7yaNBHl28Bo1fbzgY3AIuBs4Clgfp3nfxW4Nnt8C/C+Hub9D4EPNdi3BTip\nj9f1JuDXZ0gzP7uu5wBD2fU+v8f5vAJYkD3+KPDRslzPPNcH+BXgluzxtcBtfXivTwVelz1eBjxR\nJ59vAu7sdd5afR+Bq4FvAAIuAR7oY17nA88BZ5blWgI/AbwOeKRq2+8BN2SPb6j3PwSsBJ7Ofp+Q\nPT6h2blKW9OIAV2iJDv324Ev9+J8BbkI2BwRT0fEYeArpOveMxFxT0ytFnA/aR5PWeS5PteQPneQ\nPoeXZZ+NnomIZyPioezxXuAxmqy6UHLXAH8eyf2kOV6n9ikvlwFPRcQzfTr/MSLiPuCFms3Vn8FG\nZeCbgXsj4oWIeBG4l7ReYEOlDRpNrAG2Vv3d8RIlXfbjwPaIeLLB/gDukfRgtjRKP1yfVfE/16DK\nmuca99K7SXeZ9fTjeua5Pi+nyT6Hu0mfy77ImsdeCzxQZ/cbJG2U9A1Jr+lpxqbM9D6W6TN5LY1v\nCstwLStWRcSz2ePngFV10rR8XYtaeyoXlWSJkrxy5vcdNK9lXBoRY5JOBu6V9Hh2l9CTfAKfAj5C\n+if9CKkp7d3dPH9eea6npBuBI8AXGxym8Os56CQtBb4GfCAi9tTsfojUzLIv69/6G9Jk3F4biPcx\n6xv9GeCDdXaX5VoeIyJCUlfmV/Q1aMSALVEyU34lLSAtFf/6JscYy37vkHQ7qamjq/8cea+rpE8D\nd9bZlecadyzH9XwX8BbgssgaYOsco/DrWUee61NJsy37XCwnfS57StJCUsD4YkT8de3+6iASEXdJ\n+qSkkyKip4vv5Xgfe/KZzOEq4KGI2F67oyzXssp2SadGxLNZU96OOmnGSH0xFaeR+pEbGsTmqTIv\nUXI58HhEbKu3U9KwpGWVx6TO3p6u1lvTDvzWBuf/DnCu0gi0IVJ1/I5e5K9C0pXAbwA/ExEHGqTp\n1/XMc33uIH3uIH0O/75R4CtK1ofyWeCxiPijBmlOqfS1SLqIVCb0NLjlfB/vAH4xG0V1CbC7quml\nlxq2JJThWtao/gw2KgPvBq6QdELWVH1Ftq2xfvT05xwN8FZS+9ohYDtwd9W+G0mjVzYBV1VtvwtY\nnT0+hxRMNgN/CSzqQZ4/D7y3Zttq4K6qPG3Mfh4lNcP0+rr+BfBd4OHsQ3VqbT6zv68mjbZ5qk/5\n3Exqa92Q/dxSm89+Xs961wf4MCnIASzOPnebs8/hOX24hpeSmiEfrrqOVwPvrXxOgeuza7eRNODg\n3/chn3Xfx5p8CvhEdr2/S9WIyh7mc5gUBJZXbSvFtSQFsmeBl7Jy8z2kPrS/A54EvgWszNKOAp+p\neu67s8/pZuCXZjqXlxExM7PcBrF5yszM+sRBw8zMcnPQMDOz3Bw0zMwsNwcNMzPLzUHDzMxyc9Aw\nM7Pc/j/CqPHy5nuFYAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "dim1 = 79\n", - "dim2 = 11\n", - "plt.scatter(x_train[:,dim1], x_train[:,dim2], color='blue', alpha=0.1)\n", - "plt.axis([-10, 10, -10, 10])\n", - "plt.title(\"Simulated data set\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## MXFusion Model Definition\n", - "Import MXFusion and MXNet modelling components" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "from mxfusion.models import Model\n", - "import mxnet.gluon.nn as nn\n", - "from mxfusion.components import Variable\n", - "from mxfusion.components.variables import PositiveTransformation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The primary data structure in MXFusion is the Model. Models hold ModelComponents, such as Variables, Distributions, and Functions which are the what define a probabilistic model. \n", - "\n", - "The model we'll be defining for PPCA is:\n", - "\n", - "$p(z)$ ~ $N(\\mathbf{\\mu}, \\mathbf{\\Sigma)}$\n", - "\n", - "$p(x | z,\\theta)$ ~ $N(\\mathbf{Wz} + \\mu, \\Psi)$\n", - "\n", - "where:\n", - "\n", - "$z \\in \\mathbb{R}^{N x K}, \\mathbf{\\mu} \\in \\mathbb{R}^K, \\mathbf{\\Sigma} \\in \\mathbb{R}^{NxKxK}, x \\in \\mathbb{R}^{NxD}$\n", - "\n", - "$\\Psi \\in \\mathbb{R}^{NxDxD}, \\Psi = [\\Psi_0, \\dots, \\Psi_N], \\Psi_i = \\sigma^2\\mathbf{I}$\n", - "\n", - "$z$ here is our latent variable of interest, $x$ is the observed data, and all other variables are parameters or constants of the model." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First we create an MXFusion Model object to build our PPCA model on. " - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "m = Model()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We attach ```Variable``` objects to our model to collect them in a centralized place. Internally, these are organized into a factor graph which is used during Inference. " - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "m.w = Variable(shape=(K,D), initial_value=mx.nd.array(np.random.randn(K,D)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Because the mean of $x$'s distribution is composed of the dot product of $z$ and $W$, we need to create a dot product function. First we create a dot product function in MXNet and then wrap the function into MXFusion using the MXFusionGluonFunction class. ```m.dot``` can then be called like a normal python function and will apply to the variables it is called on." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "dot = nn.HybridLambda(function='dot')\n", - "m.dot = mf.functions.MXFusionGluonFunction(dot, num_outputs=1, broadcastable=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we define ```m.z``` which has an identity matrix covariance, ```cov```, and zero mean.\n", - "\n", - "```m.z``` and ```sigma_2``` are then used to define ```m.x```.\n", - "\n", - "Note that both ```sigma_2``` and ```cov``` will be added implicitly into the ```Model``` because they are inputs to ```m.x```." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "cov = mx.nd.broadcast_to(mx.nd.expand_dims(mx.nd.array(np.eye(K,K)), 0),shape=(N,K,K))\n", - "m.z = mf.distributions.MultivariateNormal.define_variable(mean=mx.nd.zeros(shape=(N,K)), covariance=cov, shape=(N,K))\n", - "sigma_2 = Variable(shape=(1,), transformation=PositiveTransformation())\n", - "m.x = mf.distributions.Normal.define_variable(mean=m.dot(m.z, m.w), variance=sigma_2, shape=(N,D))" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "def make_model():\n", - " m = Model()\n", - " m.w = Variable(shape=(K,D), initial_value=mx.nd.array(np.random.randn(K,D)))\n", - " dot = nn.HybridLambda(function='dot')\n", - " m.dot = mf.functions.MXFusionGluonFunction(dot, num_outputs=1, broadcastable=False)\n", - " cov = mx.nd.broadcast_to(mx.nd.expand_dims(mx.nd.array(np.eye(K,K)), 0),shape=(N,K,K))\n", - " m.z = mf.distributions.MultivariateNormal.define_variable(mean=mx.nd.zeros(shape=(N,K)), covariance=cov, shape=(N,K))\n", - " sigma_2 = Variable(shape=(1,), transformation=PositiveTransformation())\n", - " m.x = mf.distributions.Normal.define_variable(mean=m.dot(m.z, m.w), variance=sigma_2, shape=(N,D))\n", - " return m\n", - "m = make_model()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Posterior Definition\n", - "\n", - "Now that we have our model, we need to define a posterior with parameters for the inference algorithm to optimize. When constructing a Posterior, we pass in the Model it is defined over and ModelComponent's from the original Model are accessible and visible in the Posterior." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The covariance matrix must continue to be positive definite throughout the optimization process in order to succeed in the Cholesky decomposition when drawing samples or computing the log pdf of ```q.z```. To satisfy this, we pass the covariance matrix parameters through a Gluon function that forces it into a Symmetric matrix which for suitable initialization values should maintain positive definite-ness throughout the optimization procedure. " - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "from mxfusion.inference.score_function import ScoreFunctionInference, ScoreFunctionRBInference" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "from mxfusion.inference import BatchInferenceLoop, GradBasedInference, StochasticVariationalInference\n", - "class SymmetricMatrix(mx.gluon.HybridBlock):\n", - " def hybrid_forward(self, F, x, *args, **kwargs):\n", - " return F.sum((F.expand_dims(x, 3)*F.expand_dims(x, 2)), axis=-3)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "While this model has an analytical solution, we will run Variational Inference to find the posterior to demonstrate inference in a setting where the answer is known. \n", - "\n", - "We place a multivariate normal prior over $z$ because that is $z$'s prior in the model and we don't need to approximate anything in this case. Because the form we're optimizing over is the true model, the optimization is convex and will always converge to the same answer given by classical PCA given enough iterations.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "def make_post(m):\n", - " q = mf.models.Posterior(m)\n", - " sym = mf.components.functions.MXFusionGluonFunction(SymmetricMatrix(), num_outputs=1, broadcastable=False)\n", - " cov = Variable(shape=(N,K,K), initial_value=mx.nd.broadcast_to(mx.nd.expand_dims(mx.nd.array(np.eye(K,K) * 1e-2), 0),shape=(N,K,K)))\n", - " q.post_cov = sym(cov)\n", - " q.post_mean = Variable(shape=(N,K), initial_value=mx.nd.array(np.random.randn(N,K)))\n", - " q.z.set_prior(mf.distributions.MultivariateNormal(mean=q.post_mean, covariance=q.post_cov))\n", - " return q\n", - "q = make_post(m)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We now take our posterior and model, along with an observation pattern (in our case only ```m.x``` is observed) and create an inference algorithm. This inference algorithm is combined with a gradient loop to create the Inference method ```infr```." - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "def get_grad(inf_type, num_samples=100):\n", - " import random\n", - " random.seed(0)\n", - " np.random.seed(0)\n", - " mx.random.seed(0)\n", - " m = make_model()\n", - " q = make_post(m)\n", - " observed = [m.x]\n", - " alg = inf_type(num_samples=num_samples, model=m, posterior=q, observed=observed)\n", - " infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop())\n", - " infr.initialize(x=mx.nd.array(x_train))\n", - " infr.run(max_iter=3, learning_rate=1e-2, x=mx.nd.array(x_train), verbose=False)\n", - " return infr, q.post_mean" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "num_samples = 10\n", - "sf_infr, sf_mean = get_grad(ScoreFunctionInference, num_samples)\n", - "rb_infr, rb_mean = get_grad(ScoreFunctionRBInference, num_samples)\n", - "svi_infr, svi_mean = get_grad(StochasticVariationalInference, num_samples)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [], - "source": [ - "sf_np = sf_infr.params[sf_mean].grad.asnumpy()\n", - "rb_np = rb_infr.params[rb_mean].grad.asnumpy()\n", - "normal_np = svi_infr.params[svi_mean].grad.asnumpy()" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([[ -40.875 , -39.375 ],\n", - " [ 174.25 , 111.15625 ],\n", - " [ 75.578125, -230.125 ],\n", - " [ -0.5625 , -111.265625],\n", - " [ 52.5 , -12.5 ]], dtype=float32),\n", - " array([[ -40.875 , -39.375 ],\n", - " [ 174.25 , 111.15625 ],\n", - " [ 75.578125, -230.125 ],\n", - " [ -0.5625 , -111.265625],\n", - " [ 52.5 , -12.5 ]], dtype=float32),\n", - " array([[-4.0752361e+01, -3.9187836e+01],\n", - " [ 1.7483197e+02, 1.1223953e+02],\n", - " [ 7.5619949e+01, -2.2986652e+02],\n", - " [ 4.3126345e-02, -1.1125554e+02],\n", - " [ 5.2483418e+01, -1.2516596e+01]], dtype=float32))" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sf_np[:5], rb_np[:5], normal_np[:5]" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.allclose(sf_np, rb_np, )" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "21632.492\n", - "21632.492\n", - "21644.488\n", - "69.323715\n", - "69.323715\n" - ] - } - ], - "source": [ - "print(np.sum(np.abs(sf_np)))\n", - "print(np.sum(np.abs(rb_np)))\n", - "print(np.sum(np.abs(normal_np)))\n", - "\n", - "print(np.sum(np.abs(sf_np - normal_np)))\n", - "print(np.sum(np.abs(rb_np - normal_np)))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# import random\n", - "# random.seed(0)\n", - "# np.random.seed(0)\n", - "# mx.random.seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# observed = [m.x]\n", - "# alg = StochasticVariationalInference(num_samples=10, model=m, posterior=q, observed=observed)\n", - "# infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# alg_sf = ScoreFunctionInference(num_samples=10, model=m, posterior=q, observed=observed)\n", - "# infr_sf = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The inference method is then initialized with our training data and we run optimiziation for a while until convergence." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# infr.initialize(x=mx.nd.array(x_train))\n", - "# infr_sf.initialize(x=mx.nd.array(x_train))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "# infr.run(max_iter=1, learning_rate=1e-2, x=mx.nd.array(x_train))\n", - "# infr_sf.run(max_iter=1, learning_rate=1e-2, x=mx.nd.array(x_train))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# b = infr.params[q.post_mean].grad\n", - "# infr_sf.params[q.post_mean].grad" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once training completes, we retrieve the posterior mean (our trained representation for $\\mathbf{Wz} + \\mu$) from the inference method and plot it. \n", - "As shown, the plot recovers (up to rotation) the original 2D data quite well." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# post_z_mean = infr.params[q.z.factor.mean].asnumpy()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/notebooks/ppca_tutorial.ipynb b/examples/notebooks/ppca_tutorial.ipynb index f1bc168..40aab3a 100644 --- a/examples/notebooks/ppca_tutorial.ipynb +++ b/examples/notebooks/ppca_tutorial.ipynb @@ -13,6 +13,27 @@ "Follow the instrallation instructions in the [README](../../README.md) file to get setup." ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\").\n", + "# You may not use this file except in compliance with the License.\n", + "# A copy of the License is located at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# or in the \"license\" file accompanying this file. This file is distributed\n", + "# on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n", + "# express or implied. See the License for the specific language governing\n", + "# permissions and limitations under the License.\n", + "# ==============================================================================\n", + "```" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -484,7 +505,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.6.0" } }, "nbformat": 4, diff --git a/examples/notebooks/variational_auto_encoder.ipynb b/examples/notebooks/variational_auto_encoder.ipynb index dd54f19..1736b67 100644 --- a/examples/notebooks/variational_auto_encoder.ipynb +++ b/examples/notebooks/variational_auto_encoder.ipynb @@ -9,6 +9,27 @@ "### Zhenwen Dai (2018-8-21)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\").\n", + "# You may not use this file except in compliance with the License.\n", + "# A copy of the License is located at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# or in the \"license\" file accompanying this file. This file is distributed\n", + "# on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n", + "# express or implied. See the License for the specific language governing\n", + "# permissions and limitations under the License.\n", + "# ==============================================================================\n", + "```" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -270,7 +291,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.0" } }, "nbformat": 4, diff --git a/examples/notebooks/writing_a_new_distribution.ipynb b/examples/notebooks/writing_a_new_distribution.ipynb index 006c4b6..1aded3f 100644 --- a/examples/notebooks/writing_a_new_distribution.ipynb +++ b/examples/notebooks/writing_a_new_distribution.ipynb @@ -4,7 +4,34 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Writing a new Distribution\n", + "# Writing a new Distribution" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\").\n", + "# You may not use this file except in compliance with the License.\n", + "# A copy of the License is located at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# or in the \"license\" file accompanying this file. This file is distributed\n", + "# on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n", + "# express or implied. See the License for the specific language governing\n", + "# permissions and limitations under the License.\n", + "# ==============================================================================\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "To write and and use a new Distribution class in MXFusion, fill out the Distribution interface and either the Univariate or Multivariate interface, depending on the type of distribution you are creating.\n", "\n", "There are 4 primary methods to fill out for a Distribution in MXFusion:\n", diff --git a/mxfusion/__init__.py b/mxfusion/__init__.py index e45f53a..3eb8d2b 100644 --- a/mxfusion/__init__.py +++ b/mxfusion/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """The main module for MXFusion. Submodules diff --git a/mxfusion/__version__.py b/mxfusion/__version__.py index fc79d63..3ceb85c 100644 --- a/mxfusion/__version__.py +++ b/mxfusion/__version__.py @@ -1 +1,16 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + __version__ = '0.2.1' diff --git a/mxfusion/common/__init__.py b/mxfusion/common/__init__.py index 26c1383..e1dc5a1 100644 --- a/mxfusion/common/__init__.py +++ b/mxfusion/common/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """ Submodules diff --git a/mxfusion/common/config.py b/mxfusion/common/config.py index a4f89c6..36bc812 100644 --- a/mxfusion/common/config.py +++ b/mxfusion/common/config.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import mxnet DEFAULT_DTYPE = 'float32' diff --git a/mxfusion/common/constants.py b/mxfusion/common/constants.py index 7196302..cec95ed 100644 --- a/mxfusion/common/constants.py +++ b/mxfusion/common/constants.py @@ -1 +1,16 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + SET_PARAMETER_PREFIX = 'SET_' diff --git a/mxfusion/common/exceptions.py b/mxfusion/common/exceptions.py index 7ea4ef6..2ea8690 100644 --- a/mxfusion/common/exceptions.py +++ b/mxfusion/common/exceptions.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + class ModelSpecificationError(Exception): pass diff --git a/mxfusion/components/__init__.py b/mxfusion/components/__init__.py index 15a7510..9b2f5cc 100644 --- a/mxfusion/components/__init__.py +++ b/mxfusion/components/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """The main module for MXFusion. Submodules diff --git a/mxfusion/components/distributions/__init__.py b/mxfusion/components/distributions/__init__.py index 16137f1..5bb9012 100644 --- a/mxfusion/components/distributions/__init__.py +++ b/mxfusion/components/distributions/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """This module contains Distributions for MXFusion. Submodules diff --git a/mxfusion/components/distributions/bernoulli.py b/mxfusion/components/distributions/bernoulli.py index df83505..2b2bdd0 100644 --- a/mxfusion/components/distributions/bernoulli.py +++ b/mxfusion/components/distributions/bernoulli.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..variables import Variable from .univariate import UnivariateDistribution from .distribution import LogPDFDecorator, DrawSamplesDecorator @@ -103,7 +118,7 @@ def __init__(self, prob_true, rand_gen=None, dtype=None, ctx=None): output_names=output_names, rand_gen=rand_gen, dtype=dtype, ctx=ctx) - + def replicate_self(self, attribute_map=None): """ This functions as a copy constructor for the object. diff --git a/mxfusion/components/distributions/beta.py b/mxfusion/components/distributions/beta.py index d021810..3e9b24d 100644 --- a/mxfusion/components/distributions/beta.py +++ b/mxfusion/components/distributions/beta.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ...common.config import get_default_MXNet_mode from ..variables import Variable from .univariate import UnivariateDistribution, UnivariateLogPDFDecorator, UnivariateDrawSamplesDecorator diff --git a/mxfusion/components/distributions/categorical.py b/mxfusion/components/distributions/categorical.py index e27c842..2618405 100644 --- a/mxfusion/components/distributions/categorical.py +++ b/mxfusion/components/distributions/categorical.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..variables import Variable from .univariate import UnivariateDistribution from .distribution import LogPDFDecorator, DrawSamplesDecorator diff --git a/mxfusion/components/distributions/dirichlet.py b/mxfusion/components/distributions/dirichlet.py index 3683526..98b0fe8 100644 --- a/mxfusion/components/distributions/dirichlet.py +++ b/mxfusion/components/distributions/dirichlet.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..variables import Variable from ...common.config import get_default_MXNet_mode from .distribution import Distribution, LogPDFDecorator, DrawSamplesDecorator diff --git a/mxfusion/components/distributions/distribution.py b/mxfusion/components/distributions/distribution.py index 85aea69..b30dd0f 100644 --- a/mxfusion/components/distributions/distribution.py +++ b/mxfusion/components/distributions/distribution.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..factor import Factor from .random_gen import MXNetRandomGenerator from ...util.inference import realize_shape diff --git a/mxfusion/components/distributions/gamma.py b/mxfusion/components/distributions/gamma.py index ffe792e..e112cbe 100644 --- a/mxfusion/components/distributions/gamma.py +++ b/mxfusion/components/distributions/gamma.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import numpy as np import mxnet as mx from ...common.config import get_default_MXNet_mode diff --git a/mxfusion/components/distributions/gp/__init__.py b/mxfusion/components/distributions/gp/__init__.py index 1880696..f5a87e5 100644 --- a/mxfusion/components/distributions/gp/__init__.py +++ b/mxfusion/components/distributions/gp/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """This module contains implementations of Gaussian Processes for MXFusion. Submodules diff --git a/mxfusion/components/distributions/gp/cond_gp.py b/mxfusion/components/distributions/gp/cond_gp.py index ded9fdd..ffdc764 100644 --- a/mxfusion/components/distributions/gp/cond_gp.py +++ b/mxfusion/components/distributions/gp/cond_gp.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import numpy as np from ....common.config import get_default_MXNet_mode from ....common.exceptions import InferenceError diff --git a/mxfusion/components/distributions/gp/gp.py b/mxfusion/components/distributions/gp/gp.py index 834a61d..2b0b58f 100644 --- a/mxfusion/components/distributions/gp/gp.py +++ b/mxfusion/components/distributions/gp/gp.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import numpy as np from ....common.config import get_default_MXNet_mode from ....common.exceptions import InferenceError diff --git a/mxfusion/components/distributions/gp/kernels/__init__.py b/mxfusion/components/distributions/gp/kernels/__init__.py index cb21c09..5017ece 100644 --- a/mxfusion/components/distributions/gp/kernels/__init__.py +++ b/mxfusion/components/distributions/gp/kernels/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """This module contains implementations of Gaussian Processes for MXFusion. Submodules diff --git a/mxfusion/components/distributions/gp/kernels/add_kernel.py b/mxfusion/components/distributions/gp/kernels/add_kernel.py index bcd8770..30b5114 100644 --- a/mxfusion/components/distributions/gp/kernels/add_kernel.py +++ b/mxfusion/components/distributions/gp/kernels/add_kernel.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .kernel import CombinationKernel diff --git a/mxfusion/components/distributions/gp/kernels/kernel.py b/mxfusion/components/distributions/gp/kernels/kernel.py index 6d89c15..25d837c 100644 --- a/mxfusion/components/distributions/gp/kernels/kernel.py +++ b/mxfusion/components/distributions/gp/kernels/kernel.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from copy import copy from .....common.exceptions import ModelSpecificationError from .....util.util import rename_duplicate_names, slice_axis diff --git a/mxfusion/components/distributions/gp/kernels/linear.py b/mxfusion/components/distributions/gp/kernels/linear.py index 449fd4d..70f74ab 100644 --- a/mxfusion/components/distributions/gp/kernels/linear.py +++ b/mxfusion/components/distributions/gp/kernels/linear.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .kernel import NativeKernel from ....variables import Variable from ....variables import PositiveTransformation diff --git a/mxfusion/components/distributions/gp/kernels/multiply_kernel.py b/mxfusion/components/distributions/gp/kernels/multiply_kernel.py index 188602d..30fcd15 100644 --- a/mxfusion/components/distributions/gp/kernels/multiply_kernel.py +++ b/mxfusion/components/distributions/gp/kernels/multiply_kernel.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .kernel import CombinationKernel diff --git a/mxfusion/components/distributions/gp/kernels/rbf.py b/mxfusion/components/distributions/gp/kernels/rbf.py index 9fd6b61..cfa4ab5 100644 --- a/mxfusion/components/distributions/gp/kernels/rbf.py +++ b/mxfusion/components/distributions/gp/kernels/rbf.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .stationary import StationaryKernel from .....util.customop import broadcast_to_w_samples diff --git a/mxfusion/components/distributions/gp/kernels/static.py b/mxfusion/components/distributions/gp/kernels/static.py index dc1dbac..3782b39 100644 --- a/mxfusion/components/distributions/gp/kernels/static.py +++ b/mxfusion/components/distributions/gp/kernels/static.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .kernel import NativeKernel from ....variables import Variable from ....variables import PositiveTransformation diff --git a/mxfusion/components/distributions/gp/kernels/stationary.py b/mxfusion/components/distributions/gp/kernels/stationary.py index 70a41d8..8784054 100644 --- a/mxfusion/components/distributions/gp/kernels/stationary.py +++ b/mxfusion/components/distributions/gp/kernels/stationary.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .kernel import NativeKernel from ....variables import Variable from ....variables import PositiveTransformation diff --git a/mxfusion/components/distributions/normal.py b/mxfusion/components/distributions/normal.py index ee7d4a0..5c074c5 100644 --- a/mxfusion/components/distributions/normal.py +++ b/mxfusion/components/distributions/normal.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import numpy as np import mxnet as mx import itertools diff --git a/mxfusion/components/distributions/pointmass.py b/mxfusion/components/distributions/pointmass.py index 90359d0..b2bb30f 100644 --- a/mxfusion/components/distributions/pointmass.py +++ b/mxfusion/components/distributions/pointmass.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..variables import Variable from .univariate import UnivariateDistribution, UnivariateLogPDFDecorator, UnivariateDrawSamplesDecorator from ...util.customop import broadcast_to_w_samples diff --git a/mxfusion/components/distributions/random_gen.py b/mxfusion/components/distributions/random_gen.py index d7af03a..fdc4998 100644 --- a/mxfusion/components/distributions/random_gen.py +++ b/mxfusion/components/distributions/random_gen.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from abc import ABC import mxnet as mx from ...common.config import get_default_dtype, get_default_MXNet_mode diff --git a/mxfusion/components/distributions/univariate.py b/mxfusion/components/distributions/univariate.py index 683147c..bbc9146 100644 --- a/mxfusion/components/distributions/univariate.py +++ b/mxfusion/components/distributions/univariate.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ...common.exceptions import InferenceError from ..variables import Variable from .distribution import Distribution, LogPDFDecorator, DrawSamplesDecorator diff --git a/mxfusion/components/distributions/wishart.py b/mxfusion/components/distributions/wishart.py index efa4db0..60dca5b 100644 --- a/mxfusion/components/distributions/wishart.py +++ b/mxfusion/components/distributions/wishart.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import numpy as np import mxnet as mx diff --git a/mxfusion/components/factor.py b/mxfusion/components/factor.py index 5de8ce3..01dbf81 100644 --- a/mxfusion/components/factor.py +++ b/mxfusion/components/factor.py @@ -1,3 +1,33 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """Factor module. .. autosummary:: diff --git a/mxfusion/components/functions/__init__.py b/mxfusion/components/functions/__init__.py index b4e4740..16d7842 100644 --- a/mxfusion/components/functions/__init__.py +++ b/mxfusion/components/functions/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """This module contains functionality for using functions in MXFusion models. Submodules diff --git a/mxfusion/components/functions/function_evaluation.py b/mxfusion/components/functions/function_evaluation.py index 3cc19b0..c1fa1f4 100644 --- a/mxfusion/components/functions/function_evaluation.py +++ b/mxfusion/components/functions/function_evaluation.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from abc import abstractmethod from ..factor import Factor from ..variables import array_has_samples, get_num_samples, as_samples diff --git a/mxfusion/components/functions/gluon_func_eval.py b/mxfusion/components/functions/gluon_func_eval.py index 115177c..9cf904c 100644 --- a/mxfusion/components/functions/gluon_func_eval.py +++ b/mxfusion/components/functions/gluon_func_eval.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..variables.variable import VariableType from .function_evaluation import FunctionEvaluationWithParameters, \ FunctionEvaluationDecorator diff --git a/mxfusion/components/functions/mxfusion_function.py b/mxfusion/components/functions/mxfusion_function.py index fa299c4..2a86fba 100644 --- a/mxfusion/components/functions/mxfusion_function.py +++ b/mxfusion/components/functions/mxfusion_function.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from abc import abstractmethod from ...common.config import get_default_dtype from ..variables import Variable diff --git a/mxfusion/components/functions/mxfusion_gluon_function.py b/mxfusion/components/functions/mxfusion_gluon_function.py index 5d41577..e9e4013 100644 --- a/mxfusion/components/functions/mxfusion_gluon_function.py +++ b/mxfusion/components/functions/mxfusion_gluon_function.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import mxnet as mx from copy import copy from .gluon_func_eval import GluonFunctionEvaluation diff --git a/mxfusion/components/functions/operators/__init__.py b/mxfusion/components/functions/operators/__init__.py index 60d1477..dac45f0 100644 --- a/mxfusion/components/functions/operators/__init__.py +++ b/mxfusion/components/functions/operators/__init__.py @@ -1,3 +1,17 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + """This module contains functionality for using MXNet native operators in MXFusion models. diff --git a/mxfusion/components/functions/operators/operator_impl.py b/mxfusion/components/functions/operators/operator_impl.py index 9665f71..aa2deb8 100644 --- a/mxfusion/components/functions/operators/operator_impl.py +++ b/mxfusion/components/functions/operators/operator_impl.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from . import MXNetOperatorDecorator """ Basic Arithmetic """ diff --git a/mxfusion/components/functions/operators/operators.py b/mxfusion/components/functions/operators/operators.py index ee65c41..08f748f 100644 --- a/mxfusion/components/functions/operators/operators.py +++ b/mxfusion/components/functions/operators/operators.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..function_evaluation import FunctionEvaluation, FunctionEvaluationDecorator from ...variables import Variable diff --git a/mxfusion/components/model_component.py b/mxfusion/components/model_component.py index c992bbe..c629655 100644 --- a/mxfusion/components/model_component.py +++ b/mxfusion/components/model_component.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from uuid import uuid4 from ..common.exceptions import ModelSpecificationError diff --git a/mxfusion/components/variables/__init__.py b/mxfusion/components/variables/__init__.py index 9c5d9fb..4cb5743 100644 --- a/mxfusion/components/variables/__init__.py +++ b/mxfusion/components/variables/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """Contains the Variable class, Variable transformations, and runtime methods on variables. Submodules diff --git a/mxfusion/components/variables/runtime_variable.py b/mxfusion/components/variables/runtime_variable.py index f516676..3046dde 100644 --- a/mxfusion/components/variables/runtime_variable.py +++ b/mxfusion/components/variables/runtime_variable.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from mxnet.ndarray.ndarray import NDArray from mxnet.symbol.symbol import Symbol diff --git a/mxfusion/components/variables/var_trans.py b/mxfusion/components/variables/var_trans.py index 3fe3d08..cb9d99b 100644 --- a/mxfusion/components/variables/var_trans.py +++ b/mxfusion/components/variables/var_trans.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from abc import ABC, abstractmethod import numpy as np from ...common.config import get_default_MXNet_mode diff --git a/mxfusion/components/variables/variable.py b/mxfusion/components/variables/variable.py index eb3f77e..49e5cfe 100644 --- a/mxfusion/components/variables/variable.py +++ b/mxfusion/components/variables/variable.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from enum import Enum import mxnet as mx import numpy as np diff --git a/mxfusion/inference/__init__.py b/mxfusion/inference/__init__.py index 5a04002..9ca43c7 100644 --- a/mxfusion/inference/__init__.py +++ b/mxfusion/inference/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """This module contains inference related methods and classes. Submodules diff --git a/mxfusion/inference/batch_loop.py b/mxfusion/inference/batch_loop.py index 65b07da..1061365 100644 --- a/mxfusion/inference/batch_loop.py +++ b/mxfusion/inference/batch_loop.py @@ -1,3 +1,33 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import mxnet as mx from .grad_loop import GradLoop diff --git a/mxfusion/inference/expectation.py b/mxfusion/inference/expectation.py index c61f2f5..ae71a31 100644 --- a/mxfusion/inference/expectation.py +++ b/mxfusion/inference/expectation.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..common.exceptions import InferenceError from ..components.variables import Variable, VariableType from .variational import StochasticVariationalInference diff --git a/mxfusion/inference/forward_sampling.py b/mxfusion/inference/forward_sampling.py index 197bfb0..319fe5f 100644 --- a/mxfusion/inference/forward_sampling.py +++ b/mxfusion/inference/forward_sampling.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..common.exceptions import InferenceError from ..components.variables import Variable from .variational import StochasticVariationalInference diff --git a/mxfusion/inference/grad_based_inference.py b/mxfusion/inference/grad_based_inference.py index 5add859..f52495e 100644 --- a/mxfusion/inference/grad_based_inference.py +++ b/mxfusion/inference/grad_based_inference.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .inference import Inference from .batch_loop import BatchInferenceLoop diff --git a/mxfusion/inference/grad_loop.py b/mxfusion/inference/grad_loop.py index b6b83f3..81927b8 100644 --- a/mxfusion/inference/grad_loop.py +++ b/mxfusion/inference/grad_loop.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from abc import ABC, abstractmethod diff --git a/mxfusion/inference/inference.py b/mxfusion/inference/inference.py index 85f388d..404c9da 100644 --- a/mxfusion/inference/inference.py +++ b/mxfusion/inference/inference.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import warnings import numpy as np import mxnet as mx diff --git a/mxfusion/inference/inference_alg.py b/mxfusion/inference/inference_alg.py index 43b9d25..a025153 100644 --- a/mxfusion/inference/inference_alg.py +++ b/mxfusion/inference/inference_alg.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from abc import ABC, abstractmethod from mxnet.gluon import HybridBlock from mxnet import autograd diff --git a/mxfusion/inference/inference_parameters.py b/mxfusion/inference/inference_parameters.py index 8a61824..d19384a 100644 --- a/mxfusion/inference/inference_parameters.py +++ b/mxfusion/inference/inference_parameters.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import warnings import numpy as np import mxnet as mx diff --git a/mxfusion/inference/map.py b/mxfusion/inference/map.py index 58e36b2..0e87454 100644 --- a/mxfusion/inference/map.py +++ b/mxfusion/inference/map.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .inference_alg import InferenceAlgorithm from ..components.variables import Variable, VariableType from ..models.posterior import Posterior diff --git a/mxfusion/inference/meanfield.py b/mxfusion/inference/meanfield.py index 89bd87a..537bec1 100644 --- a/mxfusion/inference/meanfield.py +++ b/mxfusion/inference/meanfield.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..models.posterior import Posterior from ..components.variables import PositiveTransformation from ..components.variables import Variable, VariableType diff --git a/mxfusion/inference/minibatch_loop.py b/mxfusion/inference/minibatch_loop.py index 8daa72a..1ae4754 100644 --- a/mxfusion/inference/minibatch_loop.py +++ b/mxfusion/inference/minibatch_loop.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import mxnet as mx from .grad_loop import GradLoop from mxnet.gluon.data import ArrayDataset diff --git a/mxfusion/inference/prediction.py b/mxfusion/inference/prediction.py index f06f03b..40634a3 100644 --- a/mxfusion/inference/prediction.py +++ b/mxfusion/inference/prediction.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..components import FunctionEvaluation, Distribution from ..inference.inference_alg import SamplingAlgorithm from ..modules.module import Module diff --git a/mxfusion/inference/score_function.py b/mxfusion/inference/score_function.py index 532d3e9..b82a983 100644 --- a/mxfusion/inference/score_function.py +++ b/mxfusion/inference/score_function.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import mxnet as mx from mxfusion.common.exceptions import InferenceError diff --git a/mxfusion/inference/variational.py b/mxfusion/inference/variational.py index de3d96a..a438763 100644 --- a/mxfusion/inference/variational.py +++ b/mxfusion/inference/variational.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .inference_alg import InferenceAlgorithm, SamplingAlgorithm diff --git a/mxfusion/models/__init__.py b/mxfusion/models/__init__.py index a06bfbe..e5527f6 100644 --- a/mxfusion/models/__init__.py +++ b/mxfusion/models/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """The main module for MXFusion. Submodules diff --git a/mxfusion/models/factor_graph.py b/mxfusion/models/factor_graph.py index 4ecab0c..bf387f6 100644 --- a/mxfusion/models/factor_graph.py +++ b/mxfusion/models/factor_graph.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from uuid import uuid4 import warnings import networkx as nx diff --git a/mxfusion/models/model.py b/mxfusion/models/model.py index cd0f04a..b3a7b22 100644 --- a/mxfusion/models/model.py +++ b/mxfusion/models/model.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .factor_graph import FactorGraph from ..components import VariableType diff --git a/mxfusion/models/posterior.py b/mxfusion/models/posterior.py index a55a31b..be31284 100644 --- a/mxfusion/models/posterior.py +++ b/mxfusion/models/posterior.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from .factor_graph import FactorGraph diff --git a/mxfusion/modules/__init__.py b/mxfusion/modules/__init__.py index 4ac62ea..074660d 100644 --- a/mxfusion/modules/__init__.py +++ b/mxfusion/modules/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """This module contains modules. Submodules diff --git a/mxfusion/modules/gp_modules/__init__.py b/mxfusion/modules/gp_modules/__init__.py index 4ab5f1b..661241c 100644 --- a/mxfusion/modules/gp_modules/__init__.py +++ b/mxfusion/modules/gp_modules/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """This module contains Gaussian process modules. Submodules diff --git a/mxfusion/modules/gp_modules/gp_regression.py b/mxfusion/modules/gp_modules/gp_regression.py index c7fcbd2..20800ed 100644 --- a/mxfusion/modules/gp_modules/gp_regression.py +++ b/mxfusion/modules/gp_modules/gp_regression.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import numpy as np from mxnet import autograd from ..module import Module diff --git a/mxfusion/modules/gp_modules/sparsegp_regression.py b/mxfusion/modules/gp_modules/sparsegp_regression.py index 55cd48a..8dc6c69 100644 --- a/mxfusion/modules/gp_modules/sparsegp_regression.py +++ b/mxfusion/modules/gp_modules/sparsegp_regression.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import numpy as np from mxnet import autograd from ..module import Module diff --git a/mxfusion/modules/gp_modules/svgp_regression.py b/mxfusion/modules/gp_modules/svgp_regression.py index 58486a0..a1dedc0 100644 --- a/mxfusion/modules/gp_modules/svgp_regression.py +++ b/mxfusion/modules/gp_modules/svgp_regression.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import numpy as np from ..module import Module from ...models import Model, Posterior diff --git a/mxfusion/modules/module.py b/mxfusion/modules/module.py index 4e3f8c6..d1b7a5a 100644 --- a/mxfusion/modules/module.py +++ b/mxfusion/modules/module.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import warnings from mxnet.gluon import ParameterDict from mxnet import initializer diff --git a/mxfusion/util/__init__.py b/mxfusion/util/__init__.py index abf69ef..37eca96 100644 --- a/mxfusion/util/__init__.py +++ b/mxfusion/util/__init__.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + """This module contains utility functions used throughout MXFusion. Submodules diff --git a/mxfusion/util/customop.py b/mxfusion/util/customop.py index 4f8b7d8..b459b40 100644 --- a/mxfusion/util/customop.py +++ b/mxfusion/util/customop.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import mxnet as mx import numpy as np from mxnet.operator import CustomOp, CustomOpProp diff --git a/mxfusion/util/graph_serialization.py b/mxfusion/util/graph_serialization.py index 55854f9..ea1ae1f 100644 --- a/mxfusion/util/graph_serialization.py +++ b/mxfusion/util/graph_serialization.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import json from ..components import ModelComponent from ..common.exceptions import SerializationError diff --git a/mxfusion/util/inference.py b/mxfusion/util/inference.py index aa8a6b7..15e2362 100644 --- a/mxfusion/util/inference.py +++ b/mxfusion/util/inference.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + from ..common.exceptions import ModelSpecificationError from ..components.variables import Variable, VariableType diff --git a/mxfusion/util/special.py b/mxfusion/util/special.py index 9902377..71cc0b6 100644 --- a/mxfusion/util/special.py +++ b/mxfusion/util/special.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import numpy as np from mxfusion.common.config import get_default_MXNet_mode diff --git a/mxfusion/util/testutils.py b/mxfusion/util/testutils.py index 301c1f8..34c12f5 100644 --- a/mxfusion/util/testutils.py +++ b/mxfusion/util/testutils.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import mxnet as mx import numpy as np import mxfusion as mf diff --git a/mxfusion/util/util.py b/mxfusion/util/util.py index 0b3441c..453a710 100644 --- a/mxfusion/util/util.py +++ b/mxfusion/util/util.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import re import mxnet as mx import numpy as np diff --git a/testing/components/distributions/beta_test.py b/testing/components/distributions/beta_test.py index ea789f5..917c4b2 100644 --- a/testing/components/distributions/beta_test.py +++ b/testing/components/distributions/beta_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/components/distributions/categorical_test.py b/testing/components/distributions/categorical_test.py index 31233e5..9c99267 100644 --- a/testing/components/distributions/categorical_test.py +++ b/testing/components/distributions/categorical_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/components/distributions/dirichlet_test.py b/testing/components/distributions/dirichlet_test.py index b4618fd..e20b1d6 100644 --- a/testing/components/distributions/dirichlet_test.py +++ b/testing/components/distributions/dirichlet_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import numpy as np @@ -12,7 +27,7 @@ @pytest.mark.usefixtures("set_seed") class TestDirichletDistribution(object): - # scipy implementation of dirichlet throws an error if x_i does not sum to one and float32 is + # scipy implementation of dirichlet throws an error if x_i does not sum to one and float32 is # not precise enough to have sum close enough to 1 after normalisation so using float64 @pytest.mark.parametrize("dtype, a, a_is_samples, rv, rv_is_samples, num_samples", [ (np.float64, np.random.rand(2), False, np.random.rand(5, 3, 2), True, 5), diff --git a/testing/components/distributions/gamma_test.py b/testing/components/distributions/gamma_test.py index 4302277..f9407ad 100644 --- a/testing/components/distributions/gamma_test.py +++ b/testing/components/distributions/gamma_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/components/distributions/gp/cond_gp_test.py b/testing/components/distributions/gp/cond_gp_test.py index b2e62b8..84d5aff 100644 --- a/testing/components/distributions/gp/cond_gp_test.py +++ b/testing/components/distributions/gp/cond_gp_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/components/distributions/gp/gp_test.py b/testing/components/distributions/gp/gp_test.py index 87af0c4..a5283c6 100644 --- a/testing/components/distributions/gp/gp_test.py +++ b/testing/components/distributions/gp/gp_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/components/distributions/gp/kernel_test.py b/testing/components/distributions/gp/kernel_test.py index aa7e58f..e922b81 100644 --- a/testing/components/distributions/gp/kernel_test.py +++ b/testing/components/distributions/gp/kernel_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/components/distributions/normal_test.py b/testing/components/distributions/normal_test.py index 39d9585..befd362 100644 --- a/testing/components/distributions/normal_test.py +++ b/testing/components/distributions/normal_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/components/distributions/wishart_test.py b/testing/components/distributions/wishart_test.py index b1bd15b..86bba26 100644 --- a/testing/components/distributions/wishart_test.py +++ b/testing/components/distributions/wishart_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx from sklearn.datasets import make_spd_matrix diff --git a/testing/components/factor_test.py b/testing/components/factor_test.py index dc911f7..f93486c 100644 --- a/testing/components/factor_test.py +++ b/testing/components/factor_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import mxnet.gluon.nn as nn import mxnet as mx diff --git a/testing/components/functions/function_evaluation_test.py b/testing/components/functions/function_evaluation_test.py index c13001d..f5818d6 100644 --- a/testing/components/functions/function_evaluation_test.py +++ b/testing/components/functions/function_evaluation_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/components/functions/mxfusion_function_test.py b/testing/components/functions/mxfusion_function_test.py index 12429c1..424f08c 100644 --- a/testing/components/functions/mxfusion_function_test.py +++ b/testing/components/functions/mxfusion_function_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import numpy as np import mxnet.gluon.nn as nn diff --git a/testing/components/functions/mxfusion_gluon_function_test.py b/testing/components/functions/mxfusion_gluon_function_test.py index 81fe38e..576b579 100644 --- a/testing/components/functions/mxfusion_gluon_function_test.py +++ b/testing/components/functions/mxfusion_gluon_function_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import numpy as np import mxnet.gluon.nn as nn diff --git a/testing/components/functions/operators_test.py b/testing/components/functions/operators_test.py index a6e5dcf..996e0dd 100644 --- a/testing/components/functions/operators_test.py +++ b/testing/components/functions/operators_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/components/model_component_test.py b/testing/components/model_component_test.py index 8e8315a..da2d69b 100644 --- a/testing/components/model_component_test.py +++ b/testing/components/model_component_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import networkx as nx import mxfusion.components as mfc diff --git a/testing/components/variables/var_trans_test.py b/testing/components/variables/var_trans_test.py index e7f1946..b412d0d 100644 --- a/testing/components/variables/var_trans_test.py +++ b/testing/components/variables/var_trans_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np @@ -50,4 +65,4 @@ def test_logistic(self, x, upper, lower, rtol, atol): x_trans = transform.transform(x) x_inversed = transform.inverseTransform(x_trans) assert x_inversed.dtype == x.dtype - assert np.isclose(x.asnumpy(), x_inversed.asnumpy(), rtol=rtol, atol=atol) \ No newline at end of file + assert np.isclose(x.asnumpy(), x_inversed.asnumpy(), rtol=rtol, atol=atol) diff --git a/testing/components/variables/variable_test.py b/testing/components/variables/variable_test.py index 5b52a91..cb0c425 100644 --- a/testing/components/variables/variable_test.py +++ b/testing/components/variables/variable_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import mxnet as mx import numpy as np diff --git a/testing/distributions/bernoulli_test.py b/testing/distributions/bernoulli_test.py index bb76d2e..f7dad81 100644 --- a/testing/distributions/bernoulli_test.py +++ b/testing/distributions/bernoulli_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/distributions/normal_mean_precision_test.py b/testing/distributions/normal_mean_precision_test.py index 0276bdf..4930dba 100644 --- a/testing/distributions/normal_mean_precision_test.py +++ b/testing/distributions/normal_mean_precision_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np @@ -30,7 +45,7 @@ def test_log_pdf(self, dtype, mean, mean_is_samples, precision, precision_is_sam precision_np = numpy_array_reshape(precision, precision_is_samples, n_dim) rv_np = numpy_array_reshape(rv, rv_is_samples, n_dim) log_pdf_np = norm.logpdf(rv_np, mean_np, np.power(precision_np, -0.5)) - + var = NormalMeanPrecision.define_variable(shape=rv_shape, dtype=dtype).factor mean_mx = mx.nd.array(mean, dtype=dtype) if not mean_is_samples: diff --git a/testing/inference/expectation_test.py b/testing/inference/expectation_test.py index 0064083..22591dd 100644 --- a/testing/inference/expectation_test.py +++ b/testing/inference/expectation_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import mxnet as mx import numpy as np import pytest diff --git a/testing/inference/forward_sampling_test.py b/testing/inference/forward_sampling_test.py index 87ec64e..06517fc 100644 --- a/testing/inference/forward_sampling_test.py +++ b/testing/inference/forward_sampling_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import numpy as np import mxnet as mx diff --git a/testing/inference/inference_alg_test.py b/testing/inference/inference_alg_test.py index 00bce3a..be9f033 100644 --- a/testing/inference/inference_alg_test.py +++ b/testing/inference/inference_alg_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import mxnet as mx import numpy as np diff --git a/testing/inference/inference_parameters_test.py b/testing/inference/inference_parameters_test.py index e85015a..16141b8 100644 --- a/testing/inference/inference_parameters_test.py +++ b/testing/inference/inference_parameters_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import mxnet as mx from mxfusion.components import Variable diff --git a/testing/inference/inference_serialization_test.py b/testing/inference/inference_serialization_test.py index 2415fe3..438ee1a 100644 --- a/testing/inference/inference_serialization_test.py +++ b/testing/inference/inference_serialization_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import uuid import numpy as np diff --git a/testing/inference/map_test.py b/testing/inference/map_test.py index 53c9d36..5f87b1e 100644 --- a/testing/inference/map_test.py +++ b/testing/inference/map_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import mxnet as mx import numpy as np diff --git a/testing/inference/meanfield_test.py b/testing/inference/meanfield_test.py index 91e17fb..33a8998 100644 --- a/testing/inference/meanfield_test.py +++ b/testing/inference/meanfield_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import mxfusion as mf import mxnet as mx diff --git a/testing/inference/score_function_test.py b/testing/inference/score_function_test.py index b34db4e..d1e71aa 100644 --- a/testing/inference/score_function_test.py +++ b/testing/inference/score_function_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import mxnet as mx import mxnet.gluon.nn as nn import numpy as np diff --git a/testing/models/factor_graph_test.py b/testing/models/factor_graph_test.py index 7cb1cb7..66844d4 100644 --- a/testing/models/factor_graph_test.py +++ b/testing/models/factor_graph_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import uuid import numpy as np diff --git a/testing/modules/gpregression_test.py b/testing/modules/gpregression_test.py index ce2cdd4..5d89b84 100644 --- a/testing/modules/gpregression_test.py +++ b/testing/modules/gpregression_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/modules/sparsegpregression_test.py b/testing/modules/sparsegpregression_test.py index a15f6c0..a038d3c 100644 --- a/testing/modules/sparsegpregression_test.py +++ b/testing/modules/sparsegpregression_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/modules/svgpregression_test.py b/testing/modules/svgpregression_test.py index 3084776..27b14d2 100644 --- a/testing/modules/svgpregression_test.py +++ b/testing/modules/svgpregression_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/util/customop_test.py b/testing/util/customop_test.py index faa5309..fe5e10f 100644 --- a/testing/util/customop_test.py +++ b/testing/util/customop_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np diff --git a/testing/util/graph_serialization_test.py b/testing/util/graph_serialization_test.py index e19753c..6a3c0ec 100644 --- a/testing/util/graph_serialization_test.py +++ b/testing/util/graph_serialization_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import unittest import json from mxfusion.components import Variable diff --git a/testing/util/special_test.py b/testing/util/special_test.py index b807c45..7b4822c 100644 --- a/testing/util/special_test.py +++ b/testing/util/special_test.py @@ -1,3 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ============================================================================== + + import pytest import mxnet as mx import numpy as np From 09c4c52d6ff8b53876f29a8d323dd979c56a81a4 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Mon, 29 Oct 2018 11:25:03 +0000 Subject: [PATCH 59/70] Fix version loading setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c1f032a..3ba4e4d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('mxfusion/__version__.py', 'r') as rv: text = rv.read().split('=') - __version__ = re.search(r'\d+\.\d+\.\d+', text[1]).group() + __version__ = re.search(r'\d+\.\d+\.\d+', text[-1]).group() setup( name='MXFusion', # this is the name of the package as you will import it i.e import package-name From 8b30870244ae8b588577e8af81e6972ac281f356 Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Mon, 29 Oct 2018 12:05:15 +0000 Subject: [PATCH 60/70] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 30 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 17 +++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..6ae662f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: Bug report +about: Create a report to help us improve + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Build a model like '...' +2. Run inference '...' +3. See error + +Preferably this is in the form of a small snippet of runnable code that reproduces the behavior. + +**Expected behavior** +A clear and concise description of what you expected to happen. A + +**Desktop (please complete the following information):** + - OS: [e.g. iOS, Linux] + - Python version [e.g. 3.6] + - MXNet version [e.g. 1.3] + - MXFusion version [e.g. 0.2] + - MXNet context [e.g. CPU or GPU] + - MXNet dtype [e.g. float32] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..066b2d9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From 6d2a8a82679b57cfff3d659b98fa8f2041ceb785 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Mon, 29 Oct 2018 13:54:51 +0000 Subject: [PATCH 61/70] Minor doc updates --- docs/index.md | 2 +- mxfusion/__init__.py | 3 ++- mxfusion/components/__init__.py | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 29cac87..c5e70b4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,7 +6,7 @@ MXFusion helps you rapidly build and test new methods at scale, by focusing on the modularity of probabilistic models and their integration with modern deep learning techniques. * [Installation](installation.md) -* [Design Overview](design/overview.md) +* [Design Overview](design_documents/overview.md) * [API Reference](api.md) * [Tutorials](tutorials.md) diff --git a/mxfusion/__init__.py b/mxfusion/__init__.py index 3eb8d2b..eb65c14 100644 --- a/mxfusion/__init__.py +++ b/mxfusion/__init__.py @@ -22,8 +22,9 @@ :toctree: _autosummary components - models inference + models + modules util """ diff --git a/mxfusion/components/__init__.py b/mxfusion/components/__init__.py index 9b2f5cc..da4270a 100644 --- a/mxfusion/components/__init__.py +++ b/mxfusion/components/__init__.py @@ -23,7 +23,6 @@ distributions functions - modules variables factor model_component From cb223854c978f6fad67aad4fbfb8d53a8258066d Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Wed, 31 Oct 2018 16:01:17 +0000 Subject: [PATCH 62/70] Minor doc index update --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index c5e70b4..e39da2f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,9 +6,9 @@ MXFusion helps you rapidly build and test new methods at scale, by focusing on the modularity of probabilistic models and their integration with modern deep learning techniques. * [Installation](installation.md) -* [Design Overview](design_documents/overview.md) * [API Reference](api.md) * [Tutorials](tutorials.md) +* [Design Overview](design_documents/overview.md) ## Indices and tables From 0db23b2fa4405d615c08099606221e1fec5d1d4e Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Thu, 1 Nov 2018 18:23:27 +0000 Subject: [PATCH 63/70] Fix the bug: the inference methods of GP modules do not handle samples. (#108) * Remove the dependency on future and mxnet. Resolve the potential issue of self-dependency in setup.py * Bump the version number. * Re-implement to be 3.4 and 3.5 compatible. * Fix import error. * Add numpy denpendency. * Applied the suggestion. * Fix the bug of GP modules when the input variables are sampled from a prior distribution. * Update the sphinex docstring config for modules. * Update mxfusion/components/variables/runtime_variable.py * Avoid to automatically convert constance into mxnet array. --- .../components/distributions/bernoulli.py | 2 - mxfusion/components/distributions/beta.py | 5 - .../components/distributions/categorical.py | 2 - .../components/distributions/dirichlet.py | 3 - mxfusion/components/distributions/gamma.py | 10 - .../components/distributions/gp/cond_gp.py | 2 - mxfusion/components/distributions/gp/gp.py | 2 - mxfusion/components/distributions/normal.py | 28 +-- mxfusion/components/distributions/wishart.py | 9 +- mxfusion/components/factor.py | 44 +++-- .../components/variables/runtime_variable.py | 19 ++ mxfusion/components/variables/variable.py | 2 +- mxfusion/inference/__init__.py | 2 +- mxfusion/modules/__init__.py | 2 +- mxfusion/modules/gp_modules/__init__.py | 3 + mxfusion/modules/gp_modules/gp_regression.py | 35 +++- .../modules/gp_modules/sparsegp_regression.py | 48 +++-- .../modules/gp_modules/svgp_regression.py | 21 +- .../distributions/bernoulli_test.py | 0 .../distributions/dirichlet_test.py | 11 +- .../normal_mean_precision_test.py | 0 testing/modules/gpregression_test.py | 121 +++++++----- testing/modules/sparsegpregression_test.py | 108 ++++++---- testing/modules/svgpregression_test.py | 186 +++++++++++------- 24 files changed, 394 insertions(+), 271 deletions(-) rename testing/{ => components}/distributions/bernoulli_test.py (100%) rename testing/{ => components}/distributions/normal_mean_precision_test.py (100%) diff --git a/mxfusion/components/distributions/bernoulli.py b/mxfusion/components/distributions/bernoulli.py index 2b2bdd0..52903e0 100644 --- a/mxfusion/components/distributions/bernoulli.py +++ b/mxfusion/components/distributions/bernoulli.py @@ -107,8 +107,6 @@ class Bernoulli(UnivariateDistribution): :type ctx: None or mxnet.cpu or mxnet.gpu """ def __init__(self, prob_true, rand_gen=None, dtype=None, ctx=None): - if not isinstance(prob_true, Variable): - prob_true = Variable(value=prob_true) inputs = [('prob_true', prob_true)] input_names = ['prob_true'] output_names = ['random_variable'] diff --git a/mxfusion/components/distributions/beta.py b/mxfusion/components/distributions/beta.py index 3e9b24d..a819342 100644 --- a/mxfusion/components/distributions/beta.py +++ b/mxfusion/components/distributions/beta.py @@ -36,11 +36,6 @@ class Beta(UnivariateDistribution): :type ctx: None or mxnet.cpu or mxnet.gpu """ def __init__(self, a, b, rand_gen=None, dtype=None, ctx=None): - if not isinstance(a, Variable): - a = Variable(value=a) - if not isinstance(b, Variable): - b = Variable(value=b) - inputs = [('a', a), ('b', b)] input_names = [k for k, _ in inputs] output_names = ['random_variable'] diff --git a/mxfusion/components/distributions/categorical.py b/mxfusion/components/distributions/categorical.py index 2618405..a9bb467 100644 --- a/mxfusion/components/distributions/categorical.py +++ b/mxfusion/components/distributions/categorical.py @@ -122,8 +122,6 @@ class Categorical(UnivariateDistribution): def __init__(self, log_prob, num_classes, one_hot_encoding=False, normalization=True, axis=-1, rand_gen=None, dtype=None, ctx=None): - if not isinstance(log_prob, Variable): - log_prob = Variable(value=log_prob) inputs = [('log_prob', log_prob)] input_names = ['log_prob'] output_names = ['random_variable'] diff --git a/mxfusion/components/distributions/dirichlet.py b/mxfusion/components/distributions/dirichlet.py index 98b0fe8..2ca91fd 100644 --- a/mxfusion/components/distributions/dirichlet.py +++ b/mxfusion/components/distributions/dirichlet.py @@ -119,9 +119,6 @@ class Dirichlet(Distribution): """ def __init__(self, a, normalization=True, rand_gen=None, dtype=None, ctx=None): - if not isinstance(a, Variable): - a = Variable(value=a) - inputs = [('a', a)] input_names = ['a'] output_names = ['random_variable'] diff --git a/mxfusion/components/distributions/gamma.py b/mxfusion/components/distributions/gamma.py index e112cbe..66908ba 100644 --- a/mxfusion/components/distributions/gamma.py +++ b/mxfusion/components/distributions/gamma.py @@ -40,11 +40,6 @@ class Gamma(UnivariateDistribution): :type ctx: None or mxnet.cpu or mxnet.gpu """ def __init__(self, alpha, beta, rand_gen=None, dtype=None, ctx=None): - if not isinstance(alpha, Variable): - alpha = Variable(value=alpha) - if not isinstance(beta, Variable): - beta = Variable(value=beta) - inputs = [('alpha', alpha), ('beta', beta)] input_names = [k for k, _ in inputs] output_names = ['random_variable'] @@ -124,11 +119,6 @@ class GammaMeanVariance(UnivariateDistribution): :type ctx: None or mxnet.cpu or mxnet.gpu """ def __init__(self, mean, variance, rand_gen=None, dtype=None, ctx=None): - if not isinstance(mean, Variable): - mean = Variable(value=mean) - if not isinstance(variance, Variable): - variance = Variable(value=variance) - inputs = [('mean', mean), ('variance', variance)] input_names = [k for k, _ in inputs] output_names = ['random_variable'] diff --git a/mxfusion/components/distributions/gp/cond_gp.py b/mxfusion/components/distributions/gp/cond_gp.py index ffdc764..dd8497b 100644 --- a/mxfusion/components/distributions/gp/cond_gp.py +++ b/mxfusion/components/distributions/gp/cond_gp.py @@ -139,8 +139,6 @@ class ConditionalGaussianProcess(Distribution): """ def __init__(self, X, X_cond, Y_cond, kernel, mean_func=None, rand_gen=None, dtype=None, ctx=None): - if not isinstance(X, Variable): - X = Variable(value=X) inputs = [('X', X), ('X_cond', X_cond), ('Y_cond', Y_cond)] + \ [(k, v) for k, v in kernel.parameters.items()] input_names = [k for k, _ in inputs] diff --git a/mxfusion/components/distributions/gp/gp.py b/mxfusion/components/distributions/gp/gp.py index 2b0b58f..4675054 100644 --- a/mxfusion/components/distributions/gp/gp.py +++ b/mxfusion/components/distributions/gp/gp.py @@ -111,8 +111,6 @@ class GaussianProcess(Distribution): """ def __init__(self, X, kernel, mean_func=None, rand_gen=None, dtype=None, ctx=None): - if not isinstance(X, Variable): - X = Variable(value=X) inputs = [('X', X)] + [(k, v) for k, v in kernel.parameters.items()] input_names = [k for k, _ in inputs] output_names = ['random_variable'] diff --git a/mxfusion/components/distributions/normal.py b/mxfusion/components/distributions/normal.py index 5c074c5..1c7971a 100644 --- a/mxfusion/components/distributions/normal.py +++ b/mxfusion/components/distributions/normal.py @@ -44,11 +44,6 @@ class Normal(UnivariateDistribution): :type ctx: None or mxnet.cpu or mxnet.gpu """ def __init__(self, mean, variance, rand_gen=None, dtype=None, ctx=None): - if not isinstance(mean, Variable): - mean = Variable(value=mean) - if not isinstance(variance, Variable): - variance = Variable(value=variance) - inputs = [('mean', mean), ('variance', variance)] input_names = [k for k, _ in inputs] output_names = ['random_variable'] @@ -227,12 +222,6 @@ class MultivariateNormal(Distribution): """ def __init__(self, mean, covariance, rand_gen=None, minibatch_ratio=1., dtype=None, ctx=None): - self.minibatch_ratio = minibatch_ratio - if not isinstance(mean, Variable): - mean = Variable(value=mean) - if not isinstance(covariance, Variable): - covariance = Variable(value=covariance) - inputs = [('mean', mean), ('covariance', covariance)] input_names = ['mean', 'covariance'] output_names = ['random_variable'] @@ -252,7 +241,6 @@ def replicate_self(self, attribute_map=None): :type outputs: a dict of {'name' : Variable} or None """ replicant = super(MultivariateNormal, self).replicate_self(attribute_map) - replicant.minibatch_ratio = self.minibatch_ratio return replicant @MultivariateNormalLogPDFDecorator() @@ -277,7 +265,7 @@ def log_pdf(self, mean, covariance, random_variable, F=None): targets = random_variable - mean zvec = F.sum(F.linalg.trsm(lmat, F.expand_dims(targets, axis=-1)), axis=-1) sqnorm_z = - F.sum(F.square(zvec), axis=-1) - return 0.5 * (sqnorm_z - (N * np.log(2 * np.pi))) + logdetl + return (0.5 * (sqnorm_z - (N * np.log(2 * np.pi))) + logdetl)* self.log_pdf_scaling @MultivariateNormalDrawSamplesDecorator() def draw_samples(self, mean, covariance, rv_shape, num_samples=1, F=None): @@ -359,11 +347,6 @@ class NormalMeanPrecision(UnivariateDistribution): :type ctx: None or mxnet.cpu or mxnet.gpu """ def __init__(self, mean, precision, rand_gen=None, dtype=None, ctx=None): - if not isinstance(mean, Variable): - mean = Variable(value=mean) - if not isinstance(precision, Variable): - precision = Variable(value=precision) - inputs = [('mean', mean), ('precision', precision)] input_names = [k for k, _ in inputs] output_names = ['random_variable'] @@ -545,12 +528,6 @@ class MultivariateNormalMeanPrecision(Distribution): """ def __init__(self, mean, precision, rand_gen=None, minibatch_ratio=1., dtype=None, ctx=None): - self.minibatch_ratio = minibatch_ratio - if not isinstance(mean, Variable): - mean = Variable(value=mean) - if not isinstance(precision, Variable): - precision = Variable(value=precision) - inputs = [('mean', mean), ('precision', precision)] input_names = ['mean', 'precision'] output_names = ['random_variable'] @@ -570,7 +547,6 @@ def replicate_self(self, attribute_map=None): :type outputs: a dict of {'name' : Variable} or None """ replicant = super(MultivariateNormalMeanPrecision, self).replicate_self(attribute_map) - replicant.minibatch_ratio = self.minibatch_ratio return replicant @MultivariateNormalMeanPrecisionLogPDFDecorator() @@ -599,7 +575,7 @@ def log_pdf(self, mean, precision, random_variable, F=None): for ix in itertools.product(*map(range, random_variable.shape[:-1])): sqnorm_z[ix] = F.dot(F.dot(targets[ix], precision[ix], transpose_a=True), targets[ix]) - return -0.5 * (sqnorm_z + c + logdetl) + return -0.5 * (sqnorm_z + c + logdetl) * self.log_pdf_scaling @MultivariateNormalMeanPrecisionDrawSamplesDecorator() def draw_samples(self, mean, precision, rv_shape, num_samples=1, F=None): diff --git a/mxfusion/components/distributions/wishart.py b/mxfusion/components/distributions/wishart.py index 60dca5b..87159cf 100644 --- a/mxfusion/components/distributions/wishart.py +++ b/mxfusion/components/distributions/wishart.py @@ -135,12 +135,6 @@ class Wishart(Distribution): """ def __init__(self, degrees_of_freedom, scale, rand_gen=None, minibatch_ratio=1., dtype=None, ctx=None): - self.minibatch_ratio = minibatch_ratio - if not isinstance(degrees_of_freedom, Variable): - degrees_of_freedom = Variable(value=degrees_of_freedom) - if not isinstance(scale, Variable): - scale = Variable(value=scale) - inputs = [('degrees_of_freedom', degrees_of_freedom), ('scale', scale)] input_names = ['degrees_of_freedom', 'scale'] output_names = ['random_variable'] @@ -160,7 +154,6 @@ def replicate_self(self, attribute_map=None): :type outputs: a dict of {'name' : Variable} or None """ replicant = super(Wishart, self).replicate_self(attribute_map) - replicant.minibatch_ratio = self.minibatch_ratio return replicant @WishartLogPDFDecorator() @@ -198,7 +191,7 @@ def log_pdf(self, degrees_of_freedom, scale, random_variable, F=None): log_gamma_np = sp.log_multivariate_gamma(df / 2, dimension, F) tr_v_inv_x = sp.trace(sp.solve(scale, random_variable), F) - return 0.5 * ((a * log_det_X) - tr_v_inv_x - b - (df * log_det_V)) - log_gamma_np + return (0.5 * ((a * log_det_X) - tr_v_inv_x - b - (df * log_det_V)) - log_gamma_np) * self.log_pdf_scaling @WishartDrawSamplesDecorator() def draw_samples(self, degrees_of_freedom, scale, rv_shape, num_samples=1, F=None): diff --git a/mxfusion/components/factor.py b/mxfusion/components/factor.py index 01dbf81..02f087f 100644 --- a/mxfusion/components/factor.py +++ b/mxfusion/components/factor.py @@ -13,21 +13,6 @@ # ============================================================================== -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. -# ============================================================================== - - """Factor module. .. autosummary:: @@ -35,8 +20,32 @@ """ +import mxnet as mx +from mxnet.ndarray.ndarray import NDArray from copy import copy from .model_component import ModelComponent +from .variables import Variable +from ..common.config import get_default_dtype +from ..common.exceptions import ModelSpecificationError + + +def _define_variable_from_constant(v): + """ + If the input is an instance of Variable, it returns the input. If the input + is an integer, float or MXNet NDArray, it creates an Variable instance with + the value being the input. + + :param v: the input value + :type v: int, float, MXNet NDArray or Variable + """ + if isinstance(v, Variable): + return v + elif isinstance(v, (int, float)): + return Variable(value=mx.nd.array([v], dtype=get_default_dtype())) + elif isinstance(v, NDArray): + return Variable(value=v) + else: + raise ModelSpecificationError('The inputs/outputs of a factor can only be a int, float, MXNet NDArray or Variable, but get '+str(v)+'.') class Factor(ModelComponent): @@ -87,13 +96,16 @@ def __getattr__(self, value): def __init__(self, inputs, outputs, input_names, output_names): super(Factor, self).__init__() + inputs = [(k, _define_variable_from_constant(v)) for k, v + in inputs] if inputs is not None else inputs + outputs = [(k, _define_variable_from_constant(v)) for k, v + in outputs] if outputs is not None else outputs self._check_name_conflict(inputs, outputs) self._input_names = input_names if input_names is not None else [] self._output_names = output_names if output_names is not None else [] self.predecessors = inputs if inputs is not None else [] self.successors = outputs if outputs is not None else [] - def __repr__(self): out_str = str(self.__class__.__name__) if self.predecessors is not None: diff --git a/mxfusion/components/variables/runtime_variable.py b/mxfusion/components/variables/runtime_variable.py index 3046dde..a111b5c 100644 --- a/mxfusion/components/variables/runtime_variable.py +++ b/mxfusion/components/variables/runtime_variable.py @@ -97,3 +97,22 @@ def as_samples(F, array, num_samples): return array else: return F.broadcast_axis(array, axis=0, size=num_samples) + + +def arrays_as_samples(F, arrays): + """ + Broadcast the dimension of samples for a list of variables. If the number of samples of at least one of the variables is larger than one, all the variables in the list are broadcasted to have the same number of samples. + + :param F: the execution mode of MXNet. + :type F: mxnet.ndarray or mxnet.symbol + :param arrays: a list of arrays with samples to be broadcasted. + :type arrays: [MXNet NDArray or MXNet Symbol or {str: MXNet NDArray or MXNet Symbol}] + :returns: the list of variables after broadcasting + :rtypes: [MXNet NDArray or MXNet Symbol or {str: MXNet NDArray or MXNet Symbol}] + """ + num_samples = [max([get_num_samples(F, v) for v in a.values()]) if isinstance(a, dict) else get_num_samples(F, a) for a in arrays] + max_num_samples = max(num_samples) + if max_num_samples > 1: + return [{k: as_samples(F, v, max_num_samples) for k, v in a.items()} if isinstance(a, dict) else as_samples(F, a, max_num_samples) for a in arrays] + else: + return arrays diff --git a/mxfusion/components/variables/variable.py b/mxfusion/components/variables/variable.py index 49e5cfe..4a50ce7 100644 --- a/mxfusion/components/variables/variable.py +++ b/mxfusion/components/variables/variable.py @@ -156,7 +156,7 @@ def _initialize_as_param(self, value, shape, transformation): shape = value.shape if shape != value.shape: raise ModelSpecificationError("Shape mismatch in Variable creation. The numpy array shape " + str(value.shape) + " does not match with the shape argument " + str(shape) + ".") - value = mx.nd.array(value) + value = mx.nd.array(value, dtype=get_default_dtype()) elif isinstance(value, mx.nd.NDArray): if shape is None: shape = value.shape diff --git a/mxfusion/inference/__init__.py b/mxfusion/inference/__init__.py index 9ca43c7..e89b8f3 100644 --- a/mxfusion/inference/__init__.py +++ b/mxfusion/inference/__init__.py @@ -40,7 +40,7 @@ from .inference import Inference, TransferInference from .minibatch_loop import MinibatchInferenceLoop from .meanfield import create_Gaussian_meanfield -from .forward_sampling import ForwardSampling, VariationalPosteriorForwardSampling +from .forward_sampling import ForwardSampling, VariationalPosteriorForwardSampling, ForwardSamplingAlgorithm from .grad_based_inference import GradBasedInference from .variational import StochasticVariationalInference from .inference_parameters import InferenceParameters diff --git a/mxfusion/modules/__init__.py b/mxfusion/modules/__init__.py index 074660d..4fcbad0 100644 --- a/mxfusion/modules/__init__.py +++ b/mxfusion/modules/__init__.py @@ -21,8 +21,8 @@ .. autosummary:: :toctree: _autosummary - module gp_modules + module """ __all__ = ['module', 'gp_modules'] diff --git a/mxfusion/modules/gp_modules/__init__.py b/mxfusion/modules/gp_modules/__init__.py index 661241c..14ee835 100644 --- a/mxfusion/modules/gp_modules/__init__.py +++ b/mxfusion/modules/gp_modules/__init__.py @@ -21,6 +21,9 @@ .. autosummary:: :toctree: _autosummary + gp_regression + sparsegp_regression + svgp_regression """ __all__ = ['gp_regression', 'sparsegp_regression', 'svgp_regression'] diff --git a/mxfusion/modules/gp_modules/gp_regression.py b/mxfusion/modules/gp_modules/gp_regression.py index 20800ed..a30bf2c 100644 --- a/mxfusion/modules/gp_modules/gp_regression.py +++ b/mxfusion/modules/gp_modules/gp_regression.py @@ -19,12 +19,12 @@ from ...models import Model, Posterior from ...components.variables.variable import Variable from ...components.distributions import GaussianProcess, Normal -from ...inference.inference_alg import InferenceAlgorithm, \ - SamplingAlgorithm +from ...inference.inference_alg import SamplingAlgorithm from ...components.distributions.random_gen import MXNetRandomGenerator from ...util.inference import realize_shape from ...inference.variational import VariationalInference from ...util.customop import broadcast_to_w_samples +from ...components.variables.runtime_variable import arrays_as_samples class GPRegressionLogPdf(VariationalInference): @@ -41,7 +41,12 @@ def compute(self, F, variables): kern = self.model.kernel kern_params = kern.fetch_parameters(variables) - K = kern.K(F, X, **kern_params) + F.eye(N, dtype=X.dtype) * noise_var + X, Y, noise_var, kern_params = arrays_as_samples( + F, [X, Y, noise_var, kern_params]) + + K = kern.K(F, X, **kern_params) + \ + F.expand_dims(F.eye(N, dtype=X.dtype), axis=0) * \ + F.expand_dims(noise_var, axis=-2) L = F.linalg.potrf(K) if self.model.mean_func is not None: @@ -49,7 +54,9 @@ def compute(self, F, variables): Y = Y - mean LinvY = F.linalg.trsm(L, Y) logdet_l = F.linalg.sumlogdiag(F.abs(L)) - logL = - logdet_l * D - F.sum(F.square(LinvY) + np.log(2. * np.pi))/2 + tmp = F.sum(F.reshape(F.square(LinvY) + np.log(2. * np.pi), + shape=(Y.shape[0], -1)), axis=-1) + logL = - logdet_l * D - tmp/2 with autograd.pause(): self.set_parameter(variables, self.posterior.X, X[0]) @@ -78,7 +85,12 @@ def compute(self, F, variables): kern = self.model.kernel kern_params = kern.fetch_parameters(variables) - K = kern.K(F, X, **kern_params) + F.eye(N, dtype=X.dtype) * noise_var + X, noise_var, kern_params = arrays_as_samples( + F, [X, noise_var, kern_params]) + + K = kern.K(F, X, **kern_params) + \ + F.expand_dims(F.eye(N, dtype=X.dtype), axis=0) * \ + F.expand_dims(noise_var, axis=-2) L = F.linalg.potrf(K) Y_shape = realize_shape(self.model.Y.shape, variables) out_shape = (self.num_samples,)+Y_shape @@ -118,6 +130,9 @@ def compute(self, F, variables): kern = self.model.kernel kern_params = kern.fetch_parameters(variables) + X, noise_var, X_cond, L, LinvY, kern_params = arrays_as_samples( + F, [X, noise_var, X_cond, L, LinvY, kern_params]) + Kxt = kern.K(F, X_cond, X, **kern_params) LinvKxt = F.linalg.trsm(L, Kxt) mu = F.linalg.gemm2(LinvKxt, LinvY, True, False) @@ -135,7 +150,8 @@ def compute(self, F, variables): Ktt = kern.K(F, X, **kern_params) var = Ktt - F.linalg.syrk(LinvKxt, True) if not self.noise_free: - var += F.eye(N, dtype=X.dtype) * noise_var + var += F.expand_dims(F.eye(N, dtype=X.dtype), axis=0) * \ + F.expand_dims(noise_var, axis=-2) outcomes = {self.model.Y.uuid: (mu, var)} @@ -166,6 +182,9 @@ def compute(self, F, variables): kern = self.model.kernel kern_params = kern.fetch_parameters(variables) + X, noise_var, X_cond, L, LinvY, kern_params = arrays_as_samples( + F, [X, noise_var, X_cond, L, LinvY, kern_params]) + Kxt = kern.K(F, X_cond, X, **kern_params) LinvKxt = F.linalg.trsm(L, Kxt) mu = F.linalg.gemm2(LinvKxt, LinvY, True, False) @@ -179,7 +198,9 @@ def compute(self, F, variables): var = Ktt - F.sum(F.square(LinvKxt), axis=-2) if not self.noise_free: var += noise_var - die = self._rand_gen.sample_normal(shape=(self.num_samples,) + mu.shape[1:], dtype=self.model.F.factor.dtype) + die = self._rand_gen.sample_normal( + shape=(self.num_samples,) + mu.shape[1:], + dtype=self.model.F.factor.dtype) samples = mu + die * F.sqrt(F.expand_dims(var, axis=-1)) else: Ktt = kern.K(F, X, **kern_params) diff --git a/mxfusion/modules/gp_modules/sparsegp_regression.py b/mxfusion/modules/gp_modules/sparsegp_regression.py index 8dc6c69..382298a 100644 --- a/mxfusion/modules/gp_modules/sparsegp_regression.py +++ b/mxfusion/modules/gp_modules/sparsegp_regression.py @@ -18,12 +18,14 @@ from ..module import Module from ...models import Model, Posterior from ...components.variables.variable import Variable -from ...components.distributions import GaussianProcess, Normal, ConditionalGaussianProcess +from ...components.distributions import GaussianProcess, Normal, \ + ConditionalGaussianProcess from ...inference.forward_sampling import ForwardSamplingAlgorithm from ...inference.variational import VariationalInference from ...inference.inference_alg import SamplingAlgorithm from ...util.customop import broadcast_to_w_samples from ...components.distributions.random_gen import MXNetRandomGenerator +from ...components.variables.runtime_variable import arrays_as_samples class SparseGPRegressionLogPdf(VariationalInference): @@ -45,17 +47,24 @@ def compute(self, F, variables): kern = self.model.kernel kern_params = kern.fetch_parameters(variables) + X, Y, Z, noise_var, kern_params = arrays_as_samples( + F, [X, Y, Z, noise_var, kern_params]) + + noise_var_m = F.expand_dims(noise_var, axis=-2) + Kuu = kern.K(F, Z, **kern_params) if self.jitter > 0.: - Kuu = Kuu + F.eye(M, dtype=Z.dtype) * self.jitter + Kuu = Kuu + F.expand_dims(F.eye(M, dtype=Z.dtype), axis=0) * \ + self.jitter + Kuf = kern.K(F, Z, X, **kern_params) Kff_diag = kern.Kdiag(F, X, **kern_params) L = F.linalg.potrf(Kuu) LinvKuf = F.linalg.trsm(L, Kuf) - A = F.eye(M, dtype=Z.dtype) + \ - F.broadcast_div(F.linalg.syrk(LinvKuf), noise_var) + A = F.expand_dims(F.eye(M, dtype=Z.dtype), axis=0) + \ + F.broadcast_div(F.linalg.syrk(LinvKuf), noise_var_m) LA = F.linalg.potrf(A) if self.model.mean_func is not None: @@ -64,16 +73,20 @@ def compute(self, F, variables): LAInvLinvKufY = F.linalg.trsm(LA, F.linalg.gemm2(LinvKuf, Y)) logL = - D*F.linalg.sumlogdiag(LA) - logL = logL - F.sum(F.sum(F.square(Y)/noise_var + (np.log(2. * np.pi) + - F.log(noise_var)), axis=-1), axis=-1)/2 + logL = logL - F.sum(F.sum(F.square(Y)/noise_var_m + np.log(2. * np.pi) + + F.log(noise_var_m), axis=-1), axis=-1)/2 logL = logL + F.sum(F.sum( - F.square(LAInvLinvKufY)/(2*F.square(noise_var)), axis=-1), axis=-1) - logL = logL - D*F.sum(Kff_diag, axis=-1)/(2*noise_var) - logL = logL + D*F.sum(F.sum(F.square(LinvKuf)/(2.*noise_var), + F.square(LAInvLinvKufY)/(2*F.square(noise_var_m)), axis=-1), + axis=-1) + logL = logL - D*F.sum(Kff_diag/(2*noise_var), axis=-1) + logL = logL + D*F.sum(F.sum(F.square(LinvKuf)/(2.*noise_var_m), axis=-1), axis=-1) with autograd.pause(): - wv = F.broadcast_div(F.linalg.trsm(L, F.linalg.trsm(LA, LAInvLinvKufY, transpose=True), transpose=True), noise_var) + wv = F.broadcast_div( + F.linalg.trsm(L, F.linalg.trsm(LA, LAInvLinvKufY, + transpose=True), + transpose=True), noise_var_m) self.set_parameter(variables, self.graphs[1].wv, wv[0]) self.set_parameter(variables, self.graphs[1].L, L[0]) self.set_parameter(variables, self.graphs[1].LA, LA[0]) @@ -100,6 +113,9 @@ def compute(self, F, variables): kern = self.model.kernel kern_params = kern.fetch_parameters(variables) + X, Z, noise_var, L, LA, wv, kern_params = arrays_as_samples( + F, [X, Z, noise_var, L, LA, wv, kern_params]) + Kxt = kern.K(F, Z, X, **kern_params) mu = F.linalg.gemm2(Kxt, wv, True, False) @@ -121,7 +137,8 @@ def compute(self, F, variables): var = Ktt - F.linalg.syrk(LinvKxt, True) + \ F.linalg.syrk(LAinvLinvKxt, True) if not self.noise_free: - var += F.eye(N, dtype=X.dtype) * noise_var + var += F.expand_dims(F.eye(N, dtype=X.dtype), axis=0) * \ + F.expand_dims(noise_var, axis=-2) outcomes = {self.model.Y.uuid: (mu, var)} @@ -153,6 +170,9 @@ def compute(self, F, variables): kern = self.model.kernel kern_params = kern.fetch_parameters(variables) + X, Z, noise_var, L, LA, wv, kern_params = arrays_as_samples( + F, [X, Z, noise_var, L, LA, wv, kern_params]) + Kxt = kern.K(F, Z, X, **kern_params) mu = F.linalg.gemm2(Kxt, wv, True, False) @@ -169,7 +189,9 @@ def compute(self, F, variables): F.sum(F.square(LAinvLinvKxt), axis=-2) if not self.noise_free: var += noise_var - die = self._rand_gen.sample_normal(shape=(self.num_samples,) + mu.shape[1:], dtype=self.model.F.factor.dtype) + die = self._rand_gen.sample_normal( + shape=(self.num_samples,) + mu.shape[1:], + dtype=self.model.F.factor.dtype) samples = mu + die * F.sqrt(F.expand_dims(var, axis=-1)) else: Ktt = kern.K(F, X, **kern_params) @@ -274,7 +296,7 @@ def _build_module_graphs(self): rand_gen=self._rand_gen, dtype=self.dtype, ctx=self.ctx) graph.Y = Y.replicate_self() graph.Y.set_prior(Normal( - mean=0, variance=graph.noise_var, rand_gen=self._rand_gen, + mean=graph.F, variance=graph.noise_var, rand_gen=self._rand_gen, dtype=self.dtype, ctx=self.ctx)) graph.mean_func = self.mean_func graph.kernel = graph.U.factor.kernel diff --git a/mxfusion/modules/gp_modules/svgp_regression.py b/mxfusion/modules/gp_modules/svgp_regression.py index a1dedc0..f132cf3 100644 --- a/mxfusion/modules/gp_modules/svgp_regression.py +++ b/mxfusion/modules/gp_modules/svgp_regression.py @@ -25,6 +25,7 @@ from ...util.customop import make_diagonal from ...util.customop import broadcast_to_w_samples from ...components.distributions.random_gen import MXNetRandomGenerator +from ...components.variables.runtime_variable import arrays_as_samples class SVGPRegressionLogPdf(VariationalInference): @@ -51,9 +52,15 @@ def compute(self, F, variables): kern = self.model.kernel kern_params = kern.fetch_parameters(variables) + X, Y, Z, noise_var, mu, S_W, S_diag, kern_params = arrays_as_samples( + F, [X, Y, Z, noise_var, mu, S_W, S_diag, kern_params]) + + noise_var_m = F.expand_dims(noise_var, axis=-2) + Kuu = kern.K(F, Z, **kern_params) if self.jitter > 0.: - Kuu = Kuu + F.eye(M, dtype=Z.dtype) * self.jitter + Kuu = Kuu + F.expand_dims(F.eye(M, dtype=Z.dtype), axis=0) * \ + self.jitter Kuf = kern.K(F, Z, X, **kern_params) Kff_diag = kern.Kdiag(F, X, **kern_params) @@ -70,8 +77,8 @@ def compute(self, F, variables): Linvmu = F.linalg.trsm(L, mu) LinvKuf = F.linalg.trsm(L, Kuf) - LinvKufY = F.linalg.trsm(L, psi1Y)/noise_var - LmInvPsi2LmInvT = F.linalg.syrk(LinvKuf)/noise_var + LinvKufY = F.linalg.trsm(L, psi1Y)/noise_var_m + LmInvPsi2LmInvT = F.linalg.syrk(LinvKuf)/noise_var_m LinvSLinvT = F.linalg.syrk(LinvLs) LmInvSmuLmInvT = LinvSLinvT*D + F.linalg.syrk(Linvmu) @@ -80,11 +87,11 @@ def compute(self, F, variables): - F.sum(F.sum(F.square(Linvmu), axis=-1), axis=-1)/2. logL = -F.sum(F.sum(F.square(Y)/noise_var + np.log(2. * np.pi) + - F.log(noise_var), axis=-1), axis=-1)/2. - logL = logL - D/2.*F.sum(Kff_diag, axis=-1)/noise_var + F.log(noise_var_m), axis=-1), axis=-1)/2. + logL = logL - D/2.*F.sum(Kff_diag/noise_var, axis=-1) logL = logL - F.sum(F.sum(LmInvSmuLmInvT*LmInvPsi2LmInvT, axis=-1), axis=-1)/2. - logL = logL + F.sum(F.sum(F.square(LinvKuf)/noise_var, axis=-1), + logL = logL + F.sum(F.sum(F.square(LinvKuf)/noise_var_m, axis=-1), axis=-1)*D/2. logL = logL + F.sum(F.sum(Linvmu*LinvKufY, axis=-1), axis=-1) logL = logL + self.model.U.factor.log_pdf_scaling*KL_u @@ -309,7 +316,7 @@ def _build_module_graphs(self): rand_gen=self._rand_gen, dtype=self.dtype, ctx=self.ctx) graph.Y = Y.replicate_self() graph.Y.set_prior(Normal( - mean=0, variance=graph.noise_var, rand_gen=self._rand_gen, + mean=graph.F, variance=graph.noise_var, rand_gen=self._rand_gen, dtype=self.dtype, ctx=self.ctx)) graph.mean_func = self.mean_func graph.kernel = graph.U.factor.kernel diff --git a/testing/distributions/bernoulli_test.py b/testing/components/distributions/bernoulli_test.py similarity index 100% rename from testing/distributions/bernoulli_test.py rename to testing/components/distributions/bernoulli_test.py diff --git a/testing/components/distributions/dirichlet_test.py b/testing/components/distributions/dirichlet_test.py index e20b1d6..6c4750c 100644 --- a/testing/components/distributions/dirichlet_test.py +++ b/testing/components/distributions/dirichlet_test.py @@ -20,6 +20,7 @@ from scipy.stats import dirichlet as scipy_dirichlet from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples +from mxfusion.components.variables import Variable from mxfusion.components.distributions import Dirichlet from mxfusion.util.testutils import numpy_array_reshape from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -66,7 +67,7 @@ def test_log_pdf_with_broadcast(self, dtype, a, a_is_samples, rv, rv_is_samples, r.append(a) log_pdf_np = np.array(r) - dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + dirichlet = Dirichlet.define_variable(a=Variable(), shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor variables = {dirichlet.a.uuid: a_mx, dirichlet.random_variable.uuid: rv_mx} log_pdf_rt = dirichlet.log_pdf(F=mx.nd, variables=variables) @@ -111,7 +112,7 @@ def test_log_pdf_no_broadcast(self, dtype, a, a_is_samples, r.append(a) log_pdf_np = np.array(r) - dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + dirichlet = Dirichlet.define_variable(a=Variable(), shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor variables = {dirichlet.a.uuid: a_mx, dirichlet.random_variable.uuid: rv_mx} log_pdf_rt = dirichlet.log_pdf(F=mx.nd, variables=variables) @@ -133,7 +134,7 @@ def test_draw_samples_with_broadcast(self, dtype, a, a_is_samples, rv_shape, num draw_samples_np = rand / np.sum(rand) rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) - dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + dirichlet = Dirichlet.define_variable(a=Variable(), shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor variables = {dirichlet.a.uuid: a_mx} draw_samples_rt = dirichlet.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) @@ -149,7 +150,7 @@ def test_draw_samples_with_broadcast_no_numpy_verification(self, dtype, a, a_is_ if not a_is_samples: a_mx = add_sample_dimension(mx.nd, a_mx) - dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype).factor + dirichlet = Dirichlet.define_variable(a=Variable(), shape=rv_shape, dtype=dtype).factor variables = {dirichlet.a.uuid: a_mx} draw_samples_rt = dirichlet.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) @@ -168,7 +169,7 @@ def test_draw_samples_no_broadcast(self, dtype, a, a_is_samples, rv_shape, num_s draw_samples_np = rand / np.sum(rand) rand_gen = MockMXNetRandomGenerator(mx.nd.array(rand.flatten(), dtype=dtype)) - dirichlet = Dirichlet.define_variable(a=None, shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor + dirichlet = Dirichlet.define_variable(a=Variable(), shape=rv_shape, dtype=dtype, rand_gen=rand_gen).factor variables = {dirichlet.a.uuid: a_mx} draw_samples_rt = dirichlet.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples) diff --git a/testing/distributions/normal_mean_precision_test.py b/testing/components/distributions/normal_mean_precision_test.py similarity index 100% rename from testing/distributions/normal_mean_precision_test.py rename to testing/components/distributions/normal_mean_precision_test.py diff --git a/testing/modules/gpregression_test.py b/testing/modules/gpregression_test.py index 5d89b84..132eae9 100644 --- a/testing/modules/gpregression_test.py +++ b/testing/modules/gpregression_test.py @@ -14,14 +14,15 @@ import pytest +import warnings import mxnet as mx import numpy as np from mxfusion.models import Model from mxfusion.modules.gp_modules import GPRegression from mxfusion.components.distributions.gp.kernels import RBF, White -from mxfusion.components.distributions import GaussianProcess +from mxfusion.components.distributions import GaussianProcess, Normal from mxfusion.components import Variable -from mxfusion.inference import Inference, MAP, ModulePredictionAlgorithm, TransferInference +from mxfusion.inference import Inference, MAP, ModulePredictionAlgorithm, TransferInference, create_Gaussian_meanfield, StochasticVariationalInference, GradBasedInference, ForwardSamplingAlgorithm, ModulePredictionAlgorithm from mxfusion.components.variables.var_trans import PositiveTransformation from mxfusion.inference.forward_sampling import ForwardSamplingAlgorithm from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -31,10 +32,12 @@ matplotlib.use('Agg') import GPy +warnings.filterwarnings("ignore", category=DeprecationWarning) + class TestGPRegressionModule(object): - def test_log_pdf(self): + def gen_data(self): np.random.seed(0) D = 2 X = np.random.rand(10, 3) @@ -42,18 +45,28 @@ def test_log_pdf(self): noise_var = np.random.rand(1) lengthscale = np.random.rand(3) variance = np.random.rand(1) + return D, X, Y, noise_var, lengthscale, variance - m_gpy = GPy.models.GPRegression(X=X, Y=Y, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), noise_var=noise_var) - - l_gpy = m_gpy.log_likelihood() - - dtype = 'float64' + def gen_mxfusion_model(self, dtype, D, noise_var, lengthscale, variance, + rand_gen=None): m = Model() m.N = Variable() m.X = Variable(shape=(m.N, 3)) m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) - m.Y = GPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, shape=(m.N, D), dtype=dtype) + m.Y = GPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, shape=(m.N, D), dtype=dtype, rand_gen=rand_gen) + return m + + def test_log_pdf(self): + D, X, Y, noise_var, lengthscale, variance = self.gen_data() + + # GPy log-likelihood + m_gpy = GPy.models.GPRegression(X=X, Y=Y, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), noise_var=noise_var) + l_gpy = m_gpy.log_likelihood() + + # MXFusion log-likelihood + dtype = 'float64' + m = self.gen_mxfusion_model(dtype, D, noise_var, lengthscale, variance) observed = [m.X, m.Y] infr = Inference(MAP(model=m, observed=observed), dtype=dtype) @@ -64,22 +77,12 @@ def test_log_pdf(self): assert np.allclose(l_mf.asnumpy(), l_gpy) def test_draw_samples(self): - np.random.seed(0) - X = np.random.rand(10, 3) - Y = np.random.rand(10, 1) - noise_var = np.random.rand(1) - lengthscale = np.random.rand(3) - variance = np.random.rand(1) + D, X, Y, noise_var, lengthscale, variance = self.gen_data() dtype = 'float64' - rand_gen = MockMXNetRandomGenerator(mx.nd.array(np.random.rand(20), dtype=dtype)) + rand_gen = MockMXNetRandomGenerator(mx.nd.array(np.random.rand(20*D), dtype=dtype)) - m = Model() - m.N = Variable() - m.X = Variable(shape=(m.N, 3)) - m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) - kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) - m.Y = GPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, shape=(m.N, 1), dtype=dtype, rand_gen=rand_gen) + m = self.gen_mxfusion_model(dtype, D, noise_var, lengthscale, variance, rand_gen) observed = [m.X] infr = Inference(ForwardSamplingAlgorithm( @@ -89,7 +92,7 @@ def test_draw_samples(self): kern = RBF(3, True, name='rbf', dtype=dtype) + White(3, dtype=dtype) X_var = Variable(shape=(10, 3)) - gp = GaussianProcess.define_variable(X=X_var, kernel=kern, shape=(10, 1), dtype=dtype, rand_gen=rand_gen).factor + gp = GaussianProcess.define_variable(X=X_var, kernel=kern, shape=(10, D), dtype=dtype, rand_gen=rand_gen).factor variables = {gp.X.uuid: mx.nd.expand_dims(mx.nd.array(X, dtype=dtype), axis=0), gp.add_rbf_lengthscale.uuid: mx.nd.expand_dims(mx.nd.array(lengthscale, dtype=dtype), axis=0), gp.add_rbf_variance.uuid: mx.nd.expand_dims(mx.nd.array(variance, dtype=dtype), axis=0), gp.add_white_variance.uuid: mx.nd.expand_dims(mx.nd.array(noise_var, dtype=dtype), axis=0)} samples_2 = gp.draw_samples(F=mx.nd, variables=variables, num_samples=2).asnumpy() @@ -97,23 +100,13 @@ def test_draw_samples(self): assert np.allclose(samples, samples_2), (samples, samples_2) def test_prediction(self): - np.random.seed(0) - X = np.random.rand(10, 3) + D, X, Y, noise_var, lengthscale, variance = self.gen_data() Xt = np.random.rand(20, 3) - Y = np.random.rand(10, 1) - noise_var = np.random.rand(1) - lengthscale = np.random.rand(3) - variance = np.random.rand(1) m_gpy = GPy.models.GPRegression(X=X, Y=Y, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), noise_var=noise_var) dtype = 'float64' - m = Model() - m.N = Variable() - m.X = Variable(shape=(m.N, 3)) - m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) - kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) - m.Y = GPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, shape=(m.N, 1), dtype=dtype) + m = self.gen_mxfusion_model(dtype, D, noise_var, lengthscale, variance) observed = [m.X, m.Y] infr = Inference(MAP(model=m, observed=observed), dtype=dtype) @@ -167,23 +160,13 @@ def test_prediction(self): assert np.allclose(var_gpy, var_mf), (var_gpy, var_mf) def test_sampling_prediction(self): - np.random.seed(0) - X = np.random.rand(10, 3) + D, X, Y, noise_var, lengthscale, variance = self.gen_data() Xt = np.random.rand(20, 3) - Y = np.random.rand(10, 1) - noise_var = np.random.rand(1) - lengthscale = np.random.rand(3) - variance = np.random.rand(1) m_gpy = GPy.models.GPRegression(X=X, Y=Y, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), noise_var=noise_var) dtype = 'float64' - m = Model() - m.N = Variable() - m.X = Variable(shape=(m.N, 3)) - m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) - kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) - m.Y = GPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, shape=(m.N, 1), dtype=dtype) + m = self.gen_mxfusion_model(dtype, D, noise_var, lengthscale, variance) observed = [m.X, m.Y] infr = Inference(MAP(model=m, observed=observed), dtype=dtype) @@ -202,5 +185,47 @@ def test_sampling_prediction(self): gp.gp_predict.jitter = 1e-6 y_samples = infr_pred.run(X=mx.nd.array(Xt, dtype=dtype))[0].asnumpy() - # TODO: Check the correctness of the sampling + + def test_with_samples(self): + from mxfusion.common import config + config.DEFAULT_DTYPE = 'float64' + dtype = 'float64' + + D, X, Y, noise_var, lengthscale, variance = self.gen_data() + + m = Model() + m.N = Variable() + m.X = Normal.define_variable(mean=0, variance=1, shape=(m.N, 3)) + m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) + kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) + m.Y = GPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, shape=(m.N, D)) + + q = create_Gaussian_meanfield(model=m, observed=[m.Y]) + + infr = GradBasedInference( + inference_algorithm=StochasticVariationalInference( + model=m, posterior=q, num_samples=10, observed=[m.Y])) + infr.run(Y=mx.nd.array(Y, dtype='float64'), max_iter=2, + learning_rate=0.1, verbose=True) + + infr2 = Inference(ForwardSamplingAlgorithm( + model=m, observed=[m.X], num_samples=5)) + infr2.run(X=mx.nd.array(X, dtype='float64')) + + infr_pred = TransferInference(ModulePredictionAlgorithm(model=m, observed=[m.X], target_variables=[m.Y]), infr_params=infr.params) + xt = np.random.rand(13, 3) + res = infr_pred.run(X=mx.nd.array(xt, dtype=dtype))[0] + + gp = m.Y.factor + gp.attach_prediction_algorithms( + targets=gp.output_names, conditionals=gp.input_names, + algorithm=GPRegressionSamplingPrediction( + gp._module_graph, gp._extra_graphs[0], [gp._module_graph.X]), + alg_name='gp_predict') + gp.gp_predict.diagonal_variance = False + gp.gp_predict.jitter = 1e-6 + + infr_pred2 = TransferInference(ModulePredictionAlgorithm(model=m, observed=[m.X], target_variables=[m.Y]), infr_params=infr.params) + xt = np.random.rand(13, 3) + res = infr_pred2.run(X=mx.nd.array(xt, dtype=dtype))[0] diff --git a/testing/modules/sparsegpregression_test.py b/testing/modules/sparsegpregression_test.py index a038d3c..891ffbe 100644 --- a/testing/modules/sparsegpregression_test.py +++ b/testing/modules/sparsegpregression_test.py @@ -19,8 +19,9 @@ from mxfusion.models import Model from mxfusion.modules.gp_modules import SparseGPRegression from mxfusion.components.distributions.gp.kernels import RBF +from mxfusion.components.distributions import Normal from mxfusion.components import Variable -from mxfusion.inference import Inference, MAP, ModulePredictionAlgorithm, TransferInference +from mxfusion.inference import Inference, MAP, ModulePredictionAlgorithm, TransferInference, create_Gaussian_meanfield, StochasticVariationalInference, GradBasedInference, ForwardSamplingAlgorithm from mxfusion.components.variables.var_trans import PositiveTransformation from mxfusion.modules.gp_modules.sparsegp_regression import SparseGPRegressionSamplingPrediction @@ -32,7 +33,7 @@ class TestSparseGPRegressionModule(object): - def test_log_pdf(self): + def gen_data(self): np.random.seed(0) D = 2 X = np.random.rand(10, 3) @@ -41,13 +42,10 @@ def test_log_pdf(self): noise_var = np.random.rand(1) lengthscale = np.random.rand(3) variance = np.random.rand(1) + return D, X, Y, Z, noise_var, lengthscale, variance - m_gpy = GPy.models.SparseGPRegression(X=X, Y=Y, Z=Z, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), num_inducing=3) - m_gpy.likelihood.variance = noise_var - - l_gpy = m_gpy.log_likelihood() - - dtype = 'float64' + def gen_mxfusion_model(self, dtype, D, Z, noise_var, lengthscale, variance, + rand_gen=None): m = Model() m.N = Variable() m.X = Variable(shape=(m.N, 3)) @@ -56,6 +54,19 @@ def test_log_pdf(self): kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) m.Y = SparseGPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, inducing_inputs=m.Z, shape=(m.N, D), dtype=dtype) m.Y.factor.sgp_log_pdf.jitter = 1e-8 + return m + + def test_log_pdf(self): + D, X, Y, Z, noise_var, lengthscale, variance = self.gen_data() + + m_gpy = GPy.models.SparseGPRegression(X=X, Y=Y, Z=Z, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), num_inducing=3) + m_gpy.likelihood.variance = noise_var + + l_gpy = m_gpy.log_likelihood() + + dtype = 'float64' + m = self.gen_mxfusion_model(dtype, D, Z, noise_var, lengthscale, + variance) observed = [m.X, m.Y] infr = Inference(MAP(model=m, observed=observed), dtype=dtype) @@ -66,27 +77,15 @@ def test_log_pdf(self): assert np.allclose(l_mf.asnumpy(), l_gpy) def test_prediction(self): - np.random.seed(0) - X = np.random.rand(10, 3) - Y = np.random.rand(10, 1) - Z = np.random.rand(3, 3) - noise_var = np.random.rand(1) - lengthscale = np.random.rand(3)/10. - variance = np.random.rand(1) + D, X, Y, Z, noise_var, lengthscale, variance = self.gen_data() Xt = np.random.rand(20, 3) m_gpy = GPy.models.SparseGPRegression(X=X, Y=Y, Z=Z, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), num_inducing=3) m_gpy.likelihood.variance = noise_var dtype = 'float64' - m = Model() - m.N = Variable() - m.X = Variable(shape=(m.N, 3)) - m.Z = Variable(shape=(3, 3), initial_value=mx.nd.array(Z, dtype=dtype)) - m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) - kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) - m.Y = SparseGPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, inducing_inputs=m.Z, shape=(m.N, 1), dtype=dtype) - m.Y.factor.sgp_log_pdf.jitter = 1e-8 + m = self.gen_mxfusion_model(dtype, D, Z, noise_var, lengthscale, + variance) observed = [m.X, m.Y] infr = Inference(MAP(model=m, observed=observed), dtype=dtype) @@ -139,27 +138,15 @@ def test_prediction(self): assert np.allclose(var_gpy, var_mf), (var_gpy, var_mf) def test_sampling_prediction(self): - np.random.seed(0) - X = np.random.rand(10, 3) - Y = np.random.rand(10, 1) - Z = np.random.rand(3, 3) - noise_var = np.random.rand(1) - lengthscale = np.random.rand(3)/10. - variance = np.random.rand(1) + D, X, Y, Z, noise_var, lengthscale, variance = self.gen_data() Xt = np.random.rand(20, 3) m_gpy = GPy.models.SparseGPRegression(X=X, Y=Y, Z=Z, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), num_inducing=3) m_gpy.likelihood.variance = noise_var dtype = 'float64' - m = Model() - m.N = Variable() - m.X = Variable(shape=(m.N, 3)) - m.Z = Variable(shape=(3, 3), initial_value=mx.nd.array(Z, dtype=dtype)) - m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) - kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) - m.Y = SparseGPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, inducing_inputs=m.Z, shape=(m.N, 1), dtype=dtype) - m.Y.factor.sgp_log_pdf.jitter = 1e-8 + m = self.gen_mxfusion_model(dtype, D, Z, noise_var, lengthscale, + variance) observed = [m.X, m.Y] infr = Inference(MAP(model=m, observed=observed), dtype=dtype) @@ -182,3 +169,48 @@ def test_sampling_prediction(self): y_samples = infr_pred.run(X=mx.nd.array(Xt, dtype=dtype))[0].asnumpy() # TODO: Check the correctness of the sampling + + def test_with_samples(self): + from mxfusion.common import config + config.DEFAULT_DTYPE = 'float64' + dtype = 'float64' + + D, X, Y, Z, noise_var, lengthscale, variance = self.gen_data() + + m = Model() + m.N = Variable() + m.X = Normal.define_variable(mean=0, variance=1, shape=(m.N, 3)) + m.Z = Variable(shape=(3, 3), initial_value=mx.nd.array(Z, dtype=dtype)) + m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) + kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) + m.Y = SparseGPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, inducing_inputs=m.Z, shape=(m.N, D), dtype=dtype) + m.Y.factor.sgp_log_pdf.jitter = 1e-8 + + q = create_Gaussian_meanfield(model=m, observed=[m.Y]) + + infr = GradBasedInference( + inference_algorithm=StochasticVariationalInference( + model=m, posterior=q, num_samples=10, observed=[m.Y])) + infr.run(Y=mx.nd.array(Y, dtype='float64'), max_iter=2, + learning_rate=0.1, verbose=True) + + infr2 = Inference(ForwardSamplingAlgorithm( + model=m, observed=[m.X], num_samples=5)) + infr2.run(X=mx.nd.array(X, dtype='float64')) + + infr_pred = TransferInference(ModulePredictionAlgorithm(model=m, observed=[m.X], target_variables=[m.Y]), infr_params=infr.params) + xt = np.random.rand(13, 3) + res = infr_pred.run(X=mx.nd.array(xt, dtype=dtype))[0] + + gp = m.Y.factor + gp.attach_prediction_algorithms( + targets=gp.output_names, conditionals=gp.input_names, + algorithm=SparseGPRegressionSamplingPrediction( + gp._module_graph, gp._extra_graphs[0], [gp._module_graph.X]), + alg_name='sgp_predict') + gp.sgp_predict.diagonal_variance = False + gp.sgp_predict.jitter = 1e-6 + + infr_pred2 = TransferInference(ModulePredictionAlgorithm(model=m, observed=[m.X], target_variables=[m.Y]), infr_params=infr.params) + xt = np.random.rand(13, 3) + res = infr_pred2.run(X=mx.nd.array(xt, dtype=dtype))[0] diff --git a/testing/modules/svgpregression_test.py b/testing/modules/svgpregression_test.py index 27b14d2..416f799 100644 --- a/testing/modules/svgpregression_test.py +++ b/testing/modules/svgpregression_test.py @@ -14,13 +14,15 @@ import pytest +import warnings import mxnet as mx import numpy as np from mxfusion.models import Model from mxfusion.modules.gp_modules import SVGPRegression from mxfusion.components.distributions.gp.kernels import RBF +from mxfusion.components.distributions import Normal from mxfusion.components import Variable -from mxfusion.inference import Inference, MAP, ModulePredictionAlgorithm, TransferInference +from mxfusion.inference import Inference, MAP, ModulePredictionAlgorithm, TransferInference, create_Gaussian_meanfield, StochasticVariationalInference, GradBasedInference, ForwardSamplingAlgorithm from mxfusion.components.variables.var_trans import PositiveTransformation from mxfusion.modules.gp_modules.svgp_regression import SVGPRegressionSamplingPrediction @@ -29,12 +31,14 @@ matplotlib.use('Agg') import GPy +warnings.filterwarnings("ignore", category=DeprecationWarning) + class TestSVGPRegressionModule(object): - def test_log_pdf(self): + def gen_data(self): np.random.seed(0) - D = 2 + D = 1 X = np.random.rand(10, 3) Y = np.random.rand(10, D) Z = np.random.rand(3, 3) @@ -44,15 +48,13 @@ def test_log_pdf(self): noise_var = np.random.rand(1) lengthscale = np.random.rand(3) variance = np.random.rand(1) - qU_chol = np.linalg.cholesky(qU_cov_W.dot(qU_cov_W.T)+np.diag(qU_cov_diag))[None,:,:] - - m_gpy = GPy.core.SVGP(X=X, Y=Y, Z=Z, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), likelihood=GPy.likelihoods.Gaussian(variance=noise_var)) - m_gpy.q_u_mean = qU_mean - m_gpy.q_u_chol = GPy.util.choleskies.triang_to_flat(qU_chol) + qU_chol = np.linalg.cholesky( + qU_cov_W.dot(qU_cov_W.T)+np.diag(qU_cov_diag))[None, :, :] + return D, X, Y, Z, noise_var, lengthscale, variance, qU_mean, \ + qU_cov_W, qU_cov_diag, qU_chol - l_gpy = m_gpy.log_likelihood() - - dtype = 'float64' + def gen_mxfusion_model(self, dtype, D, Z, noise_var, lengthscale, variance, + rand_gen=None): m = Model() m.N = Variable() m.X = Variable(shape=(m.N, 3)) @@ -61,6 +63,21 @@ def test_log_pdf(self): kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) m.Y = SVGPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, inducing_inputs=m.Z, shape=(m.N, D), dtype=dtype) gp = m.Y.factor + return m, gp + + def test_log_pdf(self): + D, X, Y, Z, noise_var, lengthscale, variance, qU_mean, \ + qU_cov_W, qU_cov_diag, qU_chol = self.gen_data() + + m_gpy = GPy.core.SVGP(X=X, Y=Y, Z=Z, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), likelihood=GPy.likelihoods.Gaussian(variance=noise_var)) + m_gpy.q_u_mean = qU_mean + m_gpy.q_u_chol = GPy.util.choleskies.triang_to_flat(qU_chol) + + l_gpy = m_gpy.log_likelihood() + + dtype = 'float64' + m, gp = self.gen_mxfusion_model(dtype, D, Z, noise_var, lengthscale, + variance) observed = [m.X, m.Y] infr = Inference(MAP(model=m, observed=observed), dtype=dtype) @@ -75,18 +92,8 @@ def test_log_pdf(self): assert np.allclose(l_mf.asnumpy(), l_gpy) def test_prediction(self): - np.random.seed(0) - np.random.seed(0) - X = np.random.rand(10, 3) - Y = np.random.rand(10, 1) - Z = np.random.rand(3, 3) - qU_mean = np.random.rand(3, 1) - qU_cov_W = np.random.rand(3, 3) - qU_cov_diag = np.random.rand(3,) - noise_var = np.random.rand(1) - lengthscale = np.random.rand(3) - variance = np.random.rand(1) - qU_chol = np.linalg.cholesky(qU_cov_W.dot(qU_cov_W.T)+np.diag(qU_cov_diag))[None,:,:] + D, X, Y, Z, noise_var, lengthscale, variance, qU_mean, \ + qU_cov_W, qU_cov_diag, qU_chol = self.gen_data() Xt = np.random.rand(5, 3) m_gpy = GPy.core.SVGP(X=X, Y=Y, Z=Z, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), likelihood=GPy.likelihoods.Gaussian(variance=noise_var)) @@ -94,14 +101,8 @@ def test_prediction(self): m_gpy.q_u_chol = GPy.util.choleskies.triang_to_flat(qU_chol) dtype = 'float64' - m = Model() - m.N = Variable() - m.X = Variable(shape=(m.N, 3)) - m.Z = Variable(shape=(3, 3), initial_value=mx.nd.array(Z, dtype=dtype)) - m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) - kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) - m.Y = SVGPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, inducing_inputs=m.Z, shape=(m.N, 1), dtype=dtype) - gp = m.Y.factor + m, gp = self.gen_mxfusion_model(dtype, D, Z, noise_var, lengthscale, + variance) observed = [m.X, m.Y] infr = Inference(MAP(model=m, observed=observed), dtype=dtype) @@ -135,43 +136,35 @@ def test_prediction(self): # TODO: The full covariance matrix prediction with SVGP in GPy may not be correct. Need further investigation. - # # noise_free, full_cov - # mu_gpy, var_gpy = m_gpy.predict_noiseless(Xt, full_cov=True) - # - # infr2 = TransferInference(ModulePredictionAlgorithm(m, observed=[m.X], target_variables=[m.Y]), infr_params=infr.params, dtype=np.float64) - # infr2.inference_algorithm.model.Y.factor.svgp_predict.diagonal_variance = False - # infr2.inference_algorithm.model.Y.factor.svgp_predict.noise_free = True - # res = infr2.run(X=mx.nd.array(Xt, dtype=dtype))[0] - # mu_mf, var_mf = res[0].asnumpy()[0], res[1].asnumpy()[0] - # - # assert np.allclose(mu_gpy, mu_mf), (mu_gpy, mu_mf) - # assert np.allclose(var_gpy, var_mf), (var_gpy, var_mf) - # - # # noisy, full_cov - # mu_gpy, var_gpy = m_gpy.predict(Xt, full_cov=True) - # - # infr2 = TransferInference(ModulePredictionAlgorithm(m, observed=[m.X], target_variables=[m.Y]), infr_params=infr.params, dtype=np.float64) - # infr2.inference_algorithm.model.Y.factor.svgp_predict.diagonal_variance = False - # infr2.inference_algorithm.model.Y.factor.svgp_predict.noise_free = False - # res = infr2.run(X=mx.nd.array(Xt, dtype=dtype))[0] - # mu_mf, var_mf = res[0].asnumpy()[0], res[1].asnumpy()[0] - # - # assert np.allclose(mu_gpy, mu_mf), (mu_gpy, mu_mf) - # assert np.allclose(var_gpy, var_mf), (var_gpy, var_mf) + # noise_free, full_cov + mu_gpy, var_gpy = m_gpy.predict_noiseless(Xt, full_cov=True) + + infr2 = TransferInference(ModulePredictionAlgorithm(m, observed=[m.X], target_variables=[m.Y]), infr_params=infr.params, dtype=np.float64) + infr2.inference_algorithm.model.Y.factor.svgp_predict.diagonal_variance = False + infr2.inference_algorithm.model.Y.factor.svgp_predict.noise_free = True + res = infr2.run(X=mx.nd.array(Xt, dtype=dtype))[0] + mu_mf, var_mf = res[0].asnumpy()[0], res[1].asnumpy()[0] + + print(var_gpy.shape, var_mf.shape) + + assert np.allclose(mu_gpy, mu_mf), (mu_gpy, mu_mf) + assert np.allclose(var_gpy[:, :, 0], var_mf), (var_gpy[:, :, 0], var_mf) + + # noisy, full_cov + mu_gpy, var_gpy = m_gpy.predict(Xt, full_cov=True) + + infr2 = TransferInference(ModulePredictionAlgorithm(m, observed=[m.X], target_variables=[m.Y]), infr_params=infr.params, dtype=np.float64) + infr2.inference_algorithm.model.Y.factor.svgp_predict.diagonal_variance = False + infr2.inference_algorithm.model.Y.factor.svgp_predict.noise_free = False + res = infr2.run(X=mx.nd.array(Xt, dtype=dtype))[0] + mu_mf, var_mf = res[0].asnumpy()[0], res[1].asnumpy()[0] + + assert np.allclose(mu_gpy, mu_mf), (mu_gpy, mu_mf) + assert np.allclose(var_gpy[:, :, 0], var_mf), (var_gpy[:, :, 0], var_mf) def test_sampling_prediction(self): - np.random.seed(0) - np.random.seed(0) - X = np.random.rand(10, 3) - Y = np.random.rand(10, 1) - Z = np.random.rand(3, 3) - qU_mean = np.random.rand(3, 1) - qU_cov_W = np.random.rand(3, 3) - qU_cov_diag = np.random.rand(3,) - noise_var = np.random.rand(1) - lengthscale = np.random.rand(3) - variance = np.random.rand(1) - qU_chol = np.linalg.cholesky(qU_cov_W.dot(qU_cov_W.T)+np.diag(qU_cov_diag))[None,:,:] + D, X, Y, Z, noise_var, lengthscale, variance, qU_mean, \ + qU_cov_W, qU_cov_diag, qU_chol = self.gen_data() Xt = np.random.rand(5, 3) m_gpy = GPy.core.SVGP(X=X, Y=Y, Z=Z, kernel=GPy.kern.RBF(3, ARD=True, lengthscale=lengthscale, variance=variance), likelihood=GPy.likelihoods.Gaussian(variance=noise_var)) @@ -179,14 +172,8 @@ def test_sampling_prediction(self): m_gpy.q_u_chol = GPy.util.choleskies.triang_to_flat(qU_chol) dtype = 'float64' - m = Model() - m.N = Variable() - m.X = Variable(shape=(m.N, 3)) - m.Z = Variable(shape=(3, 3), initial_value=mx.nd.array(Z, dtype=dtype)) - m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) - kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) - m.Y = SVGPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, inducing_inputs=m.Z, shape=(m.N, 1), dtype=dtype) - gp = m.Y.factor + m, gp = self.gen_mxfusion_model(dtype, D, Z, noise_var, lengthscale, + variance) observed = [m.X, m.Y] infr = Inference(MAP(model=m, observed=observed), dtype=dtype) @@ -212,3 +199,54 @@ def test_sampling_prediction(self): y_samples = infr_pred.run(X=mx.nd.array(Xt, dtype=dtype))[0].asnumpy() # TODO: Check the correctness of the sampling + + def test_with_samples(self): + from mxfusion.common import config + config.DEFAULT_DTYPE = 'float64' + dtype = 'float64' + + D, X, Y, Z, noise_var, lengthscale, variance, qU_mean, \ + qU_cov_W, qU_cov_diag, qU_chol = self.gen_data() + + m = Model() + m.N = Variable() + m.X = Normal.define_variable(mean=0, variance=1, shape=(m.N, 3)) + m.Z = Variable(shape=(3, 3), initial_value=mx.nd.array(Z, dtype=dtype)) + m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) + kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) + m.Y = SVGPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, inducing_inputs=m.Z, shape=(m.N, D), dtype=dtype) + gp = m.Y.factor + gp.svgp_log_pdf.jitter = 1e-8 + + q = create_Gaussian_meanfield(model=m, observed=[m.Y]) + + infr = GradBasedInference( + inference_algorithm=StochasticVariationalInference( + model=m, posterior=q, num_samples=10, observed=[m.Y])) + infr.initialize(Y=Y.shape) + infr.params[gp._extra_graphs[0].qU_mean] = mx.nd.array(qU_mean, dtype=dtype) + infr.params[gp._extra_graphs[0].qU_cov_W] = mx.nd.array(qU_cov_W, dtype=dtype) + infr.params[gp._extra_graphs[0].qU_cov_diag] = mx.nd.array(qU_cov_diag, dtype=dtype) + infr.run(Y=mx.nd.array(Y, dtype='float64'), max_iter=2, + learning_rate=0.1, verbose=True) + + infr2 = Inference(ForwardSamplingAlgorithm( + model=m, observed=[m.X], num_samples=5)) + infr2.run(X=mx.nd.array(X, dtype='float64')) + + infr_pred = TransferInference(ModulePredictionAlgorithm(model=m, observed=[m.X], target_variables=[m.Y]), infr_params=infr.params) + xt = np.random.rand(13, 3) + res = infr_pred.run(X=mx.nd.array(xt, dtype=dtype))[0] + + gp = m.Y.factor + gp.attach_prediction_algorithms( + targets=gp.output_names, conditionals=gp.input_names, + algorithm=SVGPRegressionSamplingPrediction( + gp._module_graph, gp._extra_graphs[0], [gp._module_graph.X]), + alg_name='svgp_predict') + gp.svgp_predict.diagonal_variance = False + gp.svgp_predict.jitter = 1e-6 + + infr_pred2 = TransferInference(ModulePredictionAlgorithm(model=m, observed=[m.X], target_variables=[m.Y]), infr_params=infr.params) + xt = np.random.rand(13, 3) + res = infr_pred2.run(X=mx.nd.array(xt, dtype=dtype))[0] From 9b2c4fb066b0e824cdb8955219a959ca72f3e028 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 2 Nov 2018 07:44:43 +0000 Subject: [PATCH 64/70] fix the bug about document compilation --- docs/design_documents/inference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design_documents/inference.md b/docs/design_documents/inference.md index 505bfe0..ea2ecf8 100644 --- a/docs/design_documents/inference.md +++ b/docs/design_documents/inference.md @@ -16,7 +16,7 @@ A basic example to run variational inference with a meanfield posterior over som ### First Example -First we create the model. The model creation function is dummy here, but this applies to almost any model. See the [Model Definiton](model_definition.md) file for details on model creation. Then we define the observed variables in our model, and apply the convenience method for creating a factorized Gaussian posterior to that model, and get the posterior ```q```. +First we create the model. The model creation function is dummy here, but this applies to almost any model. See the [Model Definiton](../model_definition.md) file for details on model creation. Then we define the observed variables in our model, and apply the convenience method for creating a factorized Gaussian posterior to that model, and get the posterior ```q```. ```py m = make_model() From 0cc0f33d00f1ee9fd41bd20f9f613348a9f5adcb Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 2 Nov 2018 07:48:22 +0000 Subject: [PATCH 65/70] fix the link to readthedocs --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3e099e7..8a3dc18 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ [![Build Status](https://travis-ci.org/amzn/MXFusion.svg?branch=master)](https://travis-ci.org/amzn/MXFusion) | [![codecov](https://codecov.io/gh/amzn/MXFusion/branch/master/graph/badge.svg)](https://codecov.io/gh/amzn/MXFusion) | [![pypi](https://img.shields.io/pypi/v/mxfusion.svg?style=flat)](https://pypi.org/project/mxfusion/) | -[![Documentation Status](https://readthedocs.org/projects/mxfusion/badge/?version=latest)](https://mxfusion.readthedocs.io/en/latest/?badge=latest) | +[![Documentation Status](https://readthedocs.org/projects/mxfusion/badge/?version=master)](https://mxfusion.readthedocs.io/en/latest/?badge=master) | [![GitHub license](https://img.shields.io/github/license/amzn/mxfusion.svg)](https://github.com/amzn/mxfusion/blob/master/LICENSE) ![MXFusion](docs/images/logo/blender-small.png) -[Tutorials](https://mxfusion.readthedocs.io/en/latest/tutorials.html) | -[Documentation](https://mxfusion.readthedocs.io/en/latest/index.html) | +[Tutorials](https://mxfusion.readthedocs.io/en/master/tutorials.html) | +[Documentation](https://mxfusion.readthedocs.io/en/master/index.html) | [Contribution Guide](CONTRIBUTING.md) MXFusion is a modular deep probabilistic programming library. @@ -51,9 +51,9 @@ pip install . ## Where to go from here? -[Tutorials](https://mxfusion.readthedocs.io/en/latest/tutorials.html) +[Tutorials](https://mxfusion.readthedocs.io/en/master/tutorials.html) -[Documentation](https://mxfusion.readthedocs.io/en/latest/index.html) +[Documentation](https://mxfusion.readthedocs.io/en/master/index.html) [Contributions](CONTRIBUTING.md) From 5c0c4b31bc6d4ba8472eea82e1a3a9be036a802c Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Tue, 6 Nov 2018 10:48:22 +0000 Subject: [PATCH 66/70] Add module serialization (#107) * Refactor module algorithms * Add basic replication for module * Add reconcilation logic for module * Add loading for module and simplify to one graph file * Complete module serialization * Update mxfusion/inference/inference_parameters.py Co-Authored-By: meissnereric * Update mxfusion/util/graph_serialization.py Co-Authored-By: meissnereric * Minor refactors, address feedback --- mxfusion/components/model_component.py | 5 + mxfusion/components/variables/variable.py | 5 + mxfusion/inference/forward_sampling.py | 6 +- mxfusion/inference/inference.py | 22 +- mxfusion/inference/inference_alg.py | 12 + mxfusion/inference/inference_parameters.py | 6 +- mxfusion/models/factor_graph.py | 134 ++++++---- mxfusion/models/posterior.py | 8 +- mxfusion/modules/__init__.py | 2 + mxfusion/modules/gp_modules/gp_regression.py | 2 +- .../modules/gp_modules/sparsegp_regression.py | 2 +- .../modules/gp_modules/svgp_regression.py | 2 +- mxfusion/modules/module.py | 245 ++++++++++-------- mxfusion/util/graph_serialization.py | 27 +- .../distributions/gp/cond_gp_test.py | 2 +- .../components/distributions/gp/gp_test.py | 2 +- .../inference/inference_serialization_test.py | 71 ++++- testing/models/factor_graph_test.py | 122 ++++++--- 18 files changed, 439 insertions(+), 236 deletions(-) diff --git a/mxfusion/components/model_component.py b/mxfusion/components/model_component.py index c629655..a1f5326 100644 --- a/mxfusion/components/model_component.py +++ b/mxfusion/components/model_component.py @@ -58,6 +58,11 @@ def __eq__(self, other): def __repr__(self): return self.uuid + def as_json(self): + return {'uuid': self._uuid, + 'name': self.name, + 'attributes': [a.uuid for a in self.attributes]} + @property def graph(self): """ diff --git a/mxfusion/components/variables/variable.py b/mxfusion/components/variables/variable.py index 4a50ce7..1abef75 100644 --- a/mxfusion/components/variables/variable.py +++ b/mxfusion/components/variables/variable.py @@ -96,6 +96,11 @@ def type(self): elif isinstance(self.factor, FunctionEvaluation): return VariableType.FUNCVAR + def as_json(self): + object_dict = super(Variable, self).as_json() + object_dict['inherited_name'] = self.inherited_name if self.isInherited else None + return object_dict + def replicate_self(self, attribute_map=None): """ This functions as a copy constructor for the object. In order to do a copy constructor we first call ``__new__`` on the class which creates a blank object. diff --git a/mxfusion/inference/forward_sampling.py b/mxfusion/inference/forward_sampling.py index 319fe5f..9ddbb22 100644 --- a/mxfusion/inference/forward_sampling.py +++ b/mxfusion/inference/forward_sampling.py @@ -108,11 +108,11 @@ def merge_posterior_into_model(model, posterior, observed): :param observed: A list of observed variables :type observed: [Variable] """ - new_model, var_map = model.clone() + new_model = model.clone() for lv in model.get_latent_variables(observed): v = posterior.extract_distribution_of(posterior[lv]) new_model.replace_subgraph(new_model[v], v) - return new_model, var_map + return new_model class VariationalPosteriorForwardSampling(ForwardSampling): @@ -146,7 +146,7 @@ def __init__(self, num_samples, observed, m = inherited_inference.inference_algorithm.model q = inherited_inference.inference_algorithm.posterior - model_graph, var_map = merge_posterior_into_model( + model_graph = merge_posterior_into_model( m, q, observed=inherited_inference.observed_variables) super(VariationalPosteriorForwardSampling, self).__init__( diff --git a/mxfusion/inference/inference.py b/mxfusion/inference/inference.py index 404c9da..dd1db49 100644 --- a/mxfusion/inference/inference.py +++ b/mxfusion/inference/inference.py @@ -149,8 +149,7 @@ def set_initializer(self): pass def load(self, - primary_model_file=None, - secondary_graph_files=None, + graphs_file=None, inference_configuration_file=None, parameters_file=None, mxnet_constants_file=None, @@ -160,11 +159,8 @@ def load(self, The major pieces of this are the InferenceParameters, FactorGraphs, and InferenceConfiguration. - :param primary_model_file: The file containing the primary model to load back for this inference algorithm. - :type primary_model_file: str of filename - :param secondary_graph_files: The files containing any secondary graphs (e.g. a posterior) to load back - for this inference algorithm. - :type secondary_graph_files: [str of filename] + :param graphs_file: The file containing the graphs to load back for this inference algorithm. The first of these is the primary graph. + :type graphs_file: str of filename :param inference_configuration_file: The file containing any inference specific configuration needed to reload this inference algorithm. e.g. observation patterns used to train it. @@ -181,10 +177,9 @@ def load(self, inference algorithm. :type variable_constants_file: json dict of {uuid: constant_primitive} """ - primary_model = FactorGraph('graph_0').load_graph(primary_model_file) - secondary_graphs = [ - FactorGraph('graph_{}'.format(str(i + 1))).load_graph(file) - for i, file in enumerate(secondary_graph_files)] + graphs = FactorGraph.load_graphs(graphs_file) + primary_model = graphs[0] + secondary_graphs = graphs[1:] # { current_model_uuid : loaded_uuid} self._uuid_map = FactorGraph.reconcile_graphs( @@ -239,9 +234,8 @@ def save(self, prefix=None): prefix = prefix if prefix is not None else "inference" self.params.save(prefix=prefix) self.save_configuration(prefix + '_configuration.json') - for i, g in enumerate(self._graphs): - filename = prefix + "_graph_{}.json".format(i) - g.save(filename) + graphs = [g.as_json()for g in self._graphs] + FactorGraph.save(prefix + "_graphs.json", graphs) class TransferInference(Inference): diff --git a/mxfusion/inference/inference_alg.py b/mxfusion/inference/inference_alg.py index a025153..8ed624d 100644 --- a/mxfusion/inference/inference_alg.py +++ b/mxfusion/inference/inference_alg.py @@ -104,6 +104,18 @@ class InferenceAlgorithm(ABC): :type extra_graphs: [FactorGraph] """ + def replicate_self(self, model, extra_graphs=None): + + replicant = self.__class__.__new__(self.__class__) + replicant._model_graph = model + replicant._extra_graphs = extra_graphs if extra_graphs is not None else [] + observed = [replicant.model[o] for o in self._observed_uuid] + replicant._observed = set(observed) + replicant._observed_uuid = variables_to_UUID(observed) + replicant._observed_names = [v.name for v in observed] + return replicant + + def __init__(self, model, observed, extra_graphs=None): self._model_graph = model self._extra_graphs = extra_graphs if extra_graphs is not None else [] diff --git a/mxfusion/inference/inference_parameters.py b/mxfusion/inference/inference_parameters.py index d19384a..1ef8627 100644 --- a/mxfusion/inference/inference_parameters.py +++ b/mxfusion/inference/inference_parameters.py @@ -233,7 +233,11 @@ def with_uuid_map(item, uuid_map): old_constants = json.load(f) new_variable_constants = {with_uuid_map(k, uuid_map): v for k, v in old_constants.items()} if mxnet_constants_file is not None: - new_mxnet_constants = {with_uuid_map(k, uuid_map): v for k, v in ndarray.load(mxnet_constants_file).items()} + mxnet_constants = ndarray.load(mxnet_constants_file) + if isinstance(mxnet_constants, dict): + new_mxnet_constants = {with_uuid_map(k, uuid_map): v for k, v in mxnet_constants.items()} + else: + new_mxnet_constants = {} ip._constants = {} ip._constants.update(new_variable_constants) ip._constants.update(new_mxnet_constants) diff --git a/mxfusion/models/factor_graph.py b/mxfusion/models/factor_graph.py index bf387f6..6af3b7c 100644 --- a/mxfusion/models/factor_graph.py +++ b/mxfusion/models/factor_graph.py @@ -50,7 +50,7 @@ def __repr__(self): """ Return a string summary of this object """ - out_str = '' + out_str = '{} ({})\n'.format(self.__class__.__name__, self._uuid[:5]) for f in self.ordered_factors: if isinstance(f, FunctionEvaluation): out_str += ', '.join([str(v) for _, v in f.outputs])+' = '+str(f)+'\n' @@ -338,6 +338,7 @@ def get_descendants(self, node): """ return set(filter(lambda x: isinstance(x, Variable), networkx.algorithms.dag.descendants(self.components_graph, node).union({node}))) + def remove_subgraph(self, node): """ Removes a node and its parent graph recursively. @@ -394,17 +395,22 @@ def extract_distribution_function(component): return predecessor_direction, successor_direction return variable.replicate(replication_function=extract_distribution_function) + def clone(self, leaves=None): + new_model = self._replicate_class(name=self.name, verbose=self._verbose) + return self._clone(new_model, leaves) + + def _clone(self, new_model, leaves=None): """ - Clones a model, maintaining the same functionality and topology. Replicates all of its ModelComponents with new UUIDs. + Clones a model, maintaining the same functionality and topology. Replicates all of its ModelComponents, while maintaining the same UUIDs. + Starts upward from the leaves and copies everything in the graph recursively. :param leaves: If None, use the leaves in this model, otherwise use the provided leaves. - :returns: A tuple of (cloned_model, variable_map) + :returns: the cloned model """ - new_model = self._replicate_class(name=self.name, verbose=self._verbose) - var_map = {} # from old model to new model + var_map = {} # from old model to new model leaves = self.leaves if leaves is None else leaves for v in leaves: @@ -417,7 +423,7 @@ def clone(self, leaves=None): for v in self.variables.values(): if v.name is not None: setattr(new_model, v.name, new_model[v.uuid]) - return new_model, var_map + return new_model def get_parameters(self, excluded=None, include_inherited=False): """ @@ -444,6 +450,7 @@ def get_constants(self): """ return [v for v in self.variables.values() if v.type == VariableType.CONSTANT] + @staticmethod def reconcile_graphs(current_graphs, primary_previous_graph, secondary_previous_graphs=None, primary_current_graph=None): """ @@ -461,48 +468,53 @@ def reconcile_graphs(current_graphs, primary_previous_graph, secondary_previous_ :rtype: {previous ModelComponent : current ModelComponent} """ + + def update_with_named_components(previous_components, current_components, component_map, nodes_to_traverse_from): + name_pre = {c.name: c for c in previous_components if c.name} + name_cur = {c.name: c for c in current_components if c.name} + for name, previous_c in name_pre.items(): + current_c = name_cur[name] + component_map[previous_c.uuid] = current_c.uuid + nodes_to_traverse_from[previous_c.uuid] = current_c.uuid + + from .model import Model component_map = {} - current_level = {} - current_graph = primary_current_graph if primary_current_graph is not None else [graph for graph in current_graphs if isinstance(graph, Model)][0] - secondary_current_graphs = [graph for graph in current_graphs - if not isinstance(graph, Model)] - - # Map over the named components. - for c in primary_previous_graph.components.values(): - if c.name: - current_c = getattr(current_graph, c.name) - component_map[c.uuid] = current_c.uuid - current_level[c.uuid] = current_c.uuid + nodes_to_traverse_from = {} + current_graph = primary_current_graph if primary_current_graph is not None else current_graphs[0] + secondary_current_graphs = current_graphs[1:] + secondary_previous_graphs = secondary_previous_graphs if secondary_previous_graphs is not None else [] + if len(secondary_current_graphs) != len(secondary_previous_graphs): + raise ModelSpecificationError("Different number of secondary graphs passed in {} {}".format(secondary_current_graphs, secondary_previous_graphs)) + + update_with_named_components(primary_previous_graph.components.values(), current_graph.components.values(), component_map, nodes_to_traverse_from) # Reconcile the primary graph - FactorGraph._reconcile_graph(current_level, component_map, + FactorGraph._reconcile_graph(nodes_to_traverse_from, component_map, current_graph, primary_previous_graph) # Reconcile the other graphs - if not(secondary_current_graphs is None or - secondary_previous_graphs is None): + if len(secondary_current_graphs) > 0 and len(secondary_previous_graphs) > 0: for cg, pg in zip(secondary_current_graphs, secondary_previous_graphs): - current_level = {pc: cc for pc, cc in component_map.items() + nodes_to_traverse_from = {pc: cc for pc, cc in component_map.items() if pc in pg.components.keys()} + update_with_named_components(pg.components.values(), cg.components.values(), component_map, nodes_to_traverse_from) FactorGraph._reconcile_graph( - current_level, component_map, cg, pg) + nodes_to_traverse_from, component_map, cg, pg) # Resolve the remaining ambiguities here. - # if len(component_map) < set([graph.components for graph in previous_graphs])): # TODO the components of all the graphs not just the primary - # pass return component_map @staticmethod - def _reconcile_graph(current_level, component_map, current_graph, previous_graph): + def _reconcile_graph(nodes_to_traverse_from, component_map, current_graph, previous_graph): """ - Traverses the components in current_level of the current_graph/previous_graph, matching components where possible and generating + Traverses the components (breadth first) in nodes_to_traverse_from of the current_graph/previous_graph, matching components where possible and generating new calls to _reconcile_graph where the graph is still incompletely traversed. This method makes no attempt to resolve ambiguities in naming between the graphs and request the user to more completely specify names in their graph if such an ambiguity exists. Such - naming can be [more] completely specified by attaching names to each leaf node in the original graph. + naming can be more completely specified by attaching names to each leaf node in the original graph. - :param current_level: A list of items to traverse the graph upwards from. - :type current_level: [previous ModelComponents] + :param nodes_to_traverse_from: A list of items to traverse the graph upwards from. + :type nodes_to_traverse_from: [previous ModelComponents] :param component_map: The current mapping from the previous graph's MCs to the current_graph's MCs. This is used and modified during reconciliation. :type component_map: {previous_graph ModelComponent : current_graph ModelComponent} :param current_graph: The current graph to match components against. @@ -511,12 +523,12 @@ def _reconcile_graph(current_level, component_map, current_graph, previous_graph :type previous_graph: FactorGraph """ - def reconcile_direction(direction, c, current_c, new_level, component_map): + def reconcile_direction(direction, previous_c, current_c, new_level, component_map): if direction == 'predecessor': - previous_neighbors = c.predecessors + previous_neighbors = previous_c.predecessors current_neighbors = current_c.predecessors elif direction == 'successor': - previous_neighbors = c.successors + previous_neighbors = previous_c.successors current_neighbors = current_c.successors names = list(map(lambda x: x[0], previous_neighbors)) duplicate_names = set([x for x in names if names.count(x) > 1]) @@ -528,10 +540,12 @@ def reconcile_direction(direction, c, current_c, new_level, component_map): current_node = [item for name, item in current_neighbors if edge_name == name][0] component_map[node.uuid] = current_node.uuid new_level[node.uuid] = current_node.uuid - + if isinstance(node, Module): + module_component_map = current_node.reconcile_with_module(node) + component_map.update(module_component_map) new_level = {} - for c, current_c in current_level.items(): - reconcile_direction('predecessor', previous_graph[c], current_graph[current_c], new_level, component_map) + for previous_c, current_c in nodes_to_traverse_from.items(): + reconcile_direction('predecessor', previous_graph[previous_c], current_graph[current_c], new_level, component_map) """ TODO Reconciling in both directions currently breaks the reconciliation process and can cause multiple previous_uuid's to map to the same current_uuid. It's unclear why that happens. This shouldn't be necessary until we implement multi-output Factors though (and even then, only if not all the outputs are in a named chain). @@ -540,19 +554,7 @@ def reconcile_direction(direction, c, current_c, new_level, component_map): if len(new_level) > 0: return FactorGraph._reconcile_graph(new_level, component_map, current_graph, previous_graph) - def load_graph(self, graph_file): - """ - Method to load back in a graph. The graph file should be saved down using the save method, and is a JSON representation of the graph - generated by the [networkx](https://networkx.github.io) library. - - :param graph_file: The file containing the primary model to load back for this inference algorithm. - :type graph_file: str of filename - """ - import json - from ..util.graph_serialization import ModelComponentDecoder - with open(graph_file) as f: - json_graph = json.load(f, cls=ModelComponentDecoder) - + def load_from_json(self, json_graph): components_graph = nx.readwrite.json_graph.node_link_graph( json_graph, directed=True) components = {node.uuid: node for node in components_graph.nodes()} @@ -565,7 +567,34 @@ def load_graph(self, graph_file): self.__setattr__(node.name, node) return self - def save(self, graph_file): + @staticmethod + def load_graphs(graphs_file, existing_graphs=None): + """ + Method to load back in a graph. The graph file should be saved down using the save method, and is a JSON representation of the graph + generated by the [networkx](https://networkx.github.io) library. + + :param graph_file: The file containing the primary model to load back for this inference algorithm. + :type graph_file: str of filename + """ + import json + from ..util.graph_serialization import ModelComponentDecoder + with open(graphs_file) as f: + graphs_list = json.load(f, cls=ModelComponentDecoder) + existing_graphs = existing_graphs if existing_graphs is not None else [FactorGraph(graph['name']) for graph in graphs_list] + return [existing_graph.load_from_json(graph) for existing_graph, graph in zip(existing_graphs, graphs_list)] + + def as_json(self): + """ + Returns the FactorGraph in a form suitable for JSON serialization. + This is assuming a JSON serializer that knows how to handle ModelComponents + such as the one defined in mxfusion.util.graph_serialization. + """ + json_graph = nx.readwrite.json_graph.node_link_data(self._components_graph) + json_graph['name'] = self.name + return json_graph + + @staticmethod + def save(graph_file, json_graphs): """ Method to save this graph down into a file. The graph file will be saved down as a JSON representation of the graph generated by the [networkx](https://networkx.github.io) library. @@ -573,8 +602,9 @@ def save(self, graph_file): :param graph_file: The file containing the primary model to load back for this inference algorithm. :type graph_file: str of filename """ - json_graph = nx.readwrite.json_graph.node_link_data(self._components_graph) + json_graphs = [json_graphs] if not isinstance(json_graphs, type([])) else json_graphs import json from ..util.graph_serialization import ModelComponentEncoder - with open(graph_file, 'w') as f: - json.dump(json_graph, f, ensure_ascii=False, cls=ModelComponentEncoder) + if graph_file is not None: + with open(graph_file, 'w') as f: + json.dump(json_graphs, f, ensure_ascii=False, cls=ModelComponentEncoder) diff --git a/mxfusion/models/posterior.py b/mxfusion/models/posterior.py index be31284..0fe6e93 100644 --- a/mxfusion/models/posterior.py +++ b/mxfusion/models/posterior.py @@ -21,14 +21,14 @@ class Posterior(FactorGraph): A Posterior graph defined over an existing model. """ - def __init__(self, model, name=None): + def __init__(self, model, name=None, verbose=False): """ Constructor. :param model: The model which the posterior graph is defined over. :type model: Model """ - super(Posterior, self).__init__(name=name) + super(Posterior, self).__init__(name=name, verbose=verbose) self._model = model def __getattr__(self, name): @@ -60,3 +60,7 @@ def _replicate_class(self, **kwargs): Return a new instance of the derived FactorGraph's class. """ return Posterior(**kwargs) + + def clone(self, model, leaves=None): + new_model = self._replicate_class(model=model, name=self.name, verbose=self._verbose) + return self._clone(new_model, leaves) diff --git a/mxfusion/modules/__init__.py b/mxfusion/modules/__init__.py index 4fcbad0..af96b91 100644 --- a/mxfusion/modules/__init__.py +++ b/mxfusion/modules/__init__.py @@ -26,3 +26,5 @@ """ __all__ = ['module', 'gp_modules'] + +from .module import Module diff --git a/mxfusion/modules/gp_modules/gp_regression.py b/mxfusion/modules/gp_modules/gp_regression.py index a30bf2c..2571ed9 100644 --- a/mxfusion/modules/gp_modules/gp_regression.py +++ b/mxfusion/modules/gp_modules/gp_regression.py @@ -366,5 +366,5 @@ def replicate_self(self, attribute_map=None): rep = super(GPRegression, self).replicate_self(attribute_map) rep.kernel = self.kernel.replicate_self(attribute_map) - rep.mean_func = self.mean_func.replicate_self(attribute_map) + rep.mean_func = None if self.mean_func is None else self.mean_func.replicate_self(attribute_map) return rep diff --git a/mxfusion/modules/gp_modules/sparsegp_regression.py b/mxfusion/modules/gp_modules/sparsegp_regression.py index 382298a..c10511e 100644 --- a/mxfusion/modules/gp_modules/sparsegp_regression.py +++ b/mxfusion/modules/gp_modules/sparsegp_regression.py @@ -381,5 +381,5 @@ def replicate_self(self, attribute_map=None): rep = super(SparseGPRegression, self).replicate_self(attribute_map) rep.kernel = self.kernel.replicate_self(attribute_map) - rep.mean_func = self.mean_func.replicate_self(attribute_map) + rep.mean_func = None if self.mean_func is None else self.mean_func.replicate_self(attribute_map) return rep diff --git a/mxfusion/modules/gp_modules/svgp_regression.py b/mxfusion/modules/gp_modules/svgp_regression.py index f132cf3..7956a4e 100644 --- a/mxfusion/modules/gp_modules/svgp_regression.py +++ b/mxfusion/modules/gp_modules/svgp_regression.py @@ -397,5 +397,5 @@ def replicate_self(self, attribute_map=None): rep = super(SVGPRegression, self).replicate_self(attribute_map) rep.kernel = self.kernel.replicate_self(attribute_map) - rep.mean_func = self.mean_func.replicate_self(attribute_map) + rep.mean_func = None if self.mean_func is None else self.mean_func.replicate_self(attribute_map) return rep diff --git a/mxfusion/modules/module.py b/mxfusion/modules/module.py index d1b7a5a..efceff7 100644 --- a/mxfusion/modules/module.py +++ b/mxfusion/modules/module.py @@ -14,14 +14,14 @@ import warnings -from mxnet.gluon import ParameterDict from mxnet import initializer -from ..components.variables.variable import VariableType -from ..components.factor import Factor +from mxnet.gluon import ParameterDict +from ..common.config import get_default_dtype from ..components.distributions.random_gen import MXNetRandomGenerator from ..common.exceptions import ModelSpecificationError +from ..components.factor import Factor +from ..components.variables.variable import VariableType from ..util.inference import realize_shape -from ..common.config import get_default_dtype class Module(Factor): @@ -189,24 +189,10 @@ def attach_log_pdf_algorithms(self, targets, conditionals, algorithm, the module. :type algorithm: InferenceAlgorithm """ - if targets is not None: - targets = tuple(sorted(targets)) - if conditionals is not None: - conditionals = tuple(sorted(conditionals)) - if (targets, conditionals) in self._log_pdf_algorithms: - old_name = self._log_pdf_algorithms[(targets, conditionals)][1] - if old_name is not None: - delattr(self, old_name) - if alg_name is not None: - if not hasattr(self, alg_name): - setattr(self, alg_name, algorithm) - else: - warnings.warn('The algorithm name '+str(alg_name)+' has already existed in the module '+str(self)+'. Skip the attribute setting.') - alg_name = None - self._log_pdf_algorithms[(targets, conditionals)] = (algorithm, alg_name) + self._attach_algorithm(self._log_pdf_algorithms, targets, conditionals, algorithm, alg_name) def attach_draw_samples_algorithms(self, targets, conditionals, algorithm, - alg_name=None): + alg_name=None): """ Attach an inference algorithm for drawing samples from the module. @@ -217,37 +203,11 @@ def attach_draw_samples_algorithms(self, targets, conditionals, algorithm, :param algorithm: the inference algorithm to draw samples of the chosen target variables from the module. :type algorithm: InferenceAlgorithm """ - from ..inference.inference_alg import InferenceAlgorithm - if targets is not None: - targets = tuple(sorted(targets)) - if conditionals is not None: - conditionals = tuple(sorted(conditionals)) - if alg_name is not None: - if not hasattr(self, alg_name): - setattr(self, alg_name, algorithm) - elif isinstance(getattr(self, alg_name), InferenceAlgorithm): - setattr(self, alg_name, algorithm) - else: - warnings.warn('The algorithm name '+str(alg_name)+' has already existed in the module '+str(self)+'. Skip the attribute setting.') - alg_name = None - if conditionals in self._draw_samples_algorithms: - methods = self._draw_samples_algorithms[conditionals] - no_match = True - for i, m in enumerate(methods): - if targets == m[0]: - if m[2] is not None and m[2] != alg_name: - delattr(self, m[2]) - methods[i] = (targets, algorithm, alg_name) - no_match = False - break - if no_match: - self._draw_samples_algorithms[conditionals].append( - (targets, algorithm, alg_name)) - else: - self._draw_samples_algorithms[conditionals] = [(targets, algorithm, alg_name)] + self._attach_algorithm(self._draw_samples_algorithms, targets, conditionals, algorithm, alg_name) + def attach_prediction_algorithms(self, targets, conditionals, algorithm, - alg_name=None): + alg_name=None): """ Attach an inference algorithm for prediction from the module. @@ -258,35 +218,68 @@ def attach_prediction_algorithms(self, targets, conditionals, algorithm, :param algorithm: the inference algorithm to predict the chosen target variables from the module. :type algorithm: InferenceAlgorithm """ - from ..inference.inference_alg import InferenceAlgorithm + self._attach_algorithm(self._prediction_algorithms, targets, conditionals, algorithm, alg_name) + + def _attach_algorithm(self, algorithms, targets, conditionals, algorithm, alg_name): + """ + Attaches the given algorithm to the algorithms data structure based on targets, conditionals, and alg_name. + Also sets 'm.{alg_name} = algorithm'. + """ + targets, conditionals = self._preprocess_attach_parameters(targets, conditionals) + alg_name = self._set_algorithm_name(alg_name, algorithm) + if conditionals in algorithms: + return self._attach_duplicate_conditional_algorithm(algorithms, targets, conditionals, algorithm, alg_name) + else: + algorithms[conditionals] = [(targets, algorithm, alg_name)] + return algorithms + + def _preprocess_attach_parameters(self, targets, conditionals): + """ + Sorts and returns as tuples the targets and conditionals used during attachment. + """ if targets is not None: targets = tuple(sorted(targets)) if conditionals is not None: conditionals = tuple(sorted(conditionals)) + return targets, conditionals + + def _set_algorithm_name(self, alg_name, algorithm): + """ + Sets the attribute of self with the algorithm name, overriding an old algorithm that had the same name. If something other than an InferenceAlgorithm has that name, prints a warning and returns None for alg_name. + """ + + from ..inference.inference_alg import InferenceAlgorithm if alg_name is not None: if not hasattr(self, alg_name): setattr(self, alg_name, algorithm) elif isinstance(getattr(self, alg_name), InferenceAlgorithm): setattr(self, alg_name, algorithm) else: - warnings.warn('The algorithm name '+str(alg_name)+' has already existed in the module '+str(self)+'. Skip the attribute setting.') + warnings.warn('Something ({}) in this module ({}) is already using the attribute \"{}\". Skipping setting that name to the algorithm.'.format(str(getattr(self, alg_name)),str(self), str(alg_name))) alg_name = None - if conditionals in self._prediction_algorithms: - methods = self._prediction_algorithms[conditionals] - no_match = True - for i, m in enumerate(methods): - if targets == m[0]: - if m[2] is not None and m[2] != alg_name: - delattr(self, m[2]) - methods[i] = (targets, algorithm, alg_name) - no_match = False - break - if no_match: - self._prediction_algorithms[conditionals].append( - (targets, algorithm, alg_name)) - else: - self._prediction_algorithms[conditionals] = [(targets, algorithm, - alg_name)] + return alg_name + + def _attach_duplicate_conditional_algorithm(self, algorithms, targets, conditionals, algorithm, alg_name): + """ + Mutates the algorithms object, adding the new algorithm to it. + Also removes the name of an old inference algorithm if it had the same (targets, conditional) pair as the new algorithm. + """ + methods = algorithms[conditionals] + no_match = True + # For each algorithm that already uses those same conditionals + for i, (i_targets, i_algorithm, i_name) in enumerate(methods): + # If the targets are also the same, remove the old one + # because this (targets, conditionals) pair should be unique across algorithms. + if targets == i_targets: + # remove the name of the old algorithm + if i_name is not None and i_name != alg_name: + delattr(self, i_name) + methods[i] = (targets, algorithm, alg_name) + no_match = False + break + if no_match: + algorithms[conditionals].append( + (targets, algorithm, alg_name)) def log_pdf(self, F, variables, targets=None): """ @@ -302,18 +295,9 @@ def log_pdf(self, F, variables, targets=None): :returns: the sum of the log probability of all the target variables. :rtype: mxnet NDArray or mxnet Symbol """ - if targets is None: - target_names = tuple(sorted(self.output_names.copy())) - else: - target_names = self.get_names_from_uuid(targets) - conditionals_names = self.get_names_from_uuid(variables.keys()) - conditionals_names = tuple(sorted(set(conditionals_names) - set(target_names))) - - if (target_names, conditionals_names) in self._log_pdf_algorithms: - alg = self._log_pdf_algorithms[(target_names, conditionals_names)][0] - else: - raise ModelSpecificationError("The targets, conditionals pattern for log_pdf computation "+str((target_names, conditionals_names))+" cannot find a matched inference algorithm.") - return alg.compute(F, variables) + alg = self._get_algorithm_for_target_conditional_pair(self._log_pdf_algorithms, targets, variables, exact_match=True) + result = alg.compute(F, variables) + return result def draw_samples(self, F, variables, num_samples=1, targets=None): """ @@ -330,25 +314,14 @@ def draw_samples(self, F, variables, num_samples=1, targets=None): :returns: the samples of the target variables. :rtype: (MXNet NDArray or MXNet Symbol,) or {str(UUID): MXNet NDArray or MXNet Symbol} """ - if targets is None: - target_names = tuple(sorted(self.output_names.copy())) - else: - target_names = self.get_names_from_uuid(targets) - conditionals_names = self.get_names_from_uuid(variables.keys()) - - if conditionals_names in self._draw_samples_algorithms: - algs = self._draw_samples_algorithms[conditionals_names] - target_names = set(target_names) - for t, alg, _ in algs: - if target_names <= set(t): - alg.num_samples = num_samples - alg.target_variables = targets - return alg.compute(F, variables) - raise ModelSpecificationError("The targets-conditionals pattern for draw_samples computation "+str((target_names, conditionals_names))+" cannot find a matched inference algorithm.") + alg = self._get_algorithm_for_target_conditional_pair(self._draw_samples_algorithms, targets, variables) + alg.num_samples = num_samples + alg.target_variables = targets + return alg.compute(F, variables) def predict(self, F, variables, num_samples=1, targets=None): """ - prediction + Predict some variables. :param F: the MXNet computation mode (``mxnet.symbol`` or ``mxnet.ndarray``). :param variables: The set of variables @@ -360,21 +333,33 @@ def predict(self, F, variables, num_samples=1, targets=None): :returns: the sum of the log probability of all the target variables. :rtype: mxnet NDArray or mxnet Symbol """ + alg = self._get_algorithm_for_target_conditional_pair(self._prediction_algorithms, targets, variables) + alg.num_samples = num_samples + alg.target_variables = targets + return alg.compute(F, variables) + + def _get_algorithm_for_target_conditional_pair(self, algorithms, targets, variables, exact_match=False): + """ + Searches through the algorithms to find the right algorithm for the target/conditional pair. + :param exact_match: This indicates whether the targets passed in must be precisely those in the algorithm, or whether a subset of targets will suffice. + """ if targets is None: target_names = tuple(sorted(self.output_names.copy())) else: target_names = self.get_names_from_uuid(targets) conditionals_names = self.get_names_from_uuid(variables.keys()) + conditionals_names = conditionals_names if not exact_match else tuple(sorted(set(conditionals_names) - set(target_names))) - if conditionals_names in self._prediction_algorithms: - algs = self._prediction_algorithms[conditionals_names] + if conditionals_names in algorithms: + algs = algorithms[conditionals_names] target_names = set(target_names) for t, alg, _ in algs: - if target_names <= set(t): - alg.target_variables = targets - alg.num_samples = num_samples - return alg.compute(F, variables) - raise ModelSpecificationError("The targets-conditionals pattern for prediction "+str((target_names, conditionals_names))+" cannot find a matched inference algorithm.") + if not exact_match and target_names <= set(t): + return alg + if exact_match and target_names == set(t): + return alg + + raise ModelSpecificationError("The targets-conditionals pattern for draw_samples computation "+str((target_names, conditionals_names))+" cannot find a matched inference algorithm.") def prepare_executor(self, rv_scaling=None): """ @@ -401,17 +386,57 @@ def prepare_executor(self, rv_scaling=None): v.factor.log_pdf_scaling = 1 return var_trans, excluded + def _clone_algorithms(self, algorithms, replicant): + """ + Clones all of the algorithms using the replicant graphs. + """ + algs = {} + for conditionals, algorithms in algorithms.items(): + for targets, algorithm, alg_name in algorithms: + graphs_index = {g: i for i,g in enumerate(self._extra_graphs)} + extra_graphs = [replicant._extra_graphs[graphs_index[graph]] for graph in algorithm.graphs if graph in graphs_index] + algs[conditionals] = (targets, algorithm.replicate_self(replicant._module_graph, extra_graphs), alg_name) + return algs + + def reconcile_with_module(self, previous_module): + from ..models import FactorGraph + current_graphs = [self._module_graph] + self._extra_graphs + primary_previous_graph = previous_module._module_graph + secondary_previous_graphs = previous_module._extra_graphs + primary_current_graph = self._module_graph + component_map = FactorGraph.reconcile_graphs(current_graphs, primary_previous_graph, secondary_previous_graphs=secondary_previous_graphs, primary_current_graph=primary_current_graph) + return component_map + def replicate_self(self, attribute_map=None): """ The copy constructor for the function. """ - rep = super(Module, self).replicate_self(attribute_map) + replicant = super(Module, self).replicate_self(attribute_map) + + replicant._rand_gen = self._rand_gen + replicant.dtype = self.dtype + replicant.ctx = self.ctx + replicant._module_graph = self._module_graph.clone() - rep._rand_gen = self._rand_gen - rep.dtype = self.dtype - rep.ctx = self.ctx - rep._module_graph = self._module_graph.replicate_self(attribute_map) - rep._extra_graphs = [m.replicate_self(attribute_map) for m in + # Note this assumes the extra graphs are A) posteriors and B) derived from self._module_graph. + replicant._extra_graphs = [m.clone(self._module_graph) for m in self._extra_graphs] - rep._attach_default_inference_algorithms() - return rep + + replicant._log_pdf_algorithms = self._clone_algorithms(self._log_pdf_algorithms, replicant) + replicant._draw_samples_algorithms = self._clone_algorithms(self._draw_samples_algorithms, replicant) + replicant._prediction_algorithms = self._clone_algorithms(self._prediction_algorithms, replicant) + return replicant + + def load_module(self, module_json): + from ..models import FactorGraph + self._module_graph = FactorGraph(module_json['graphs'][0]['name']).load_from_json(module_json['graphs'][0]) + if len(module_json['graphs']) > 1: + self._extra_graphs = [FactorGraph(extra_graph['name']).load_from_json(extra_graph) for extra_graph in module_json['graphs'][1:]] + return self + + + def as_json(self): + mod_dict = super(Module, self).as_json() + graphs = [g.as_json()for g in [self._module_graph] + self._extra_graphs] + mod_dict['graphs'] = graphs + return mod_dict diff --git a/mxfusion/util/graph_serialization.py b/mxfusion/util/graph_serialization.py index ea1ae1f..8242fbd 100644 --- a/mxfusion/util/graph_serialization.py +++ b/mxfusion/util/graph_serialization.py @@ -14,7 +14,7 @@ import json -from ..components import ModelComponent +import mxfusion as mf from ..common.exceptions import SerializationError @@ -27,16 +27,11 @@ def default(self, obj): """ Serializes a ModelComponent object. Note: does not serialize the successor attribute as it isn't necessary for serialization. """ - import mxfusion.components as mf - if isinstance(obj, mf.ModelComponent): - return { - "type": obj.__class__.__name__, - "uuid": obj.uuid, - "name": obj.name, - "inherited_name": obj.inherited_name if hasattr(obj, 'inherited_name') else None, - "attributes": [a.uuid for a in obj.attributes], - "version": __GRAPH_JSON_VERSION__ - } + if isinstance(obj, mf.components.ModelComponent): + object_dict = obj.as_json() + object_dict["version"] = __GRAPH_JSON_VERSION__ + object_dict["type"] = obj.__class__.__name__ + return object_dict return super(ModelComponentEncoder, self).default(obj) @@ -53,10 +48,14 @@ def object_hook(self, obj): return obj if obj['version'] != __GRAPH_JSON_VERSION__: raise SerializationError('The format of the stored model component '+str(obj['name'])+' is from an old version '+str(obj['version'])+'. The current version is '+__GRAPH_JSON_VERSION__+'. Backward compatibility is not supported yet.') - v = ModelComponent() + if 'graphs' in obj: + v = mf.modules.Module(None, None, None, None) + v.load_module(obj) + else: + v = mf.components.ModelComponent() + v.inherited_name = obj['inherited_name'] if 'inherited_name' in obj else None + v.name = obj['name'] v._uuid = obj['uuid'] v.attributes = obj['attributes'] - v.name = obj['name'] - v.inherited_name = obj['inherited_name'] v.type = obj['type'] return v diff --git a/testing/components/distributions/gp/cond_gp_test.py b/testing/components/distributions/gp/cond_gp_test.py index 84d5aff..1cacbb8 100644 --- a/testing/components/distributions/gp/cond_gp_test.py +++ b/testing/components/distributions/gp/cond_gp_test.py @@ -166,7 +166,7 @@ def test_clone_cond_gp(self, dtype, X, X_isSamples, X_cond, X_cond_isSamples, Y_ m.Y_cond_var = Variable(shape=(8,1)) m.Y = ConditionalGaussianProcess.define_variable(X=m.X_var, X_cond=m.X_cond_var, Y_cond=m.Y_cond_var, kernel=rbf, shape=rv_shape, dtype=dtype) - gp = m.clone()[0].Y.factor + gp = m.clone().Y.factor variables = {gp.X.uuid: X_mx, gp.X_cond.uuid: X_cond_mx, gp.Y_cond.uuid: Y_cond_mx, gp.rbf_lengthscale.uuid: rbf_lengthscale_mx, gp.rbf_variance.uuid: rbf_variance_mx, gp.random_variable.uuid: rv_mx} log_pdf_rt = gp.log_pdf(F=mx.nd, variables=variables).asnumpy() diff --git a/testing/components/distributions/gp/gp_test.py b/testing/components/distributions/gp/gp_test.py index a5283c6..28b99c5 100644 --- a/testing/components/distributions/gp/gp_test.py +++ b/testing/components/distributions/gp/gp_test.py @@ -131,7 +131,7 @@ def test_clone_gp(self, dtype, X, X_isSamples, rbf_lengthscale, rbf_lengthscale_ m.X_var = Variable(shape=(5,2)) m.Y = GaussianProcess.define_variable(X=m.X_var, kernel=rbf, shape=rv_shape, dtype=dtype) - gp = m.clone()[0].Y.factor + gp = m.clone().Y.factor variables = {gp.X.uuid: X_mx, gp.rbf_lengthscale.uuid: rbf_lengthscale_mx, gp.rbf_variance.uuid: rbf_variance_mx, gp.random_variable.uuid: rv_mx} log_pdf_rt = gp.log_pdf(F=mx.nd, variables=variables).asnumpy() diff --git a/testing/inference/inference_serialization_test.py b/testing/inference/inference_serialization_test.py index 438ee1a..ace8f26 100644 --- a/testing/inference/inference_serialization_test.py +++ b/testing/inference/inference_serialization_test.py @@ -63,6 +63,21 @@ def make_net(self): net.initialize(mx.init.Xavier(magnitude=3)) return net + def make_gpregr_model(self, lengthscale, variance, noise_var): + from mxfusion.models import Model + from mxfusion.components.variables import Variable, PositiveTransformation + from mxfusion.modules.gp_modules import GPRegression + from mxfusion.components.distributions.gp.kernels import RBF + + dtype = 'float64' + m = Model() + m.N = Variable() + m.X = Variable(shape=(m.N, 3)) + m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype)) + kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype) + m.Y = GPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, shape=(m.N, 1), dtype=dtype) + return m + def test_meanfield_saving(self): dtype = get_default_dtype() x = np.random.rand(1000, 1) @@ -126,8 +141,7 @@ def test_meanfield_save_and_load(self): infr2.initialize(y=y_nd, x=x_nd) # Load previous parameters - infr2.load(primary_model_file=self.PREFIX+'_graph_0.json', - secondary_graph_files=[self.PREFIX+'_graph_1.json'], + infr2.load(graphs_file=self.PREFIX+'_graphs.json', parameters_file=self.PREFIX+'_params.json', inference_configuration_file=self.PREFIX+'_configuration.json', mxnet_constants_file=self.PREFIX+'_mxnet_constants.json', @@ -150,3 +164,56 @@ def test_meanfield_save_and_load(self): infr2.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd) self.remove_saved_files(self.PREFIX) + + + def test_gp_module_save_and_load(self): + np.random.seed(0) + X = np.random.rand(10, 3) + Xt = np.random.rand(20, 3) + Y = np.random.rand(10, 1) + noise_var = np.random.rand(1) + lengthscale = np.random.rand(3) + variance = np.random.rand(1) + dtype = 'float64' + m = self.make_gpregr_model(lengthscale, variance, noise_var) + + observed = [m.X, m.Y] + from mxfusion.inference import MAP, Inference + infr = Inference(MAP(model=m, observed=observed), dtype=dtype) + + loss, _ = infr.run(X=mx.nd.array(X, dtype=dtype), Y=mx.nd.array(Y, dtype=dtype)) + + infr.save(prefix=self.PREFIX) + + + m2 = self.make_gpregr_model(lengthscale, variance, noise_var) + + observed2 = [m2.X, m2.Y] + infr2 = Inference(MAP(model=m2, observed=observed2), dtype=dtype) + infr2.initialize(X=mx.nd.array(X, dtype=dtype), Y=mx.nd.array(Y, dtype=dtype)) + + # Load previous parameters + infr2.load(graphs_file=self.PREFIX+'_graphs.json', + parameters_file=self.PREFIX+'_params.json', + inference_configuration_file=self.PREFIX+'_configuration.json', + mxnet_constants_file=self.PREFIX+'_mxnet_constants.json', + variable_constants_file=self.PREFIX+'_variable_constants.json') + + for original_uuid, original_param in infr.params.param_dict.items(): + original_data = original_param.data().asnumpy() + reloaded_data = infr2.params.param_dict[infr2._uuid_map[original_uuid]].data().asnumpy() + assert np.all(np.isclose(original_data, reloaded_data)) + + for original_uuid, original_param in infr.params.constants.items(): + if isinstance(original_param, mx.ndarray.ndarray.NDArray): + original_data = original_param.asnumpy() + reloaded_data = infr2.params.constants[infr2._uuid_map[original_uuid]].asnumpy() + else: + original_data = original_param + reloaded_data = infr2.params.constants[infr2._uuid_map[original_uuid]] + + assert np.all(np.isclose(original_data, reloaded_data)) + + loss2, _ = infr2.run(X=mx.nd.array(X, dtype=dtype), Y=mx.nd.array(Y, dtype=dtype)) + + self.remove_saved_files(self.PREFIX) diff --git a/testing/models/factor_graph_test.py b/testing/models/factor_graph_test.py index 66844d4..dd1dec2 100644 --- a/testing/models/factor_graph_test.py +++ b/testing/models/factor_graph_test.py @@ -23,8 +23,11 @@ import mxfusion as mf from mxfusion.common.exceptions import ModelSpecificationError from mxfusion.components.distributions.normal import Normal +from mxfusion.components.distributions.gp.kernels import RBF +from mxfusion.modules.gp_modules import GPRegression from mxfusion.components import Variable -from mxfusion.models import Model +from mxfusion.components.variables import PositiveTransformation +from mxfusion.models import Model, FactorGraph from mxfusion.components.variables.runtime_variable import add_sample_dimension, array_has_samples, get_num_samples from mxfusion.util.testutils import MockMXNetRandomGenerator @@ -34,18 +37,18 @@ class FactorGraphTests(unittest.TestCase): Tests the MXFusion.core.factor_graph.FactorGraph class. """ - def shape_match(self, old, new, var_map): + def shape_match(self, old, new): if len(old) != len(new): return False for o, n in zip(old, new): if isinstance(n, mfc.Variable): - if n != var_map[o]: + if n != o: return False elif n != o: return False return True - def make_model(self, net): + def make_bnn_model(self, net): component_set = set() m = mf.models.Model(verbose=False) m.N = mfc.Variable() @@ -61,13 +64,6 @@ def make_model(self, net): component_set.union(set([m.N, m.f, m.x, m.r, m.y])) return m, component_set - def make_simple_model(self): - m = Model() - mean = Variable() - variance = Variable() - m.r = Normal.define_variable(mean=mean, variance=variance) - return m - def make_net(self): D = 100 net = nn.HybridSequential(prefix='hybrid0_') @@ -78,6 +74,22 @@ def make_net(self): net.initialize(mx.init.Xavier(magnitude=3)) return net + def make_simple_model(self): + m = Model() + mean = Variable() + variance = Variable() + m.r = Normal.define_variable(mean=mean, variance=variance) + return m + + def make_gpregr_model(self): + m = Model() + m.N = Variable() + m.X = Variable(shape=(m.N, 3)) + m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array([1.])) + kernel = RBF(input_dim=3, variance=mx.nd.array([1.]), lengthscale=mx.nd.array([1.])) + m.Y = GPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, shape=(m.N, 2)) + return m + def setUp(self): self.TESTFILE = "testfile_" + str(uuid.uuid4()) + ".json" self.fg = mf.models.FactorGraph(name='test_fg') @@ -89,11 +101,10 @@ def setUp(self): def test_bnn_model(self): - bnn_fg, component_set = self.make_model(self.bnn_net) + bnn_fg, component_set = self.make_bnn_model(self.bnn_net) self.assertTrue(component_set <= set(bnn_fg.components_graph.nodes().keys()), "Variables are all added to _components_graph {} {}".format(component_set, bnn_fg.components_graph.nodes().keys())) self.assertTrue(component_set <= bnn_fg.components.keys(), "Variable is added to _components dict. {} {}".format(component_set, bnn_fg.components.keys())) - # assert False def test_add_unresolved_components_distribution(self): v = mfc.Variable() @@ -142,13 +153,22 @@ def test_remove_nonexistant_variable_failure(self): with self.assertRaises(ModelSpecificationError): self.fg.remove_component(v) + + def test_replicate_gp_model(self): + m = self.make_gpregr_model() + m2 = m.clone() + self.assertTrue(all([v in m.Y.factor._module_graph.components for v in m2.Y.factor._module_graph.components]), (set(m2.Y.factor._module_graph.components) - set(m.Y.factor._module_graph.components))) + self.assertTrue(all([v in m.Y.factor._extra_graphs[0].components for v in m2.Y.factor._extra_graphs[0].components]), (set(m2.Y.factor._extra_graphs[0].components) - set(m.Y.factor._extra_graphs[0].components))) + self.assertTrue(all([v in m.components for v in m2.components]), (set(m2.components) - set(m.components))) + self.assertTrue(all([v in m2.components for v in m.components]), (set(m.components) - set(m2.components))) + self.assertTrue(all([self.shape_match(m[i].shape, m2[i].shape) for i in m.variables]), (m.variables, m2.variables)) + def test_replicate_bnn_model(self): - m, component_set = self.make_model(self.bnn_net) - m2, var_map = m.clone() - self.assertTrue(all([k.uuid == v.uuid for k, v in var_map.items()])) + m, component_set = self.make_bnn_model(self.bnn_net) + m2 = m.clone() self.assertTrue(all([v in m.components for v in m2.components]), (set(m2.components) - set(m.components))) self.assertTrue(all([v in m2.components for v in m.components]), (set(m.components) - set(m2.components))) - self.assertTrue(all([self.shape_match(m[i].shape, m2[i].shape, var_map) for i in m.variables]), (m.variables, m2.variables)) + self.assertTrue(all([self.shape_match(m[i].shape, m2[i].shape) for i in m.variables]), (m.variables, m2.variables)) def test_replicate_simple_model(self): m = mf.models.Model(verbose=False) @@ -157,9 +177,8 @@ def test_replicate_simple_model(self): m.x_var = mfc.Variable(value=mx.nd.array([1e6])) d = mf.components.distributions.Normal(mean=m.x_mean, variance=m.x_var) m.x.set_prior(d) - m2, var_map = m.clone() + m2 = m.clone() # compare m and m2 components and such for exactness. - self.assertTrue(all([k.uuid == v.uuid for k, v in var_map.items()])) self.assertTrue(set([v.uuid for v in m.components.values()]) == set([v.uuid for v in m2.components.values()])) self.assertTrue(all([v in m.components for v in m2.components]), (set(m2.components) - set(m.components))) @@ -239,11 +258,17 @@ def test_reconcile_simple_model(self): self.assertTrue(len(component_map) == len(m1.components)) def test_reconcile_bnn_model(self): - m1, _ = self.make_model(self.make_net()) - m2, _ = self.make_model(self.make_net()) + m1, _ = self.make_bnn_model(self.make_net()) + m2, _ = self.make_bnn_model(self.make_net()) component_map = mf.models.FactorGraph.reconcile_graphs([m1], m2) self.assertTrue(len(component_map) == len(m1.components)) + def test_reconcile_gp_model(self): + m1 = self.make_gpregr_model() + m2 = self.make_gpregr_model() + component_map = mf.models.FactorGraph.reconcile_graphs([m1], m2) + self.assertTrue(len(component_map) == len(set(m1.components).union(set(m1.Y.factor._module_graph.components)).union(set(m1.Y.factor._extra_graphs[0].components)))) + def test_reconcile_model_and_posterior(self): x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) @@ -253,8 +278,8 @@ def test_reconcile_model_and_posterior(self): net1(x_nd) net2 = self.make_net() net2(x_nd) - m1, _ = self.make_model(net1) - m2, _ = self.make_model(net2) + m1, _ = self.make_bnn_model(net1) + m2, _ = self.make_bnn_model(net2) from mxfusion.inference.meanfield import create_Gaussian_meanfield from mxfusion.inference import StochasticVariationalInference @@ -273,10 +298,10 @@ def test_reconcile_model_and_posterior(self): set(alg1.graphs[1].components.values()))) def test_save_reload_bnn_graph(self): - m1, _ = self.make_model(self.make_net()) - m1.save(self.TESTFILE) + m1, _ = self.make_bnn_model(self.make_net()) + FactorGraph.save(self.TESTFILE, m1.as_json()) m1_loaded = Model() - m1_loaded.load_graph(self.TESTFILE) + FactorGraph.load_graphs(self.TESTFILE, [m1_loaded]) m1_loaded_edges = set(m1_loaded.components_graph.edges()) m1_edges = set(m1.components_graph.edges()) @@ -288,9 +313,9 @@ def test_save_reload_bnn_graph(self): def test_save_reload_then_reconcile_simple_graph(self): m1 = self.make_simple_model() - m1.save(self.TESTFILE) + FactorGraph.save(self.TESTFILE, m1.as_json()) m1_loaded = Model() - m1_loaded.load_graph(self.TESTFILE) + FactorGraph.load_graphs(self.TESTFILE, [m1_loaded]) self.assertTrue(set(m1.components) == set(m1_loaded.components)) m2 = self.make_simple_model() @@ -316,14 +341,45 @@ def test_save_reload_then_reconcile_simple_graph(self): import os os.remove(self.TESTFILE) + def test_save_reload_then_reconcile_gp_module(self): + m1 = self.make_gpregr_model() + FactorGraph.save(self.TESTFILE, m1.as_json()) + m1_loaded = Model() + FactorGraph.load_graphs(self.TESTFILE, [m1_loaded]) + self.assertTrue(set(m1.components) == set(m1_loaded.components)) + self.assertTrue(len(set(m1.Y.factor._module_graph.components)) == len(set(m1_loaded[m1.Y.factor.uuid]._module_graph.components))) + self.assertTrue(len(set(m1.Y.factor._extra_graphs[0].components)) == len(set(m1_loaded[m1.Y.factor.uuid]._extra_graphs[0].components))) + + m2 = self.make_gpregr_model() + component_map = mf.models.FactorGraph.reconcile_graphs([m2], m1_loaded) + self.assertTrue(len(component_map.values()) == len(set(component_map.values())), "Assert there are only 1:1 mappings.") + sort_m1 = list(set(map(lambda x: x.uuid, set(m1.components.values()).union(set(m1.Y.factor._module_graph.components.values())).union(set(m1.Y.factor._extra_graphs[0].components.values())) ))) + sort_m1.sort() + + sort_m2 = list(set(map(lambda x: x.uuid, set(m2.components.values()).union(set(m2.Y.factor._module_graph.components.values())).union(set(m2.Y.factor._extra_graphs[0].components.values())) ))) + sort_m2.sort() + + sort_component_map_values = list(set(component_map.values())) + sort_component_map_values.sort() + + sort_component_map_keys = list(set(component_map.keys())) + sort_component_map_keys.sort() + + zippy_values = zip(sort_m2, sort_component_map_values) + zippy_keys = zip(sort_m1, sort_component_map_keys) + self.assertTrue(all([m1_item == component_map_item for m1_item, component_map_item in zippy_values])) + self.assertTrue(all([m2_item == component_map_item for m2_item, component_map_item in zippy_keys])) + import os + os.remove(self.TESTFILE) + def test_save_reload_then_reconcile_bnn_graph(self): - m1, _ = self.make_model(self.make_net()) - m1.save(self.TESTFILE) + m1, _ = self.make_bnn_model(self.make_net()) + FactorGraph.save(self.TESTFILE, m1.as_json()) m1_loaded = Model() - m1_loaded.load_graph(self.TESTFILE) + FactorGraph.load_graphs(self.TESTFILE, [m1_loaded]) self.assertTrue(set(m1.components) == set(m1_loaded.components)) - m2, _ = self.make_model(self.make_net()) + m2, _ = self.make_bnn_model(self.make_net()) component_map = mf.models.FactorGraph.reconcile_graphs([m2], m1_loaded) self.assertTrue(len(component_map.values()) == len(set(component_map.values())), "Assert there are only 1:1 mappings.") self.assertTrue(len(component_map) == len(m1.components)) @@ -347,5 +403,5 @@ def test_save_reload_then_reconcile_bnn_graph(self): os.remove(self.TESTFILE) def test_print_fg(self): - m, component_set = self.make_model(self.bnn_net) + m, component_set = self.make_bnn_model(self.bnn_net) print(m) From 9b4b57a3ba49956136b854c3394ad9c94979c6b4 Mon Sep 17 00:00:00 2001 From: Eric Meissner Date: Tue, 6 Nov 2018 13:39:06 +0000 Subject: [PATCH 67/70] Fix bug to allow the same variable for multiple inputs to a factor --- mxfusion/components/factor.py | 4 ++-- mxfusion/components/model_component.py | 8 ++++---- mxfusion/models/factor_graph.py | 2 +- requirements/test_requirements.txt | 2 +- testing/components/model_component_test.py | 14 +++++++------- testing/models/factor_graph_test.py | 21 +++++++++++++++++++++ 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/mxfusion/components/factor.py b/mxfusion/components/factor.py index 02f087f..514dfb6 100644 --- a/mxfusion/components/factor.py +++ b/mxfusion/components/factor.py @@ -150,7 +150,7 @@ def inputs(self): Return a list of nodes whose edges point into this node. """ if self.graph is not None: - pred = {e['name']: v for v, e in self.graph.pred[self].items()} + pred = {e['name']: v for v, edges in self.graph.pred[self].items() for e in edges.values()} return [(name, pred[name]) for name in self.input_names] else: return self._predecessors @@ -161,7 +161,7 @@ def outputs(self): Return a list of nodes pointed to by the edges of this node. """ if self.graph is not None: - succ = {e['name']: v for v, e in self.graph.succ[self].items()} + succ = {e['name']: v for v, edges in self.graph.succ[self].items() for e in edges.values()} return [(name, succ[name]) for name in self.output_names] else: return self._successors diff --git a/mxfusion/components/model_component.py b/mxfusion/components/model_component.py index a1f5326..0d6c569 100644 --- a/mxfusion/components/model_component.py +++ b/mxfusion/components/model_component.py @@ -129,7 +129,7 @@ def successors(self): Note: The ordering of this list is not guaranteed to be consistent with assigned order. """ if self.graph is not None: - succ = [(e['name'], v) for v, e in self.graph.succ[self].items()] + succ = [(e['name'], v) for v, edges in self.graph.succ[self].items() for e in edges.values()] return succ else: return self._successors @@ -154,7 +154,7 @@ def add_predecessor(successor, predecessor, successor_name): self.graph.remove_edge(self, successor) for name, successor in successors: successor.graph = self.graph - self.graph.add_edge(self, successor, name=name) + self.graph.add_edge(self, successor, key=name, name=name) else: self._successors = successors for name, successor in successors: @@ -169,7 +169,7 @@ def predecessors(self): Note: The ordering of this list is not guaranteed to be consistent with assigned order. """ if self.graph is not None: - pred = [(e['name'], v) for v, e in self.graph.pred[self].items()] + pred = [(e['name'], v) for v, edges in self.graph.pred[self].items() for e in edges.values()] return pred else: return self._predecessors @@ -194,7 +194,7 @@ def add_successor(predecessor, successor, predecessor_name): self.graph.remove_edge(predecessor, self) for name, predecessor in predecessors: predecessor.graph = self.graph - self.graph.add_edge(predecessor, self, name=name) + self.graph.add_edge(predecessor, self, key=name, name=name) else: self._predecessors = predecessors for name, predecessor in predecessors: diff --git a/mxfusion/models/factor_graph.py b/mxfusion/models/factor_graph.py index 6af3b7c..58566ba 100644 --- a/mxfusion/models/factor_graph.py +++ b/mxfusion/models/factor_graph.py @@ -43,7 +43,7 @@ def __init__(self, name, verbose=False): self._uuid = str(uuid4()) self._var_ties = {} - self._components_graph = nx.DiGraph() + self._components_graph = nx.MultiDiGraph() self._verbose = verbose def __repr__(self): diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt index d855858..4c7d901 100644 --- a/requirements/test_requirements.txt +++ b/requirements/test_requirements.txt @@ -4,7 +4,7 @@ flake8>=3.5.0 pytest>=3.5.1 pytest-cov>=2.5.1 scipy>=1.1.0 -GPy>=1.8.5 +GPy>=1.9.6 matplotlib scikit-learn>=0.20.0 mxnet>=1.3 diff --git a/testing/components/model_component_test.py b/testing/components/model_component_test.py index da2d69b..9398558 100644 --- a/testing/components/model_component_test.py +++ b/testing/components/model_component_test.py @@ -34,7 +34,7 @@ def test_switch_simple_backwards(self): # # successors = set([(node_b.uuid, node_a.uuid), (node_c.uuid, node_a.uuid)]) - graph = nx.DiGraph() + graph = nx.MultiDiGraph() node_a.graph = graph @@ -51,7 +51,7 @@ def test_switch_simple_forwards(self): node_b.successors = [('edge_1', node_a)] node_c.successors = [('edge_2', node_a)] - graph = nx.DiGraph() + graph = nx.MultiDiGraph() node_c.graph = graph @@ -69,7 +69,7 @@ def test_switch_multilayer(self): node_a.predecessors = [('edge_1', node_b), ('edge_2', node_c)] node_b.predecessors = [('edge_1', node_d), ('edge_2', node_e)] - graph = nx.DiGraph() + graph = nx.MultiDiGraph() node_a.graph = graph @@ -84,7 +84,7 @@ def test_switch_multilayer(self): def test_join_attach_new_successor_not_to_graph(self): node_a = mfc.ModelComponent() - graph = nx.DiGraph() + graph = nx.MultiDiGraph() node_b = mfc.ModelComponent() node_d = mfc.ModelComponent() @@ -107,7 +107,7 @@ def test_join_attach_new_successor_not_to_graph(self): def test_join_predecessors_not_in_graph_to_node_in_graph(self): node_a = mfc.ModelComponent() - graph = nx.DiGraph() + graph = nx.MultiDiGraph() node_a.graph = graph node_b = mfc.ModelComponent() @@ -130,7 +130,7 @@ def test_join_successors_not_in_graph_to_node_in_graph(self): node_d = mfc.ModelComponent() node_e = mfc.ModelComponent() node_b.successors = [('edge_1', node_d), ('edge_2', node_e)] - graph = nx.DiGraph() + graph = nx.MultiDiGraph() node_a.graph = graph node_a.successors = [('edge_1', node_b)] @@ -148,7 +148,7 @@ def test_multiple_successors_same_name(self): node_c = mfc.ModelComponent() node_a.predecessors = [('edge_1', node_b), ('edge_1', node_c)] - graph = nx.DiGraph() + graph = nx.MultiDiGraph() node_a.graph = graph diff --git a/testing/models/factor_graph_test.py b/testing/models/factor_graph_test.py index dd1dec2..a55b2cf 100644 --- a/testing/models/factor_graph_test.py +++ b/testing/models/factor_graph_test.py @@ -198,6 +198,27 @@ def test_set_prior_after_factor_attach(self): self.assertTrue(set([v for _, v in x.predecessors]) == set([d])) self.assertTrue(x.graph == d.graph and d.graph == fg.components_graph) + def test_same_variable_as_multiple_inputs_to_factor_in_graph(self): + fg = mf.models.Model() + + fg.x = mfc.Variable() + fg.y = mf.components.distributions.Normal.define_variable(mean=fg.x, variance=fg.x) + + self.assertTrue(set([v for _, v in fg.y.factor.predecessors]) == set([fg.x])) + self.assertTrue(set([v for _, v in fg.x.successors]) == set([fg.y.factor])) + self.assertTrue(len(fg.y.factor.predecessors) == 2) + self.assertTrue(len(fg.x.successors) == 2) + + def test_same_variable_as_multiple_inputs_to_factor_not_in_graph(self): + + x = mfc.Variable() + y = mf.components.distributions.Normal.define_variable(mean=x, variance=x) + + self.assertTrue(set([v for _, v in y.factor.predecessors]) == set([x])) + self.assertTrue(set([v for _, v in x.successors]) == set([y.factor])) + self.assertTrue(len(y.factor.predecessors) == 2) + self.assertTrue(len(x.successors) == 2) + def test_compute_log_prob(self): m = Model() v = Variable(shape=(1,)) From 2a05a59d2b3712a1cb735f7e2fb9d9a940ae74a9 Mon Sep 17 00:00:00 2001 From: Eric R Meissner Date: Wed, 7 Nov 2018 10:09:52 +0000 Subject: [PATCH 68/70] Fix empty operator bug (#113) --- mxfusion/components/functions/operators/operators.py | 6 +++++- testing/components/functions/operators_test.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/mxfusion/components/functions/operators/operators.py b/mxfusion/components/functions/operators/operators.py index 08f748f..9fce1ac 100644 --- a/mxfusion/components/functions/operators/operators.py +++ b/mxfusion/components/functions/operators/operators.py @@ -13,6 +13,7 @@ # ============================================================================== +from ....common.exceptions import ModelSpecificationError from ..function_evaluation import FunctionEvaluation, FunctionEvaluationDecorator from ...variables import Variable @@ -82,7 +83,10 @@ def eval(self, F, **input_kws): input_kws.update(self.properties) return func(F, **input_kws) - op = CustomOperator(inputs=[(n, all_args[n]) for n in self.input_names if n in all_args], + if not len(all_args) >= len(self.input_names): + raise ModelSpecificationError("Must pass in arguments matching the input names {} but received {}.".format(self.input_names, all_args)) + + op = CustomOperator(inputs=[(n, all_args[n]) for n in self.input_names], outputs=[('output_'+str(i), Variable()) for i in range(self.num_outputs)], operator_name=self.operator_name, properties={n: all_args[n] for n in self.property_names if n in all_args} diff --git a/testing/components/functions/operators_test.py b/testing/components/functions/operators_test.py index 996e0dd..196c078 100644 --- a/testing/components/functions/operators_test.py +++ b/testing/components/functions/operators_test.py @@ -17,6 +17,7 @@ import mxnet as mx import numpy as np from mxfusion import Variable, Model +from mxfusion.common.exceptions import ModelSpecificationError from mxfusion.components.functions.operators import * @@ -128,3 +129,13 @@ def test_operators(self, mxf_operator, mxnet_operator, inputs, properties): inputs_unsampled = [v[0] for v in inputs] mxnet_result = mxnet_operator(*inputs_unsampled, **properties) assert np.allclose(mxf_result.asnumpy(), mxnet_result.asnumpy()), (mxf_result, mxnet_result) + + + @pytest.mark.parametrize("mxf_operator", [ + (add), + (reshape), + ]) + def test_empty_operator(self, mxf_operator): + with pytest.raises(ModelSpecificationError, message="Operator should fail if not passed the correct arguments.") as excinfo: + mxf_result = mxf_operator() + assert excinfo.value is not None From b7347699f9801a8300bbecdb2d21dbc94bd43aa2 Mon Sep 17 00:00:00 2001 From: Mark Pullin Date: Thu, 8 Nov 2018 16:46:19 +0000 Subject: [PATCH 69/70] Small fixes to doc --- docs/design_documents/model_definition.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/design_documents/model_definition.md b/docs/design_documents/model_definition.md index ac5f333..2e20780 100644 --- a/docs/design_documents/model_definition.md +++ b/docs/design_documents/model_definition.md @@ -28,7 +28,7 @@ When a ModelComponent is attached to a Model, it is automatically updated in the All ModelComponents in MXFusion are identified uniquely by a UUID. -###Variables +### Variables In a model, there are typically four types of variables: a random variable following a probabilistic distribution, a variable which is the outcome of a deterministic function, a parameter (with no prior distribution), and a @@ -52,7 +52,7 @@ values are always positive (v>=0). ### Factors -####Distributions +#### Distributions In a probabilistic model, random variables relate to each other through probabilistic distributions. @@ -61,7 +61,7 @@ random variable *x* from a zero mean unit variance Gaussian distribution looks like: ```python -m.x = Normal.generate_variable(mean=0, variance=1, shape=(2,)) +m.x = Normal.define_variable(mean=0, variance=1, shape=(2,)) ``` The two dimensions are @@ -72,7 +72,7 @@ example: ```python m.mean = Variable(shape=(2,)) m.y_shape = Variable() -m.y = Normal.generate_variable(mean=m.mean, variance=1, shape=m.y_shape) +m.y = Normal.define_variable(mean=m.mean, variance=1, shape=m.y_shape) ``` MXFusion also allows users to specify a prior distribution over pre-existing @@ -95,11 +95,11 @@ Because Models are FactorGraphs, it is common to want to know what ModelComponen ```python m.mean = Variable() m.var = Variable() -m.y = Normal.generate_variable(mean=m.mean, variance=m.var) +m.y = Normal.define_variable(mean=m.mean, variance=m.var) ``` -####Functions +#### Functions The last building block of probabilistic models are deterministic functions. The ability to define sophisticated functions allows users to build expressive models with a family of standard probabilistic distributions. As MXNet already From 006df6b02ba07abe473c735e767d408752ecbf00 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 9 Nov 2018 10:04:07 +0000 Subject: [PATCH 70/70] Add the tutorial for Gaussian process regression. (#114) --- docs/tutorials.md | 1 + examples/notebooks/gp_regression.ipynb | 445 +++++++++++++++++++++++++ 2 files changed, 446 insertions(+) create mode 100644 examples/notebooks/gp_regression.ipynb diff --git a/docs/tutorials.md b/docs/tutorials.md index db4a478..beb28fb 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -7,6 +7,7 @@ Below is a list of tutorial / example notebooks demonstrating MXFusion's functio * [Bayesian Neural Network Classification](examples/notebooks/bnn_classification.ipynb) * [Bayesian Neural Network Regression](examples/notebooks/bnn_regression.ipynb) * [Variational Auto-Encoder](examples/notebooks/variational_auto_encoder.ipynb) +* [Gaussian Process Regression](examples/notebooks/gp_regression.ipynb) ## Developer Tutorials * [Writing your own Distribution](examples/notebooks/writing_a_new_distribution.ipynb) diff --git a/examples/notebooks/gp_regression.ipynb b/examples/notebooks/gp_regression.ipynb new file mode 100644 index 0000000..b306327 --- /dev/null +++ b/examples/notebooks/gp_regression.ipynb @@ -0,0 +1,445 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gaussian Process Regression\n", + "\n", + "**Zhenwen Dai (2018-11-2)**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "Gaussian process (GP) is a Bayesian non-parametric model used for various machine learning problems such as regression, classification. This notebook shows about how to use a Gaussian process regression model in MXFusion." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings('ignore')\n", + "import os\n", + "os.environ['MXNET_ENGINE_TYPE'] = 'NaiveEngine'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Toy data\n", + "\n", + "We generate some synthetic data for our regression example. The data set is generate from a sine function with some additive Gaussian noise. " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "%matplotlib inline\n", + "from pylab import *\n", + "\n", + "np.random.seed(0)\n", + "X = np.random.uniform(-3.,3.,(20,1))\n", + "Y = np.sin(X) + np.random.randn(20,1)*0.05" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The generated data are visualized as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAGoRJREFUeJzt3X2QVfWd5/H3R0SIzygdRQFhssRSQVG7caqymm1Fw4xToImLuEnEmhgm7LqTrSlBHaeBhZhSTK0ZN1lKTFQyw0SRjLFTMymj0o6morEbhxHBMeJzpzW2+DBEwIB+9497Gu653U0/nNt9nz6vqlv3nnN+597vbej76fM75/5+igjMzMy6HFTqAszMrLw4GMzMLMXBYGZmKQ4GMzNLcTCYmVmKg8HMzFIcDGZmluJgMDOzFAeDmZmlHFzqAgZj7NixMWnSpFKXYWZWUTZu3PhORNT11a4ig2HSpEm0tbWVugwzs4oi6bX+tHNXkpmZpTgYzMwsxcFgZmYpRTnHIOku4M+AtyNiag/bBfwt8KfATuCqiHgm2TYf+Juk6bciYs1gatizZw/t7e3s3r17MLvbAIwePZrx48czcuTIUpdiZkOgWCef7wG+B/yol+1/AkxJbucAq4BzJB0DLAXqgQA2SmqOiPcGWkB7eztHHHEEkyZNIpdDNhQigu3bt9Pe3s7kyZNLXY6ZDYGidCVFxOPAuwdoMgf4UeQ8BRwtaRzwBeDhiHg3CYOHgVmDqWH37t0ce+yxDoUhJoljjz3WR2ZWXlauhJaW9LqWltx6G7DhOsdwIvBG3nJ7sq639YPiUBge/jlbyRUGQUMDXHIJ/MVf5JZbWmDu3Nx6G7DhCoaePkniAOu7P4G0QFKbpLbOzs6iFmdmFaahIffBnx8OEtx7LyxZktu2bh00Npauxgo2XMHQDkzIWx4PdBxgfTcRsToi6iOivq6uzy/uldyyZcv4zne+c8A2P/3pT9m6deuQ1tHR0cFll13WZ7tvf/vbQ1qHWVE1NuY++OfO3R8EDzwA3/wmrFgBCxc6FDIYrmBoBq5Uzh8DH0TEm8BDwEWSxkgaA1yUrBtaZdIfORzBcMIJJ7B+/fo+2zkYbMgM1e9bY2MuALqCAGDVKmhqyt0Xvqb1W1GCQdKPgSeBkyW1S/qapG9I+kbS5J+Bl4FtwJ3AfweIiHeBFUBrcluerBtahYehReqPvOmmmzj55JOZOXMmL7zwwr71d955Jw0NDZxxxhl86UtfYufOnfzqV7+iubmZRYsWMX36dF566aUe2xVatmwZX/3qVzn//POZMmUKd955J5C7WmjRokVMnTqVadOmcd999wHw6quvMnVq7grie+65hy9+8YvMmjWLKVOmsHjxYgCuv/56du3axfTp0/nyl7/Mhx9+yMUXX8wZZ5zB1KlT9z2X2aAM0e8bLS37g+D223PnGNatg+XL9x9NOBwGJyIq7nb22WdHoa1bt3Zbd0AbNkSMHRvR1JS737BhYPsXaGtri6lTp8aHH34YH3zwQXzmM5+JW2+9NSIi3nnnnX3tbrzxxrj99tsjImL+/Plx//3379vWW7t8S5cujdNPPz127twZnZ2dMX78+Pjtb38b69evj5kzZ8bevXvjrbfeigkTJkRHR0e88sorcdppp0VExN133x2TJ0+O999/P3bt2hUTJ06M119/PSIiDjvssH2vsX79+rj66qv3Lb///vvd6hjwz9tqW5F/3/Y9X9fzLFgQcdRR6efdsCHilluyvU6VAdqiH5+xtfvN58LD0Iz9kU888QSXXnophx56KEceeSSzZ8/et+25557j3HPPZdq0aaxdu5YtW7b0+Bz9bTdnzhw+9alPMXbsWBobG3n66af55S9/yRVXXMGIESM47rjj+PznP09ra2u3fS+44AKOOuooRo8ezamnnsprr3UfU2vatGk88sgjXHfddTzxxBMcddRRg/ypmCWK/PtGa2v65PIdd+TOMeT/n29shOSo2AamdoMh/zC0SP2RvV3GedVVV/G9732PzZs3s3Tp0l6/A9DfdoWvI4ncHwN9GzVq1L7HI0aMYO/evd3afPazn2Xjxo1MmzaNG264geXLl/fruc16Vezft8WLu4eLg6BoajMYuvo4i9gfed555/HAAw+wa9cuduzYwc9+9rN923bs2MG4cePYs2cPa9eu3bf+iCOOYMeOHX22K/Tggw+ye/dutm/fzmOPPUZDQwPnnXce9913Hx9//DGdnZ08/vjjzJgxo9/1jxw5kj179gC5K5kOPfRQvvKVr3DttdfyzDPPDORHYZY2BL9vNrQqcj6GzAoPQ7sufWttHfQh7llnncXll1/O9OnTOemkkzj33HP3bVuxYgXnnHMOJ510EtOmTdsXBvPmzePrX/86t99+O+vXr++1XaEZM2Zw8cUX8/rrr9PU1MQJJ5zApZdeypNPPskZZ5yBJFauXMnxxx/Pq6++2q/6FyxYwOmnn85ZZ53FlVdeyaJFizjooIMYOXIkq1atGtTPxAwYkt83G1rqbxdEOamvr4/CiXqef/55TjnllBJVNHyWLVvG4YcfzrXXXlvSOmrl521WTSRtjIj6vtrVZleSmZn1qja7kirYsmXLSl2CmVW5qjpiqMRusUrkn7NZdauaYBg9ejTbt2/3h9YQi2Q+htGjR5e6FDMbIlXTlTR+/Hja29vxyKtDr2sGNzOrTlUTDCNHjvSMYmZmRVA1XUlmZlYcDgYzM0txMJhZbSqTeVnKkYPBzGrTQOeJqKEgcTCYWU4NffABPU8PeqB5oodqwqEy5GAws5wa+uDbZyDzRAw0SCqYg8HMcmrog2+fgc4TUewJh8qUg8HM9quRDz5gcPNEDMEEX+XIwWBm+9XIBx9w4HkielJDEw4VZT4GSbOAvwVGAD+IiJsLtt8GdP3pcSjw6Yg4Otn2MbA52fZ6RMymDz3Nx2BmGeV/8DU2dl+udStX5s635P8sWlpyQVIhU4r2dz6GzMEgaQTwG+BCoB1oBa6IiK29tP+fwJkR8efJ8u8j4vCBvKaDwWwIVMEHnx1Yf4OhGGMlzQC2RcTLyQvfC8wBegwG4ApgaRFe18yKqacP/8ZGHy3UoGKcYzgReCNvuT1Z142kk4DJwIa81aMltUl6StIlRajHzMwyKMYRg3pY11v/1DxgfUR8nLduYkR0SPojYIOkzRHxUrcXkRYACwAmTpyYtWYzM+tFMY4Y2oEJecvjgY5e2s4Dfpy/IiI6kvuXgceAM3vaMSJWR0R9RNTX1dVlrdnMzHpRjGBoBaZImizpEHIf/s2FjSSdDIwBnsxbN0bSqOTxWOBz9H5uwszMhkHmrqSI2CvpGuAhcper3hURWyQtB9oioiskrgDujfRlUKcAd0j6hFxI3dzb1UxmZjY8ivI9huHmy1XNzAauv5er+pvPZmaW4mAwM8uqyoYsdzCYmWVVZUOWF+N7DGZmtS1/yPKFC3MDEFbwGFM+YjAzK4YqGrLcwWBmVgxVNGS5g8HMLKuBzNVQASeqHQxmZlkNZNKfCjhR7S+4mZkNt64wGOYT1f6Cm5lZuSrzE9UOBrNqUwF92DWvzE9UOxjMqk0F9GHXtIGcqC4RB4NZtcn/stWSJfs/hMqsu6JmDeREdYn45LNZtVqyJNeH3dSU+8vUap5PPpvVsjLvw7by5mAwqzYV0Idt5c3BYFZtKqAP28qbzzGYmdUIn2MwM7NBcTCYmVlKUYJB0ixJL0jaJun6HrZfJalT0qbkdnXetvmSXkxu84tRj5mZDV7mGdwkjQC+D1wItAOtkpojYmtB0/si4pqCfY8BlgL1QAAbk33fy1qXmZkNTjGOGGYA2yLi5Yj4A3AvMKef+34BeDgi3k3C4GFgVhFqMjOzQSpGMJwIvJG33J6sK/QlSc9KWi9pwgD3NTOzYVKMYFAP6wqvgf0ZMCkiTgceAdYMYN9cQ2mBpDZJbZ2dnYMu1qwiecRUG0bFCIZ2YELe8nigI79BRGyPiI+SxTuBs/u7b95zrI6I+oior6urK0LZZhXEI6bWrhL8UVCMYGgFpkiaLOkQYB7QnN9A0ri8xdnA88njh4CLJI2RNAa4KFlnZvk8YmrtKsEfBZmvSoqIvZKuIfeBPgK4KyK2SFoOtEVEM/CXkmYDe4F3gauSfd+VtIJcuAAsj4h3s9ZkVpXyZ/1qanIo1Ir8PwqGaSpQD4lhVilKNE+wlYkiDKPuITHMqolHTK1twzyMuoPBrBJ4xNTaVYI/CtyVZGZWzlauzJ1ozu82bGnJ/VGwePGAnqq/XUkOBjOzGuFzDGZmNigOBjMzS3EwmJlZioPBzMxSHAxm5cSD5VkZcDCYlRMPlmdlIPNYSWZWRCUYF8eskI8YzMpN/mB5Cxc6FGzYORjMys0wj4tjVsjBYFZOPFielQEHg1k58WB5VgY8VpKZWY3wWElmZjYoDgYzM0txMJiZWYqDwczMUooSDJJmSXpB0jZJ1/ew/a8kbZX0rKRHJZ2Ut+1jSZuSW3Mx6jEzs8HLPCSGpBHA94ELgXagVVJzRGzNa/avQH1E7JS0EFgJXJ5s2xUR07PWYWZmxVGMI4YZwLaIeDki/gDcC8zJbxARLRGxM1l8ChhfhNc1M7MhUIxgOBF4I2+5PVnXm68BP89bHi2pTdJTki4pQj1mZpZBMYJBPazr8Vtzkr4C1AO35q2emHzh4r8B35X0mV72XZAESFtnZ2fWms3Ki+dhsDJSjGBoBybkLY8HOgobSZoJ3AjMjoiPutZHREdy/zLwGHBmTy8SEasjoj4i6uvq6opQtlkZ8TwMVkaKEQytwBRJkyUdAswDUlcXSToTuINcKLydt36MpFHJ47HA54D8k9ZmtSF/HoYlS/YPpOcht60EMl+VFBF7JV0DPASMAO6KiC2SlgNtEdFMruvocOB+SQCvR8Rs4BTgDkmfkAupmwuuZjKrHfnzMDQ1ORSsZDyInlm56Oo+8sxtNkQ8iJ5ZJfE8DFZGHAxm5cDzMFgZcTCYDbeeLk1taOgeAo2NsHjx8NVllnAwmA03X5pqZS7zVUlmNkD5l6b6RLOVIR8xmJVC/qWpCxc6FKysOBjMSqGlJXek0NSUu/fVR1ZGHAxmw82XplqZczCYDTdfmmplzt98NjOrEf7ms5mZDYqDwczMUhwMZmaW4mAwM7MUB4OZmaU4GMzMLMXBYGZmKQ4GMzNLcTCYmVmKg8HMzFKKEgySZkl6QdI2Sdf3sH2UpPuS7b+WNClv2w3J+hckfaEY9ZiZ2eBlDgZJI4DvA38CnApcIenUgmZfA96LiP8E3Abckux7KjAPOA2YBfy/5PnMzKxEinHEMAPYFhEvR8QfgHuBOQVt5gBrksfrgQskKVl/b0R8FBGvANuS5zMzsxIpRjCcCLyRt9yerOuxTUTsBT4Aju3nvmZmNoyKEQzqYV3hWN69tenPvrknkBZIapPU1tnZOcASzcysv4oRDO3AhLzl8UBHb20kHQwcBbzbz30BiIjVEVEfEfV1dXVFKNvMzHpSjGBoBaZImizpEHInk5sL2jQD85PHlwEbIjdDUDMwL7lqaTIwBXi6CDWZmdkgHZz1CSJir6RrgIeAEcBdEbFF0nKgLSKagR8CfydpG7kjhXnJvlskrQO2AnuB/xERH2etyczMBs9Te5qZ1QhP7WlmZoPiYDAzsxQHg5mZpTgYzMwsxcFgZmYpDgYzM0txMJiZWYqDwczMUhwMZmaW4mAwM7MUB4OZmaU4GMzMLMXBYGZmKQ4GMzNLcTCYmVmKg8HMzFIcDGZmluJgMDOzFAeDmZmlOBjMzCwlUzBIOkbSw5JeTO7H9NBmuqQnJW2R9Kyky/O23SPpFUmbktv0LPWYmVl2WY8YrgcejYgpwKPJcqGdwJURcRowC/iupKPzti+KiOnJbVPGeszMLKOswTAHWJM8XgNcUtggIn4TES8mjzuAt4G6jK9rZmZDJGswHBcRbwIk958+UGNJM4BDgJfyVt+UdDHdJmlUxnrMzCyjg/tqIOkR4PgeNt04kBeSNA74O2B+RHySrL4BeItcWKwGrgOW97L/AmABwMSJEwfy0mZmNgB9BkNEzOxtm6TfSRoXEW8mH/xv99LuSOCfgL+JiKfynvvN5OFHku4Grj1AHavJhQf19fXRV91mZjY4WbuSmoH5yeP5wIOFDSQdAjwA/Cgi7i/YNi65F7nzE89lrMfMzDLKGgw3AxdKehG4MFlGUr2kHyRt5gLnAVf1cFnqWkmbgc3AWOBbGevp2cqV0NKSXtfSkltvZmYpfXYlHUhEbAcu6GF9G3B18vjvgb/vZf/zs7x+vzU0wNy5sG4dNDbmQqFr2czMUmrjm8+NjbkQmDsXlixJh4TVNh9NmnVTG8EAuRBYuBBWrMjdOxQM9h9NdoVD19FkQ0Np6zIrodoJhpYWWLUKmppy94V/JVpt8tGkWTe1EQz55xSWL9//QeBwMPDRpFmB2giG1tb0X4FdfyW2tpa2LisPPpo0S1FE5X1XrL6+Ptra2kpdhlWD/KPJwivWfORgVUbSxoio76tdbRwxmPXGR5Nm3fiIwcysRviIwczMBsXBYGZmKQ4GMzNLcTCYmVmKg8HMzFIcDGZmluJgMDOzFAeDmZmlOBjMzCzFwWBmZikOBjMzS3EwmJlZSqZgkHSMpIclvZjcj+ml3ceSNiW35rz1kyX9Otn/PkmHZKnHzMyyy3rEcD3waERMAR5NlnuyKyKmJ7fZeetvAW5L9n8P+FrGeszMLKOswTAHWJM8XgNc0t8dJQk4H1g/mP3NzGxoZA2G4yLiTYDk/tO9tBstqU3SU5K6PvyPBd6PiL3JcjtwYsZ6zMwso4P7aiDpEeD4HjbdOIDXmRgRHZL+CNggaTPwHz2063XWIEkLgAUAEydOHMBLF8HKldDQkJ7qsaUlN8vX4sXDW4uZ2RDr84ghImZGxNQebg8Cv5M0DiC5f7uX5+hI7l8GHgPOBN4BjpbUFU7jgY4D1LE6Iuojor6urm4Ab7EIGhpy8wB3TRLfNS9wQ8Pw1mFmNgyydiU1A/OTx/OBBwsbSBojaVTyeCzwOWBr5OYUbQEuO9D+ZaFrHuC5c2HJEk8Wb2ZVLWsw3AxcKOlF4MJkGUn1kn6QtDkFaJP0b+SC4OaI2Jpsuw74K0nbyJ1z+GHGeoZOYyMsXAgrVuTuHQpmVqWU+8O9stTX10dbW9vwvmhX99HChbBqlY8YzKziSNoYEfV9tfM3n/ujKxTWrYPly/d3K3Wdc7DKsHJl93+zlpbcejPbx8HQH62t6SOErnMOra2lrcsGxhcRmPWLu5KstrhL0GqYu5KGg7smKo8vIjDrk4MhC3dNVJ6WltyRQlNT7t7nicy6cTBk4e83VBZfRGDWLw6GrNw1UTl8EYFZv/jkc1Y+mWlmFcInn4eDuybMrAo5GLJw14SZVSF3JZmZ1Qh3JZmZ2aA4GMzMLMXBYGZmKQ4GMzNLcTCYmVmKg8HMzFIcDFaZPLKt2ZBxMFhl8si2ZkPm4FIXYDYo+SPbepwqs6LKdMQg6RhJD0t6Mbkf00ObRkmb8m67JV2SbLtH0it526ZnqcdqjEe2NRsSWbuSrgcejYgpwKPJckpEtETE9IiYDpwP7AR+kddkUdf2iNiUsR6rJZ50x2xIZA2GOcCa5PEa4JI+2l8G/DwidmZ8Xat1HtnWbMhkDYbjIuJNgOT+0320nwf8uGDdTZKelXSbpFEZ67Fa4ZFtzYZMn6OrSnoEOL6HTTcCayLi6Ly270VEt/MMybZxwLPACRGxJ2/dW8AhwGrgpYhY3sv+C4AFABMnTjz7tdde6+OtmZlZvv6OrtrnVUkRMfMAL/I7SeMi4s3kQ/7tAzzVXOCBrlBInvvN5OFHku4Grj1AHavJhQf19fWVN1a4mVmFyNqV1AzMTx7PBx48QNsrKOhGSsIESSJ3fuK5jPWYmVlGWYPhZuBCSS8CFybLSKqX9IOuRpImAROAfynYf62kzcBmYCzwrYz1mJlZRpm+4BYR24ELeljfBlydt/wqcGIP7c7P8vpmZlZ8HhLDzMxSKnLOZ0mdQF+XJY0F3hmGcoZTNb4n8PuqNNX4vqrxPUH393VSRNT1tVNFBkN/SGrrz2VZlaQa3xP4fVWaanxf1fieYPDvy11JZmaW4mAwM7OUag6G1aUuYAhU43sCv69KU43vqxrfEwzyfVXtOQYzMxucaj5iMDOzQajaYJC0Ihm1dZOkX0g6odQ1FYOkWyX9e/LeHpB0dN97lT9J/1XSFkmfSKroq0MkzZL0gqRtkrrNUVKpJN0l6W1JVTN0jaQJklokPZ/8//tmqWsqBkmjJT0t6d+S9/W/B7R/tXYlSToyIv4jefyXwKkR8Y0Sl5WZpIuADRGxV9ItABFxXYnLykzSKcAnwB3Atcm35yuOpBHAb8gNEdMOtAJXRMTWkhZWBJLOA34P/Cgippa6nmJIxmsbFxHPSDoC2AhcUun/Xsn4c4dFxO8ljQR+CXwzIp7qz/5Ve8TQFQqJw4CqSMCI+EVE7E0WnwLGl7KeYomI5yPihVLXUQQzgG0R8XJE/AG4l9yEVhUvIh4H3i11HcUUEW9GxDPJ4x3A8/QwfE+liZzfJ4sjk1u/PwOrNhgAJN0k6Q3gy8CSUtczBP4c+Hmpi7CUE4E38pbbqYIPmlqQDPZ5JvDr0lZSHJJGSNpEbjqEhyOi3++rooNB0iOSnuvhNgcgIm6MiAnAWuCa0lbbf329r6TNjcBecu+tIvTnfVUB9bCuKo5Wq5mkw4GfAP+roLehYkXExxExnVyvwgxJ/e7+yzS6aqkdaBKhAv8A/BOwdAjLKZq+3pek+cCfARdEBZ0kGsC/VyVrJzfEfJfxQEeJarF+SPrgfwKsjYh/LHU9xRYR70t6DJhFP+e8qegjhgORNCVvcTbw76WqpZgkzQKuA2ZHxM5S12PdtAJTJE2WdAi5ec6bS1yT9SI5SftD4PmI+D+lrqdYJNV1XbEo6VPATAbwGVjNVyX9BDiZ3JUurwHfiIjflraq7CRtA0YB25NVT1XJ1VaXAv8XqAPeBzZFxBdKW9XgSPpT4LvACOCuiLipxCUVhaQfA/+F3IidvwOWRsQPS1pURpL+M/AEucnCPklW/3VE/HPpqspO0unAGnL/Bw8C1kXE8n7vX63BYGZmg1O1XUlmZjY4DgYzM0txMJiZWYqDwczMUhwMZmaW4mAwM7MUB4OZmaU4GMzMLOX/A/xoEVK9w+cWAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot(X, Y, 'rx', label='data points')\n", + "_=legend()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Gaussian process regression with Gaussian likelihood\n", + "\n", + "Denote a set of input points $X \\in \\mathbb{R}^{N \\times Q}$. A Gaussian process is often formulated as a multi-variate normal distribution conditioned on the inputs:\n", + "$$\n", + "p(F|X) = \\mathcal{N}(F; 0, K),\n", + "$$\n", + "where $F \\in \\mathbb{R}^{N \\times 1}$ is the corresponding output points of the Gaussian process and $K$ is the covariance matrix computed on the set of inputs according to a chosen kernel function $k(\\cdot, \\cdot)$.\n", + "\n", + "For a regression problem, $F$ is often referred to as the noise-free output and we usually assume an additional probability distribution as the observation noise. In this case, we assume the noise distribution to be Gaussian:\n", + "$$\n", + "p(Y|F) = \\mathcal{N}(Y; F, \\sigma^2 \\mathcal{I}),\n", + "$$\n", + "where $Y \\in \\mathbb{R}^{N \\times 1}$ is the observed output and $\\sigma^2$ is the variance of the Gaussian distribution.\n", + "\n", + "The following code defines the above GP regression in MXFusion. First, we change the default data dtype to double precision to avoid any potential numerical issues." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from mxfusion.common import config\n", + "config.DEFAULT_DTYPE = 'float64'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the code below, the variable ```Y``` is defined following the probabilistic module ```GPRegression```. A probabilistic module in MXFusion is a pre-built probabilistic model with dedicated inference algorithms for computing log-pdf and drawing samples. In this case, ```GPRegression``` defines the above GP regression model with a Gaussian likelihood. It understands that the log-likelihood after marginalizing $F$ is closed-form and exploits this property when computing log-pdf.\n", + "\n", + "The model is defined by the input variable ```X``` with the shape ```(m.N, 1)```, where the value of ```m.N``` is discovered when data is given during inference. A positive noise variance variable ```m.noise_var``` is defined with the initial value to be 0.01. For GP, we define a RBF kernel with input dimensionality being one and initial value of variance and lengthscale to be one. We define the variable ```m.Y``` following the GP regression distribution with the above specified kernel, input variable and noise_variance." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from mxfusion import Model, Variable\n", + "from mxfusion.components.variables import PositiveTransformation\n", + "from mxfusion.components.distributions.gp.kernels import RBF\n", + "from mxfusion.modules.gp_modules import GPRegression\n", + "\n", + "m = Model()\n", + "m.N = Variable()\n", + "m.X = Variable(shape=(m.N, 1))\n", + "m.noise_var = Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=0.01)\n", + "m.kernel = RBF(input_dim=1, variance=1, lengthscale=1)\n", + "m.Y = GPRegression.define_variable(X=m.X, kernel=m.kernel, noise_var=m.noise_var, shape=(m.N, 1))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the above model, we have not defined any prior distributions for any hyper-parameters. To use the model for regrssion, we typically do a maximum likelihood estimate for all the hyper-parameters conditioned on the input and output variable. In MXFusion, this is done by first creating an inference algorithm, which is ```MAP``` in this case, by specifying the observed variables. Then, we create an inference body for gradient optimization inference methods, which is called ```GradBasedInference```. The inference method is triggered by calling the ```run``` method, in which all the observed data are given as keyword arguments and any necessary configruation parameters are specified." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 11 loss: -13.523289192527265\n", + "Iteration 21 loss: -16.077990179961076\n", + "Iteration 31 loss: -16.784414553096843\n", + "Iteration 41 loss: -16.820970924702017\n", + "Iteration 51 loss: -16.859865329532193\n", + "Iteration 61 loss: -16.895666914166453\n", + "Iteration 71 loss: -16.899409131167452\n", + "Iteration 81 loss: -16.901728290347176\n", + "Iteration 91 loss: -16.903122097339737\n", + "Iteration 100 loss: -16.903135093930537" + ] + } + ], + "source": [ + "import mxnet as mx\n", + "from mxfusion.inference import GradBasedInference, MAP\n", + "\n", + "infr = GradBasedInference(inference_algorithm=MAP(model=m, observed=[m.X, m.Y]))\n", + "infr.run(X=mx.nd.array(X, dtype='float64'), Y=mx.nd.array(Y, dtype='float64'), \n", + " max_iter=100, learning_rate=0.05, verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "All the inference outcomes are in the attribute ```params``` of the inference body. The inferred value of a parameter can be access by passing the reference of the queried parameter to the ```params``` attribute. For example, to get the value ```m.noise_var```, we can call ```inference.params[m.noise_var]```. The estimated parameters from the above experiment are as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The estimated variance of the RBF kernel is 0.616992.\n", + "The estimated length scale of the RBF kernel is 1.649073.\n", + "The estimated variance of the Gaussian likelihood is 0.002251.\n" + ] + } + ], + "source": [ + "print('The estimated variance of the RBF kernel is %f.' % infr.params[m.kernel.variance].asscalar())\n", + "print('The estimated length scale of the RBF kernel is %f.' % infr.params[m.kernel.lengthscale].asscalar())\n", + "print('The estimated variance of the Gaussian likelihood is %f.' % infr.params[m.noise_var].asscalar())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can compare the estimated values with the same model implemented in GPy. The estimated values from GPy are very close to the ones from MXFusion." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Name : GP regression\n", + "Objective : -16.903456670910902\n", + "Number of Parameters : 3\n", + "Number of Optimization Parameters : 3\n", + "Updates : True\n", + "Parameters:\n", + " \u001b[1mGP_regression. \u001b[0;0m | value | constraints | priors\n", + " \u001b[1mrbf.variance \u001b[0;0m | 0.6148038604494702 | +ve | \n", + " \u001b[1mrbf.lengthscale \u001b[0;0m | 1.6500299722611123 | +ve | \n", + " \u001b[1mGaussian_noise.variance\u001b[0;0m | 0.002270049772204339 | +ve | \n" + ] + } + ], + "source": [ + "import GPy\n", + "\n", + "m_gpy = GPy.models.GPRegression(X, Y, kernel=GPy.kern.RBF(1))\n", + "m_gpy.optimize()\n", + "print(m_gpy)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prediction\n", + "\n", + "The above section shows how to estimate the model hyper-parameters of a GP regression model. This is often referred to as training. After training, we are often interested in using the inferred model to predict on unseen inputs. The GP modules offers two types of predictions: predicting the mean and variance of the output variable or drawing samples from the predictive posterior distributions.\n", + "\n", + "### Mean and variance of the posterior distribution\n", + "\n", + "To estimate the mean and variance of the predictive posterior distribution, we use the inference algorithm ```ModulePredictionAlgorithm```, which takes the model, the observed variables and the target variables of prediction as input arguments. We use ```TransferInference``` as the inference body, which allows us to take the inference outcome from the previous inference. This is done by passing the inference parameters ```infr.params``` into the ```infr_params``` argument." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from mxfusion.inference import TransferInference, ModulePredictionAlgorithm\n", + "infr_pred = TransferInference(ModulePredictionAlgorithm(model=m, observed=[m.X], target_variables=[m.Y]), \n", + " infr_params=infr.params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To visualize the fitted model, we make predictions on 100 points evenly spanned from -5 to 5. We estimate the mean and variance of the noise-free output $F$." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "xt = np.linspace(-5,5,100)[:, None]\n", + "res = infr_pred.run(X=mx.nd.array(xt, dtype='float64'))[0]\n", + "f_mean, f_var = res[0].asnumpy()[0], res[1].asnumpy()[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The resulting figure is shown as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEKCAYAAADuEgmxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd8zff3B/DXO4MQBImdkrQ2QUmMGnWpWaVWzOKnaLVKq0bVN1IxSkrtmjVLK6hqzeJeNVokRkvsVUIRK2Jlnt8fJyRIuEnuvZ87zvPxuI/c8cm952bc83mv81ZEBCGEEMIYTloHIIQQwnZI0hBCCGE0SRpCCCGMJklDCCGE0SRpCCGEMJokDSGEEEaTpCGEEMJokjSEEEIYTZKGEEIIo7loHYCpeXl5kY+Pj9ZhCCGETTlw4MANIir0suPsLmn4+PggIiJC6zCEEMKmKKX+NeY4TbunlFILlVLXlVJHM3i8oVIqRil1OOUyytIxCiGESKV1S2MxgJkAlr7gmF1E1Moy4QghhHgRTVsaRLQTwC0tYxBCCGE8rVsaxqijlPobwBUAQ4go8tkDlFL9APQDgJIlS1o4PCFEdiUkJCAqKgqPHj3SOhS75+bmBm9vb7i6umbp+609aRwEUIqI7imlWgL4BUCZZw8ionkA5gGAv7+/bBAihI2JiopC3rx54ePjA6WU1uHYLSLCzZs3ERUVBV9f3yw9h1Wv0yCiu0R0L+X6RgCuSikvjcMSQpjYo0eP4OnpKQnDzJRS8PT0zFaLzqqThlKqqEr5K1JK1QTHe1PbqIQQ5iAJwzKy+3PWtHtKKfUjgIYAvJRSUQCCAbgCABHNAdABQH+lVCKAhwA6kxn3p01OBm7cAAoXNtcrCCGEbdM0aRBRl5c8PhM8JdcivvwSWLEC2LoVKFfOUq8qhBC2w6q7pyytSxcgLg6oXx84fFjraIQQwvpI0kijalVg1y7AzQ1o2BDYu1friIQQlnLhwgWUL18effr0QeXKldGtWzds27YNdevWRZkyZbB//37cv38fvXv3RkBAAF5//XWsW7fuyffWr18f1atXR/Xq1fHnn38CAHbs2IGGDRuiQ4cOKF++PLp16wYz9rBbhLVPubW4smWB3buBxo2BNm2Ac+cAd3etoxLCcXz6qelb+tWqAVOnvvy4M2fOYNWqVZg3bx4CAgKwYsUK7N69G7/++ivGjx+PihUrolGjRli4cCHu3LmDmjVr4q233kLhwoWxdetWuLm54fTp0+jSpcuTGniHDh1CZGQkihcvjrp162LPnj2oV6+ead+gBUnSSEfJksC2bcDp05IwhHAkvr6+8PPzAwBUqlQJjRs3hlIKfn5+uHDhAqKiovDrr79i0qRJAHiq8MWLF1G8eHEMGDAAhw8fhrOzM06dOvXkOWvWrAlvb28AQLVq1XDhwgVJGvaoVCm+AMCqVUCNGsCrr2obkxCOwJgWgbnkzJnzyXUnJ6cnt52cnJCYmAhnZ2esWbMG5Z6ZKfPVV1+hSJEi+Pvvv5GcnAw3N7d0n9PZ2RmJiYlmfhfmJWMaL3H3LvDRR0CLFsAtqZIlhENr1qwZZsyY8WRc4tChQwCAmJgYFCtWDE5OTli2bBmSkpK0DNOsJGm8RL58wNq1wIULQLt2QHy81hEJIbQSFBSEhIQEVKlSBZUrV0ZQUBAA4KOPPsKSJUtQu3ZtnDp1Cu523K+tbH0k/1n+/v5kjk2YVqwAunUDevQAFi8GZPGqEKZz/PhxVKhQQeswHEZ6P2+l1AEi8n/Z90pLw0hduwKjRwNLlwKbN2sdjRBCaEMGwjMhKIgX/ul0WkcihBDakJZGJiiVmjD++Qf416gddYUQwn5I0siCR4+A5s2B9u35uhBCOApJGlng5gbMng0cOMCrV4UQwlFI0siiNm2AoUOBuXN5Sq4QQjgCSRrZMHYsrxTv0we4fFnraIQQWXXp0iXodDpUqFABlSpVwrRp00z6/IcPH8bGjRszfNzHxwc3btww6WuaiySNbMiRg9dv9OsHeMkmtELYLBcXF0yePBnHjx/H3r17MWvWLBw7dsxkz/+ypGFLJGlkU9mywNdfAzlzAnZcOUAIu1asWDFUr14dAJA3b15UqFABl9PpPmjTpg2WLl0KAJg7dy66dev23DGrVq1C5cqVUbVqVTRo0ADx8fEYNWoUVq5ciWrVqmHlypW4efMmmjZtitdffx0ffPCBTZVLl3UaJnLoENC5M/Dzz0ClSlpHI4Rta9jw+fsCA7kO3IMHQMuWzz/eqxdfbtwAOnR4+rEdO4x/7QsXLuDQoUOoVavWc4/NmzcPdevWha+vLyZPnoy96Wy6ExISgi1btqBEiRK4c+cOcuTIgZCQEERERGDmTN6IdODAgahXrx5GjRqFDRs2YN68ecYHqDFpaZiItzcXNPy//wNsvIilEA7r3r17aN++PaZOnYp8+fI993iRIkUQEhICnU6HyZMno2DBgs8dU7duXfTq1Qvz58/PsHDhzp070b17dwDA22+/jQIFCpj2jZiRtDRMpFAhYNYsoFMnYPJkYPhwrSMSwna9qGWQO/eLH/fyylzL4rGEhAS0b98e3bp1Q7t27TI87siRI/D09MSVK1fSfXzOnDnYt28fNmzYgGrVquFwBjtKKRstYCctDRPq2JEX/AUHAydOaB2NEMJYRIT3338fFSpUwODBgzM8bv/+/di0aRMOHTqESZMm4fz5888dc/bsWdSqVQshISHw8vLCpUuXkDdvXsTGxj45pkGDBli+fDkAYNOmTbh9+7bp35SZSNIwIaW4tZEnD/Ddd1pHI4Qw1p49e7Bs2TLo9XpUq1YN1apVe262U1xcHPr27YuFCxeiePHimDx5Mnr37v3cIPbQoUPh5+eHypUro0GDBqhatSp0Oh2OHTv2ZCA8ODgYO3fuRPXq1fH777+jZMmS2Yo/KYnLGiUkZOtpjCKl0c3g7FnA1xdwkpQshFGkNHrWEQHnzgG3bwNlygAeHi//HimNbmVee40TxtWrPJNDCCHM5coVThje3sYljOySpGEm9+8DVaoAL+geFUKIbMufHyhWDChSxDKvJ0nDTNzdgQ8+AJYtAwwGraMRQtibx1P73d2BEiUst5uoJA0z+vJL4NVXgf79ZW9xIYTpxMcDkZHcBW5pkjTMKFcuYPp04ORJIGUhqBBCZEtyMnDmDM+YssQYxrM0XdynlFoIoBWA60RUOZ3HFYBpAFoCeACgFxEdtGyU2fP221zy4MwZrSMRInMiI4H9+wFPT6BoUe4CKVFC66gcGxFw4QKXUildmk9MLU3rlsZiAM1f8HgLAGVSLv0AzLZATCa3dq2s2xDWLT4eWLMGeOcd4Pp1vu/334HevXnvmFq1eHZO+fLAf/8BCA19frDOYOD77cBXX32FSZMmvfCYX375xaSVcNNz5coVdEhTSOv6dS5XVLw4D4A/Nn78eLPGkZamSYOIdgK49YJD2gBYSmwvgPxKqWKWic50cuTgr0eOAGb+GxMiUxITgW+/5RZEhw7AwYPAqVP8WK9ePP8/IgJYvx6YOhUIyhmKoscNQEAAHrYOxN6vDSC9gWd9BAYCAQHmD9pKEpYlkkbx4sWxevXqJ7ddXICCBXm2VFqWTBogIk0vAHwAHM3gsfUA6qW5vR2AfzrH9QMQASCiZMmSZI3i4oiKFiWqX58oOVnraIQgevCAyN+fCCBq2pRo40aixMSXfJNeT+TlRUnb9NTbV0+3kY8eOOWmBPd8/FgWHTt2zPiDU2J48nrP3s6isWPHUtmyZalx48bUuXNn+uabb4iIaN68eeTv709VqlShdu3a0f3792nPnj1UoEAB8vHxoapVq9KZM2fSPe5ZwcHB1L17d9LpdFS6dGmaN28eERElJyfTkCFDqFKlSlS5cmX66aefiIjo/PnzVKlSJUpOJlq0aBG1bduWmjVrRqVLl6ahQ4cSEdHw4cPJycmJqlatSl27dqV79+5Ry5YtqUqVKlSpUqUnz5VWej9vABFkzGe2MQeZ8/KSpLEhnaRR40XPV6NGjef/GqzEnDn8E1+7VutIhGDrG0wkfZCekpP5ZGb3bqK5nfW0uOJEKleOyN2dqFgxosqViRo3JgoJIToxR0/JXl6U+GUQxefITQRQCILoiy+IHj3KWhyZShpEqYkiKMgkCSMiIoIqV65M9+/fp5iYGHrttdeeJI0bN248OW7kyJE0ffp0IiLq2bMnrVq16sljGR2XVnBwMFWpUoUePHhA0dHR5O3tTZcvX6bVq1fTW2+9RYmJiXT16lV65ZVX6MqVK0+SxokTRDNmLCJfX1+6c+cOPXz4kEqWLEkXL14kIiJ3d/cnr7F69Wrq06fPk9t37tx5Lo7sJA2txzReJgrAK2luewNIv7SkDXj/fe4T/uILKZ8utJGUBAwbxt1QAPD2VwF487tArPrIgIoVgf/VM6DtT4HYcT8AlSoBffvyRI7SpXnVcXAwUP5DHaY87A/n8WPg4kyIGxaET91mIzzUgMhIC70RnY7nso8Zw191umw93a5du9C2bVvkzp0b+fLlQ+vWrZ88dvToUdSvXx9+fn5Yvnw5IjN4k8Ye16ZNG+TKlQteXl7Q6XTYv38/du/ejS5dusDZ2RlFihTBm2++ifDwcABcTyo2ltdhNG7cGB4eHnBzc0PFihXx77//Pvf8fn5+2LZtG4YPH45du3bBw8RTrKw9afwKoIditQHEENF/WgeVVS4uwIQJPAX3+++1jkY4mkePgC5dgG++4TEKAFh7R4dOCINuTiCGxo7CxjyByLMhDIsu6LBmDTBlCrBgAU/mOHCA1wVs+cKAD+Km4z5y4e7DHFgWpYMKC8Nmj0BUj+GxhrNnzfxmDAZg9mwgKIi/mmAFbUalynv16oWZM2fiyJEjCA4OxqNHj7J13LOvo5TKcOe+27f5BLNIEV7ElzNnziePOTs7IzGds8+yZcviwIED8PPzw4gRIxASEpLuc2eVpklDKfUjgL8AlFNKRSml3ldKfaiU+jDlkI0AzgE4A2A+gI80CtVkWrcGmjThMwchLCUxkQe6V60CJk3iFkTbtkC7dsBpbx3ievdH78tjkOuz/nhUR4c9e4B583iB6sWL/Bw3bgCJWw1ouiAQ7r074b/5GzCt4Vq0WRGI3r2BiKFhQHg4tmwBypXj5GSWeqgGAw+6h4UBISH8NTAwW4mjQYMGWLt2LR4+fIjY2Fj89ttvTx6LjY1FsWLFkJCQ8KScOYDnyp1ndNyz1q1bh0ePHuHmzZvYsWMHAgIC0KBBA6xcuRJJSUmIjo7Gzp07UblyTVy+zHXsvL1fHL+rqysSUkrcXrlyBblz50b37t0xZMgQHDxo2lUKmq7TIKIuL3mcAHxsoXAsQilgyxbLLfkXgggYOBDYsIGnfjdowFNoo6OBiROBwa8b4NJ1Nq72DYLbhNloN0aHHeDuHmdn3lisZEnghx+AK5+F43KxMORVOvR9HRjVBzgxOwy1RoejzpfD0KuXDuPf42Q0bBjPvpo1y8QVn8PDOVE87pLS6fh2eHiWu6mqV6+OTp06oVq1aihVqhTq16//5LExY8agVq1aKFWqFPz8/J4kis6dO6Nv376YPn06Vq9eneFxz6pZsybefvttXLx4EUFBQShevDjatm2Lv/76C1WrVoVSCqGhociduyicnC4gR46Xf17069cPVapUQfXq1dGjRw8MHToUTk5OcHV1xezZJl6pYMzAhy1drHkgPK3kZJ6tcu2a1pEIexcXR/TOO0TDhhEZDEQeHjy4fegQUeJWPSV58kDymTNE3Yrr6V5uL/prvJ7OnydKSkp9nvPniWbMIGrdmihPntRZV/HxPAD+5ZdEzs5E5coRnThBNHw4H9Or18tnZWV6INxGBQcHPxlgN0ZCgnnisOeBcLv1779Aq1Y8xiGEOeXIwWMS/v5As2a8JmPvXl7QN6NHOIaWDAM11OG114BlUTq4rw9Dbedw+Pg83ULw8QEGDADWrQOioriV4usLuLoCOXMCI0cCej1w8yZQuzbQqBH3Hi1eDDyzn5F4geho4N49vu5ijRtyG5NZbOliKy0NIqL/+z+inDmJLl3SOhJhj86dI2rShL8aDEQ5chC98QbRzZtEwcFEShEVL060Zk321w4dPswzX+fNIzp7lsjPj8jJiWjuXKI9e17+/Y7S0niZu3eJwsOJzpwx7+tIS8NGjRrFxcfGjdM6EmFvEhOB7t2Bfft4tt677/KubmvWAIMGAaNHA++9xxUK2rXL/hhbvnyAnx/Qrx8wZAiP2zVvzgvF9+/nY8LDeZyDMhgcp4wecBAJCTwGlDMnUKqU+V4nuz9na2z8OAwfH57FMm8eMHQol1EXwhTGjQP+/JOrLPfpw9M1N23irqr9+4GxY3lmlKkmZPj6Atu2camRoUOB06d5plbu3MBnn/GmZAkJPKPKxQV4tuqFm5sbbt68CU9PzwynvtozStmyNSmJk7u5uqWICDdv3oSbm1uWn0P2CNfYlSs8m2XuXKBxY62jEfbgzz+B+vV51tPJk1xhecsWLgvl7MwVUnPnNt/rb90K7G4TilcDA9BtgQ7/938882pRDwPynghHh/3D8O23nEweS0hIQFRUVIZrG+xdbCwXIvT0BPLkMe9rubm5wdvbG66urk/db+we4dLS0Fjx4lwgzqRTEoVjCg0FAgIQMlGHkiWBAgWAfAcN2NU9HIP/NwyFCwPLl5s3YQC8DqnKwgAU/iQQalcY5s/XoXK0AW8vDcR/tdthVH0DPv9cB19f7jaDwQDX8HD4Dhtm3sCsWFISTxawRL3H7JKkYQWcnHgmy5492a6GIBxZQAAQGIh1y8Kw9JIOK/oZ8KtbIKZEhWH7DmDJEsutDyrSWQcUCUNSh0DMS+6PT5NnY3z9MOzcBWzME4jocmH48Ucd3vVIs1DPAZ09C7i58Yy2d97ROhojGTNabksXW5o9ldZXX/Fsk1OntI5E2KqrV4kebdJTYkEv+iZXEN109qIZ7fQEEIWGahPTgyFBRACNUUG0bBkXPWzspKeHeb0oaaRpig3aqthYoooViSpVeno9jFZgK1VuTX2x1aRx9SqRmxtRz55aRyJsUXIyUbNmRFWrEi3z5Q/qfc2CCCD6+GONyvGnVKJ9NCyIbrt4UWMnPa1aRRQQQDTehWO8+2kQ9exJdO+eBvFpKDmZKDCQTxS3btU6GiZJwwZ9+imvqDX3HG1hf1at4v/m8U30dB1etK95EMV5eNHIN/QUF6dBQM/scXHvNz3dcvGipq562vu1nm44edG3eYLoYV4vaqT01LGjY+0zM2kS/74mTNA6klSSNGzQlSvc2ujdW+tIhC15+JDolVeIevvqKVp5Uf/yei7bYaLNibJk4sTnXvf2z3raUa4fJXl60bmFesqXj+j9V7lsSUPoadw4y4ephT/+4BZG+/bWlSiNTRoyZ8eKFCvGi6P++YfntAthjJkzgUuXgHo5w9FZheF+TR2CgwFqmKaQn6UNG/bcrI78bXV4s/drcFoVhqKddZg7F1j8rw5j/cLQr1o4/vc/IE1xWbtVrRqXY1m0yDYLl8o6DSvz8CHPprDFPyahjVatuN7T3r1A167Ajz8CH37IFW2tVYsWvEapZ0/g8885x2zfzutI9u61z7//e/f4/eXKpXUk6TN2nYYkDSt15w6vEi1QQOtIhLVLSACqVuUPpTx5+GtkJJA3r9aRZWzrVk4cTZtyC3vhQl7g2rkzlySxN8nJXK7l6lVg927rLERobNKQ7ikrFBPDZRkmTtQ6EmHNrl3jy9KlwPHjwBtv8NfvvrPuhAHwAsDvvgMqbwqF300DatcGBg/m6s9xmw3Y2SoUSUlaR2k6I0dydeCuXa0zYWSKMQMftnSx5YHwtDp1Isqbl+j2ba0jEdaqd2+iggWJihThaax58vA0TlsyqwPP9lr1kZ6KFSPqWkxPD/LwwPiIEVpHZxrz5/OUow8+sK6B72dBZk/ZtkOH+LczdqzWkQhrdOYMT8+uU4f/TvbsITpyhNf72JKEBKLJrfSUUMCLLvYKouvwoi/r6KlvX35fP/ygdYTZs3Ur/56aNzffhkqmYmzSkO4pK1WtGvf5Tp3KBeaESOvrr3nzo7//5vITb7wBVK4MFCmidWSZ4+ICDP5NB5cB/fHK4jE481Z/jP+La2e9+Sbw/vvAX39pHWXW+fryXuwrV9pBt1QKSRpWbMQI4MYNYPNmrSMR1uTCBa4jVbo0EBfHiWP4cK2jygaDAZg9G+tfD0KFP2bjqzcNCA4GZpQMRQdPA3r3Rur4hsHAhRmtXHQ0D36/9hqXiLenwX1JGlasXj0e2GzXTutIhDX5/Xcucnn8OG+revEib61qkwypBQtvDAxB24QwDNkfiPe8DRj1awAW3g/E7yMMcHZOc6yVl4K9ehWoU4c3u7JHkjSsmFJA+fJ8PTFR21iElQgNRb8yBnTsyInj6FFgqL8Bzf62/rPvdIWH8wJEnQ69egEle+jwzsMwDKgVjq2JOgwtGQbvzwNBQaPwqHUg4n8Is+pS0Hfu8I6F//0HdOumdTRmYszAhy1d7GUgPK2RI3nA05pnXgjLiPmFq9g2dtJTlSpEjRQPIttLpdjYWKLy5YmKFiWaOZMHw7e9wcUNRyOI2rcnLpFihW7d4llsrq5EmzdrHU3mQQbC7Ye3Nw8G/vGH1pEILUVHA8W76TC4RBh+TA5E91Oj8EvOQLisse6z78zIk4cbHs7OvGBxUisDqvw5G6c7B2GI+2zcXGPAgAEZ7zOuFSKgTRseX1qzBmjWTOuIzMiYzGJLF3tsaTx4QFSoEFHLllpHIrQ0ejSfebu4EK2vwWff9wYHaR2WWTx6RER6PSV7edH7r+mpYEGiaz/pKTYXr+H46CPr2IMirW3biDZs0DqKrIO0NOxHrlxc4GzjRi4PIRzPw4dcmLBUKeDNZAOan58NBAXBfelsHiC2MzlzAsn7w7G6Yxg6zNQhIQF4d5oOOX8Jw9A3wzF3LnDokNZR8v7rS5bw9caNgZYttY3HIozJLLZ0sceWBhFRdDRRrlxE77+vdSRCC/PmcSujsZOebjpbSflzM7t2jcjTk8jfn2j5cn7/Q4bw2N6RI6nHaTXW9+ef/KMvVIjozh1tYjAl2EJLQynVXCl1Uil1Rin1RTqP91JKRSulDqdc+mgRpzXw8gKWLQOCgrSORGhh1iygaFGgenI4AikMqpGOp6HqNCx/bmaFC3MRw4gI4ORJ4KOPgEmTgPXreSEjAPzyC9CwIXD9umVjW76cf/T58wN79gAeHpZ9fU0Zk1nMcQHgDOAsgFcB5ADwN4CKzxzTC8DMzDyvvbY0hGM7c4Zrkfn4EOXIQXTpktYRWc5773Epjr/+Inr9daICBYguXODHwsK4Bf7KK0QHDlgmng8+4FZP/frcA2AvYAMtjZoAzhDROSKKB/ATgDYaxmMTIiKATp2AR4+0jkRY0oYNQGwsEBUF9OnDM+ocxdSp3NIeMIAbVUlJvMYvPh7o2JFLjQNA3brAt9/C7NVxy5QB/vc/QK/nuByNlkmjBIBLaW5Hpdz3rPZKqX+UUquVUq9YJjTrFRPD/zgrVmgdiTCb0NAng9uHDnFlgN1jDBjnEQoiYOhQjeOzsIIFeZe7KVO4dMqiRcD+/bxxEwBUr84nU40b84ZOv/9u2te/cYM3tQoL49uffw6MGWM/taQyS8ukkd7eXM/Ovv4NgA8RVQGwDcCSdJ9IqX5KqQilVER0dLSJw7QujRoBVarwGRVZ2Vx1YSIBAXwqbTBg5kzAfb8Bs24E4o1BAdi+HfDx0TpAy2vRAqhfn6+3aQP8Wi8Uf08zYM0avq9wYeC3wQac6ReK5s35vpUreSwkq27d4vxdpgywYAHPlBLQdEyjDoAtaW6PADDiBcc7A4h52fM6wpjGokXcp/r771pHIsxGr6ckTy8a5xxE0cqLupfQW+1KaEsKCiJq2pQobrOebrl40du59XTqFD03i+zhQ555pRRR+/ZEa9fyanNjhYYSubnx/1mzZkSRkeZ5P9YENjCmEQ6gjFLKVymVA0BnAL+mPUApVSzNzdYAjlswPqvVpQuXwJ42TetIhNnodNhbrT++TBqD76g/drvq8MsvWgelveLFuftpzS0d4peFYfHDQOjrjQKlFD18vDLezQ04dgz48kvee7xtW8DTE1i8mJ/n2jXgp5+AH37g/6OgIG7NXLjAj/v4AD168ArvzZuBihW1eLdWypjMYq4LgJYAToFnUY1MuS8EQOuU618DiATPrDIAKP+y53SElgYR0YwZRBMmSD0qe5W4VU83nLxoUm7emKgh9DZZz8jUEhN53Ubx4kR37xKd7sIr49dWCcrwfyEujhsgn39OdPgw37d+PbciHl+UIqpYkWdoOSrIzn1C2KiU8hlL/09PTk5EbfLxgr7k7fa3gC8r9u7lT67ZgdwltaMBJ9ZfBxv/83nwgOjoUaJTp4hu3LDeIoiWZGzSkDIiNiw+HvjxR+DuXa0jESYVHg4VFoYzr+iQnAysu6tD+NAwqAj7W8CXFbVqAaEtDOgQFojY78NQTx+Cb/zDUPvbQJyYbVxJlVy5gEqVeJDb05MLJArjKE4w9sPf358iIiK0DsMiwsOBmjWB6dOBTz7ROhphKmfOcD/8qFGp9128yPWYBLsfHIo7ZQJQojuPYdy8CQyoZEDlR+Hoe2oYChfWOEAbpJQ6QET+Lz1OkoZtq1OH/2FOnOBNeYSNCg3lqbY6HQYP5hOB+kkGjGwSjqs9hqF7d60DtF43b3Jr4dAh3iu9Zk1g2zbeQ10Yz9ikIR8zNm7QIOD0aWDTJq0jEdmSsjYjfosBS5YAb+c2YI1zIBoND5CE8QKffsrdVXFxwOuvA/PnAzt3AkOGaB2Z/ZKkYePat+dpiDNnah2JyJaUwoPJHQMx6NYoLIgNxITXw3C1gn1srmQuLVsCZ89yQUcA6N6dE8n06bxyXJieJA0b5+oKfPABcOkS8OCB1tGIbNHp8KNHf4zCGMxV/fFNhA7nzmkdlHVr2pT35A4J4XIfAPf0NW7M/xeP61IJ05GkYQeGDweOHAFy59Y6EpEdDzca8O7V2RjnFIQPMRt9XjOm0DB2AAAgAElEQVSgbl2to7J+kyZxMceQEL7t6gqsWsUL9Nq2TV2wJ0xDkoYdyJkTUAq4d493eBM2yGBArp6B2P1JGP6XHIKOFIbp1wKhdtjfrnymVqkS0K8fsHo1cP8+31egAPDbb0BiIvDOO5xUhGlI0rATFy9yuewl6ZZ0FNYu4c9w3F0QhuAdOuTPDxzy0EHZ6eZK5jB+PM8gdHdPva9cOa4scvw4139MSNAuPnsiU26NRAT89x/Pob9yBYiO5suDB0ByMl9y5OCpf56eQIkSXK/mlVcsMxWWCPD351kkR45wy0PYjqVL+Ww5Lg6oXZv3hpg0SeuobE9SEk/BTbtOY/58/tn26QPMmyf/Gxkxdsqtg1aEf7HERC5UFh4OHDzIlxMnUpu+jynFK0udnTkxPHzIq7TTcnfnUuaNGwNvvcUfCOZYpKUUb1LTuzfwxx+8BaawHQsW8N+FszOwZQuQN6/WEdmmt97i/4Xt21OTQ9++wL//AuPGAaVK8QZKIuukpZHi9m3ej/iPP3jP38d9oAUK8CYvlSvzBjBlynDroVAh3hwmbfkBIm553LzJ3UXHjvFl3z5OQElJ/GHQsSPQqxdvrmPKs56HD7mLSqfj/l1hG06eBMqX50193n2XB3FF1syYAQwcyJVpmzVLvZ8I6NkTWLYMWLgQ+L//0y5Ga2VsS0PzAoOmvmS1YOHt20ROTlzp8sMPiVasIDp3znRVZO/cIVq3jqhXLyJ3dy64Vro00dy5RI8emeY1iIiGDuX9lK9dM91zCvMaNoz/9h5XXF2/XuuIbFdcHJGvL1HVqkRJSc8/1qQJ/6zXrNEmPmsGqXKbebduZflbM+XePaKlS4kCAvg34O3Npc7j47P/3FFRRBER2X8eYRkJCUQheSZSK3c9eXgQ5c/PFVhJryeaOFHr8GzSDz/w/9WKFc8/du8eUZ06RDlyyCZmzzI2acjsqTQKFLDM67i7A++9x91WW7bwfPJPPgGqVQN27Mjec5coAdSoYYoohSW4uAANhwZg4f1A+Mca0KMHkGuvgaf7BARoHZ5N6tIF8PPjCtDPcncHNmzg7sB33+WuaJE5kjQ0pBSvaN21i+eUP3jA4xHdu/O4SFbducN9trLTm21YdUOHLk5h+DE5ECPjR3HCSLMLncgcJydODGvXpv94gQK8+5+3N68ml8SROZI0rESrVkBkJM/sCAvjVsfOnVl7rrx5ucUi28Fat6gooEMHXltz0EOH30r0R+E5Y4D+/SVhZNMrr/AkldjY52c0ArxdssHAdduaNeMTN2EcSRpWJHduYMwY4K+/eI9jnQ4IDuZZV5nh7Ax8+CEnjshIs4Qqsis0FDuCDVizhjfR+m2wAT1jpvHc7Nmz+RNNZEtUFPDqqzydOT3Fi/P/iLc37w+e3a5hh2HMwIctXexlu9e7d4l69OABvbffJoqNzdz3R0cT5cxJ9NFH5olPZE/ydj3ddPKiVu566lJUT8n58hF5ePAAuJ63MSW9bO+aHcnJRPXrExUrljK5IAP//UdUoQL/v/z8s+XiszaQgXDbljcvd1t89x3vlVG/Pp85GcvLi7vGly3jmlTCuux21aF9chgW3g/Ex3fGISlJcSe8TvekTLqUEMkepYCxY7mSw+zZGR9XtCh3T1Wrxt2F8+dbLkabZExmsaWLvbQ00tq0iShvXqLixYn++cf479u3j+jTT7nVIaxLr1487XM0gogAutgrSOuQ7FbjxkSFC/N02xe5d4+oRQtu3QcHm26Nlq2AtDTsx+MZHkrxSejhw8Z9X82awJQp3OoQ1qV0aaCJiwEfqdmY5RkE7/UyjmEuo0cD168Dv/764uPc3YF167haw+jRQKdOskdNeiRp2Ag/Py5xkjs30KgRcOCAcd9HxHV4ZEDcurRwM2DRg0B0pDDcHxbCFW0DAyVxmEHdurx/eJcuLz/W1ZXLjHzzDZfiqVePNzgTqSRp2JDXXuPE4eHBk2yMKbH14AHQrh0wYYL54xPG2b0bOLEsHN1dw7DLWYf33oOMY5hZtWr8NS7u5ccqxXuM//YbV7WuXp3HFQV7YdJQSpW0VCDCOL6+nDgKFOBpgqdPv/h4d3egRw/+PIqOtkyMImOnT/Okhp6Rw5D8pg5DhwLFiqU8qNMBw4ZpGp89mzuXuwWNnRjy9tvA/v08NbdlS/7VyJ4cL29pPFlTrJRaY+ZYhJFKluTyIwAvTLp69cXHf/ghL3BatMj8sYkXW7yYz2QTE7lU99dfax2R46halWcgfved8d9Tvjywdy+vt/zmG+CNN4CjR80Xoy14WdJIW7j7VXMGIjKnbFlg40Ye4GvRgheIZaRSJaBBA2DOHN4sSmgjKYmnUXt4cOJ/Vf6jLKp2bT7JmjTp+b1xXiRXLk40q1fzvhzVq/MiXEdtdbwsaVAG14UVCAgAfv6Zz3wCA1+8cvyjj3i/jXPnLBefeNr27cDly1wb7MoVICRE64gcT3Awd9POnZv5723fnieUdOgAjBrFyWP7dtPHaO1eljSqKqXuKqViAVRJuX5XKRWrlHrBua1xlFLNlVInlVJnlFJfpPN4TqXUypTH9ymlfLL7mvamaVM+C9qyBfjiuZ9gqvbteWOo0qUtF5t42rp1XB4G4O6pXr00Dcch1anDu/tNnsy/g8wqVAhYsYJ/l/fu8XO9+y4PmDsKzXbuU0o5AzgFoAmAKADhALoQ0bE0x3wEoAoRfaiU6gygLRF1etHzmmuPcGs3YAAwaxavAO/ePePjEhN5Bom7u+ViEywhgescxcdzQb2//5b9qrUQGQnkyMG7cGbHo0e8Dmr8eL7eowfw5Zc8y9EWGbtzn5ZTbmsCOENE54goHsBPANo8c0wbAEtSrq8G0Fgp+TdLz5QpvC94nz4Zz9p8+JBbGjL4qg29nsegHpeul79kbVSqlJowsnPO7OYGjBgBnDrF3b8rVgDlyvFeOfZ83qpl0igBIO2ymaiU+9I9hogSAcQA8LRIdDbG1ZX3li5alPcgv337+WNy5eJFggsWpF8uWphP69b8AZMrF/+uunbVOiLHdv8+0KZN5mZSZaRYMd6G4Nw53p987Voeb6xViyc+2FvtNy2TRnrnWc/mfWOOgVKqn1IqQikVEe3AixG8vHg9xpUrfCab3llU//7AtWuyQZMlHTvGC8X++Qfo3ZtXGBcponVUji13buDWLV70asyCP2MUKwZ8+y1Pdpg+nWc09uoFFC7ME1V+/tk+EoiWSSMKwCtpbnsDuJLRMUopFwAeAG49+0RENI+I/InIv1ChQmYK1zbUrAlMnMgDdTNmPP94s2a8QNAUZ1jCOIsX825ySUn8ISIJQ3tK8YZnUVHA0qWmfW4PD96++dgxXv3fuzfv1dG+PVCwIFdzCA3lfXNMlbAsScuBcBfwQHhjAJfBA+FdiSgyzTEfA/BLMxDejogCX/S8jjoQnhYRN703bwb+/BPwf2Zoa+JEnml17BhQoYI2MTqKxEQe9I6L4zUy7dpxbSOhPSLuQrpxAzh5krsNzSUxkXfi3LyZL0eO8P05cvDU3SpVeKylUiUeSPf25v3jLcnYgXDNkgYAKKVaApgKwBnAQiIap5QKAZfo/VUp5QZgGYDXwS2MzkT0wpUGkjTYrVtcbydnTq6Km3a21M2bPFjetCmfAQvzWb8eeOcdvu7iAvTrx7PchHX47Tceb1q8GOjZ03Kve/UqtzT++gvYt4+TSNpxSCcnoEQJHqMsVIi7nvPl4/9jd3dONs7O/DdFxK3YpCTuIuvRI2sx2UTSMAdJGql27OCKuP37yweVVv75h7ukDh3i2/v2cReisA5E3I3brRvgqeEUGyJOJMeOAefP88rzf//l2XY3bvCCxNhYHsB/0SSWmjX5bywrJGkIAMDnn/Pg3ObNPJ7xWHw8r2qtVImnCArzSEoCfHz4n71IEf5QkKm2IjsSEviSlMTdXkpxq+NxyyNHjqw9ry2s0xAWMG4cULEiD8bdSjOFwNUV2LqVB+Ts7LzBauzcyVuHRkVx10PPnpIwrJVezydPtlCbzdWVZ3/lzcvVrvPn5+u5c2c9YWSGJA075+YG/PADN3MHDEi9XylekHT0KM/wEKY3aBCvEPbw4Jk6L1qpL7R15Qr/n/z2m9aRWD9JGg7g9deBoCDgxx+f/qfo0oU/0GS8w/QOH+ZLbCz3l48ZwzNihHXq3JmrDo8dKy3vl5Gk4SC++IJXg3/4IRATw/flzs2LANes4TMtYTqLFnH/cmIiUKqU45bRthUuLrxiPyKCu21FxiRpOIgcOYDvv+cZGmk3h/v4Yy71bIuLjKxVXBx3dXh48GXsWEkatqBHD24Njh2rdSTWzcLLR4SWAgKAwYN5E5rOnXl30dKludtKmM4//3DV0wcPOFl3786tOmHdcuTgsiJ37/KAuKxhSp9MuXUwDx7wtpdEvKAoVy6+/+hRrotTu7a28dmLIUN4zwaA93Rv0EDbeIR4GZlyK9KVOzdv+3r2LJ9VAZxA2rcHPv1U29jsQUICj2P89BMvFvP1BerV0zoqkRlxccDs2cDBg1pHYp0kaTigxo15Rs+ECbwXgFI8trFvX8Z7cQjjjB/PeypcvszdHT16SDeHrYmPB0aOBEaP1joS6yR/zg5q8mTumvroI25p9OoF5MmTfmVcYZykJC5GeP8+tzLOnXvxFrzCOuXNy63uX3/l8SnxNEkaDqpIEd7Bb/t2HgjPl48Tx8qVwH//aR2dbdq+nfdhv3ED6NSJF1Y+3hNc2JZPPuHkMX681pFYH0kaDqxfPy5wNngwr90YNIir4h44oHVktmnBAh4zSkriKbd792odkciqAgW4yzYsjMumi1Qy5dYG/fEHr+w+e5b7zv/7j2vw79jBj2/YwN0jAQFcxCwjzs68GVNAAPfffvstr+OQ6aGZd+0ab/NZoAB3+z18yMUghe367DMuXW4Pu+2ZkrQ0bMTRo6nX58zh0h+nT/OHVOPGT8/QGTQIqFOH6/D368eD3RmpUQNYXTMU/0wzIDKSEwYRcGuNgasZCqN4eHDijY7mMY0OHbh7Q9iuwoX5RKxGDa0jsS6SNKwYEbBlC8/x9/NL3ZNh8mTeSOnoUX588eKnV7Hu3cvjFK1a8VaW5cu/+PO/8RcB+IkC8X13A4iAya0MoMBAJFUPMOv7sydubtzacHbmhX29emkdkTCVGzd4+2TBJGlYqStXeGvQ5s15U5Zp03gqJwAUL/7iLiQvL17xvXQpb+QyYkTq4rKYGO46ScvjXR32DAzDiMOBONZxFD7ZHYgOyWH4JUZnnjdn60JDAYPhyc0//wR+eN+A/PNDUagQ15p6800N4xMm9dVXQGAgdwULAERkV5caNWqQrYuLI/L2JnJzI5o4kW+bSrduRGXLEu3Z8/T9iYlEc4sEEQH0aFgQvfYaUa1aRMnJpnttu6HXE3l58VciCm6gp2h4UUPoado0onXrNI5PmNS5c0TOzkSDBmkdiXmBt9l+6Wes5h/ypr7YctKIj0+9vnYt0alTpn+NrVuJfHyIXFyI5s5N84BeT/H5vWg0guhebi/6ZZCeAKJt20wfg11ISRyxnwXRdXhRYCE9+foSJSVpHZgwh549iXLlIrp6VetIzEeSho25eJGoRg2i7783/2vFxBC1aMG//UGDiBK3pp45d+lC1NRVTwkFvKijl57atTN/PDYriFtmoxFEANGbbxIdPap1UMIcTp4kUopo2DCtIzEfY5OGjGlYgfBwwN+fZzkVKmT+18uXj6fsfvYZsHw5EGsI5wnpOh0mTgR2uegwrkoYpnUPx/Ll5o/HJhkMoNmzMc0jCB87zYYOBvzxh6xxsVdly/I4oSx8hbQ0tLZrF1HevNxlFBlp+dd/3NxOTiZKSODrX33FrZAdO/h2XJyMbTwlpWvq1ho9NWlC1MpdTzecvKhlLj3du6d1cMJcHv9/2CtIS8P6XbnCs6OKFQN27QIqVrR8DEWK8Ncvv+RKt/HxwNChwCuvcP2dyEg+y9qyxfKxWa1wbpkVaKfDgAHA+vs6dHUOQ2+/cLi7ax2cMBeXlKXQJ0+m7n7piCRpaKh4cS4QuHOn9vtHe3tzgbZu3XjNwTff8B7Xu3bx46NGyd7JTwwbhstldbh0CZg3jxf2/Z6gQ4lpw17+vcKmXbzIK/2nT9c6Eu1I0tDAvn18AXiP7sdn+1r6+GNOFKtXp85Lr1sXCA4GPv/8ycm1SDF+PC+a3LCB63fVqQPUqqV1VMLcSpYEWrYEpkwBYmO1jkYbkjQs7Ngx/qPr14+3lLQmn3/OSWzMGOCXX4CpU4Hr14FLl4Bq1Xg3uvv3tY5SezExwJIlXO/LyYkLFe7Zw/uSCPsXFATcvs2lfByRJA0LungRaNqUN+dZu9b6NudRigsYNmzIO9D5+wM9e/Jq9BEjgKgort7q6JYs4eQZFQU0asTjP5IwHEdAANCsGZfzccRihlb2sWW/YmO5FtS9ezyo/OqrWkeUPjc3QK/n7imAu2FcXLhras8ebiE5sqQk7s8uU4bPNiMieO8F4ViCg/l/2hHL32uSNJRSBZVSW5VSp1O+FsjguCSl1OGUy6+WjtOUZszgrqlVq4AqVbSO5sUenzXPnctN8BEjgDVruPWhFBdLdFQHDnA9rxw5uArqnTtAixZaRyUsrU4drkX11ltaR2J5WrU0vgCwnYjKANiecjs9D4moWsqlteXCM73hw7nMcpMmWkdivKNHeXe/N97gLpjH+wv4+HD3miOqWZNbipGRQP78POuseXOtoxJa8PTkr4624E+rpNEGwJKU60sAvKtRHGa3cSOvx3B2fnrPC1vw9dc8W6R/fy69fugQcOQId818+CGXjHYkcXH8dd067rI7fRro3fvFG10J+zZsGE8SefBA60gsR6ukUYSI/gOAlK+FMzjOTSkVoZTaq5SyucTyzz+8Gc/nn2sdSdbkyQPMn8/lTSIjgZklQ7HlCwNmzeL+/AEDwCXCHWSzpo4deaxn8WKgQgW+r3dvTUMSGmvdmmcYzpmjdSQWZMyy8axcAGwDcDSdSxsAd5459nYGz1E85eurAC4AeC2D4/oBiAAQUbJkSRMvrs+a27eJSpcmKlbM9itjvv8+l4beFaKn6/Ci+V31NHYsUUPo6VHe1BLh9iwykkurPC70uGuXQ7xtYYTGjYmKFCG6f1/rSLIH1lzlFsBJAMVSrhcDcNKI71kMoMPLjrOG2lPJyURt2nD58d27tY4m+27eJDIY+Pr4Jpw4bnwcRLdcvGheF8f45Ozdm0tj+/oS1a6tdTTCmuzaxZ+kkyZpHUn2GJs0tOqe+hVAz5TrPQE8t5miUqqAUipnynUvAHUBHLNYhNnw3Xfc7z1pEq+qtnUFC/LaDQDoOl+HBS794TlrDPIM6Y++K+x/d7/z53kXxFmlQlHqvAH58gGzZ6c86EDdcyJ99erxBJdFixyj1I5WSWMCgCZKqdMAmqTchlLKXym1IOWYCgAilFJ/AzAAmEBENpE0unYFvv0WGDhQ60hM6+uvgYnNDRiYYzZCEASaPRswGBARwf389mryZB7sPuwagNUqEIlbDYiKAieMwEBe7SUc2oIFvGbDERZ5KrKz1Ojv708RERGavHZsLJAzJ8/ht0fb/2dAlXGBuFG/HUJOdYa7OzD/biBCKoVh925g0Ufh8J5uf0X7YmKAn37iGWODqhgw8p9AuA7sj/wrZj/Zh0QIAEhM5Iubm9aRZJ5S6gAR+b/sOFkRbiJEXLfpzTd51bA90uUNR1DZMAQf74xFDwJx9hzwW/cwDC/1E36iQHyxJsDuirgRcRXbgwf5hGDFfzpsL9Mf+aeP4bnIkjBEirt3gcqVuWVq14wZ+LCli1YD4XPm8GDYxImavLzFGAz8PsP66+m2qxd9kyuIkjy96PAUPTk5EXXubD8bNp06RVS1Ku+r7uZGpNPxjLG4fF681auXY8wcE8Z75x2iAgV49qStgTXPnjLnRYukcfw4z6xp0oQoKcniL29xjRsTFS9OdLkP75G9MSCIiIjGj+e/qFWrNA7QRN57j3+vAwfy/tDbRurpbk4vStqWkij0ekkc4imHDvH/QFCQ1pFkniQNC4mLI6pencjTk+jKFYu+tGYiI4milvEH5kb/ILoOLzo9X09JSURLlhAlJmodYfYdPMiJYuBAIg8Pog4diJuRzyYIvd7+m5ciUzp2JMqTh+j6da0jyRxjk4aMaWTTtWu8ReqCBbxtqyOoeM2AEp8FAmFhqLUlBH3zhcGrfyDUDgN69OCZRlevAtHRWkeaNURcZ8vTky8xMUDt2sDFzsOeH8PQ6biWhBApQkK4rMiCBS8/1hbJ7CkTSExM3T/YIYSGIr5qANrP1KFePcDLC/ihjwGTO4Wj+k/DEB/Pu9qVKgX8/jvg6qp1wJmzeTNXrp0+nUvDlyvHJdDbt+e9NIR4mT//5J0cbakumcyeMrOYGGDoUJ4x4VAJAwCGDUOOZjokJ/O6tvbtgfsBOrz9xzDcvctTjkNCuKrvZ59pHWzmNWkCLF/Oye7qVa4zdf8+MGiQ1pEJW/HGG5ww4uO1jsT0JGlk0cCBvE/wyZNaR6Kd0aOBW7d4z41Zs7irLiiIH+venbeHnTWLix7aiqQk/mcPDOSpkzVqAJs2AQ0aANWrax2dsCXbtnGV6DNntI7EtCRpZMGaNVxWYuRIx14M7O8PvPMOf7iWLcvLFmbO5K4cAJgwgfea+Phj29jh7OJF4JtCodj7tQFLl/I/e5MmgO+/BswqJaVCROZUrswLfv/3P60jMS1JGpn033/ABx/wB6a9/TFkxVdfcZn0mTO5/79wYf75JCbyGfuPP/Kix8elxK0VEdC3L7DzYQD8vwnE7yMMqFmTB/3XOAeiQg8HPjsQWVK0KDB4MLByJe/4aDeMmWJlSxdzT7nt1IkXeh0/btaXsSnLlnElXCKilSt5IvfUqc8fd/8+UXS0ZWMz1vffc9wzZhCtHciVfM924wV8cZtlHYbImpgYno7/1ltaR/JykHUa5nH+PNHPP5v1JWxacjLvOZEnD9HFi0/f36gRUUAA0b172sWXnqgoXovRoAFRbCxR0aJEi0vywkWbXKUlrMqUKfynFBGhdSQvZmzSkO4pI8XEcBeGjw/Qtq3W0VifvXt5yUJMDA9+JydzgT9KmdGtFM8+OnCAdzN8vHWqNVi1ime5fP89MHcuUP6qAZ1vz8Y4pyA8/JYr+QqRVf37A7t28aQKeyBJwwhJSTzg27Wr1pFYr1y5eIrt5MmAry+XUd+48el1Da1b84fy5s1Aly487mENPv0UOHaM15vsCjFgrWsgFjQNQ7AKQcz8MJ5KJYlDZFHOnLznBmBdJ0tZJUnDCFOm8JlCixZaR2K9qlblz9YpU3jP5AEDgPr1+QP58uXU4/r0AaZNA9au5UFCLf3xB/D333zdx4cH9cvFhuP8hDAM26RD165A0S46Ln8eHq5lqMIOTJsGVKoEPHyodSTZZEwfli1dTD2mceQIUY4cRO++az/VW83lxAkiJyeizz7j26dPc8G/li2f/9nNmsVVZLVy/jwPUAYEcGyRkbwP+gcfEI0axX3QR49qF5+wP48rRI8bp3Uk6YMMhGdfXByXxi5c2PaKj2mld2+inDlTB8GnTuW/su+/T//45GROIPfvWy7Gu3e5yKSHByeu5GSuUJw/P9HVq0SlShG1bm25eITjaN2aJ4lcvap1JM+TpGECx49zCfBffjHZU9q9CxeI5s4lio/n20lJRA0bErm7E508+fzxe/dyNdl69Yhu3TJ/fHfv8ms5OxOtX8/3/fIL/ydMm8a3b93i9yGEqZ04QeTiwi1aa2Ns0pCChS/x4AGQO7fJns4hRUXxmEepUsBff/HAYFphYVx25NVXgZ9/BipWNF8so0cDY8bwosOOHfn36+fH23OGh/NXJxnpE2Y0cCBXwL14kSdfWAspWJgN9+7xLKCEBEkYWbV4MU+5BQBvb2DRIuDQIWDEiOePDQwEtm7lleU1a3KZFnP58kueCNWxY+rtc+d4RfvcuTwt8s4d872+EMHBwOHD1pUwMkOSRjoGD+YKtna19N/CLl3iD+Fdu/h269Y8o2rKFGD9+uePf/NNTir+/rwntynt28cFB2/e5Mq19evz/X/8wTNaPv6Ya4h9/TXvn5E/v2lfX4i0PD25VhvABT9tjjF9WLZ0ye6Yxrp13L89bFi2nsbh3b9PVKIEkb9/6ha4Dx8SVavGA9AnTqT/fWlnWY0bR7R4cdZnrSUmEk2axH3IPj5Pz4aKjSXy9SV67TVeoT52LP/e9+7N2msJkVlDhvDf4IMHWkfCIAPhmXf1KlGhQvzB9uhRlp9GpFi6lP/Cli1Lve/CBf4Zly374oHvxESi+vX5+/39+Tky8ztZvpyoXDn+/nffff61PvyQB+B37uTHPDyI3nknc+9PiOzQ6/nvc8wYrSNhkjSyoF07ni4q8/NNIymJP/C9vbmV8diuXUSurkTNmhElJLz4+xcu5AQDcLL54w9+LDHx6RZITAzRli2p97VvT+TnR7Rq1fMtlbAwfr7H60m+/ppvHz6c/fcsRGa0b89rmdLWadOKJI0sOHiQ6IcfsvztIh379hFt2vT8/fPn81/fxx+/vPspKYlo61ZuMRw4wPf9+CNR7tx8yZmTWw0ALygkIrp9O7VbLK1Dh/h73ngjteUSH88JRwhLO3+ek0a7dlpHYnzSkCm3wmKSk5+ezjp0KDBpEjB8OA9CK2X8c+3Zw7OsnJx43448eYA6dbjGj5tb+t8THc0D7cnJPL22aFGuBfTsFGAhLGn8eODbb4EjR4BixbSLw9gpt462u7XQyNdf82ylTZtSk0NoKE9vnjiRpzaPGmX889WtyxdjxcVxdd3r14Hduzlh7NnDU2/Xr5etXIV2hgzhjcs8PbWOxDgy5VZYRMGCwJYtwLJlqfcpxWXUe/bkuevjxqWWUjelB8gKt3QAAAkxSURBVA94yu/OnVz+vEYNrlw8YAC3UsqVM/1rCmGsHDk4YSQl8fRwa6dJ0lBKdVRKRSqlkpVSGTaHlFLNlVInlVJnlFJfWDJGYVp9+3L30eDBwI0bqfc7OfEHebduvH1u796mLR8dGwu8/TYvHvz++9Ty9vPn8wKrSZMAd3fTvZ4QWRUSwmuITpzQOpKXMGbgw9QXABUAlAOwA4B/Bsc4AzgL4FUAOQD8DaDiy57b3Dv3iaz75x9eM9G16/OPJScTBQfzYHb9+qYpEHntGlGdOlxnasWKp+8vWJBrYknlYmEtrl0jKlCAa6OlN4nD3GDNO/cR0XEiOvmSw2oCOENE54goHsBPANqYPzphLn5+PG6xahVw/PjTjynF+1n8+CMPUletCqxcmfXuqrVrgcqVgYMHubZVly6pjy1bxmMpM2dmbvBdCHMqXJgrJuzeDcyZo3U0GbPmMY0SAC6luR2Vcp+wYV9+yeVCKlRI//HOnYE//+RZJJ07A82bAydfdnqRRlQU0KsX0K4d17w6cICvpzV4MG++VKlSlt+GEGbRowfQtCnPKLx4Ueto0me2pKGU2qaUOprOxdjWQnrngOmedyql+imlIpRSEdHR0VkPWpids3Pqh/X27VwU8lmvvw7s3w9Mn857j5cvz/uPL1nCYxTPevSIn6t9e96Bb9kyHh/Zu/fpxPDvv8CpU9y6KF/eLG9PiGxRimu2lS3LtdKskabrNJRSOwAMIaLnFlYopeoA+IqImqXcHgEARPT1i55T1mnYhogILhI4fDgwYULGx129ymWkFy8Gzp7l+4oU4TLrefLwfRcvcjeWpyfw/vs8ffHVV59+nqQkoFEjThrnz2e8lkMIa0Bk+a5Te1inEQ6gjFLKF8BlAJ0BdNU2JGEq/v5Av368RqN8ee5SSk/RotxqGDmS11Xs2MEthn//5VZH3br8vZUrA61aZZwMhg/nKbeLFknCENZPKZ4qPmoU779RsqTWEaVhzGi5qS8A2oLHKOIAXAOwJeX+4gA2pjmuJYBT4FlUI415bpk9ZTvi44neeotnVG3fbr7XmTOHZ2V98on5XkMIUzt3jreGbdTIMrOpIGVEhC2IieHWwuXLwOnTpt+YZu9eLi3SrBmwbh3gYs1tayGesWABr3GaOhUYNMi8ryU79wmb4OEBbNgAzJhhnp3Mqlfn7q2ffpKEIWzP++9zt+sXXzw/TV0r0tIQVmXbNp5hpdNl73k2bQKqVAFKyCRtYeOuXuU1TuXK8U6Y5hogl5aGsDlEPPDXvDmwfHnWF/bNnctnZyNHmjY+IbRQtCgviF240DoWo0rSEFZDKe6qqlUL6N4daNMmcwuc/v0X6NQJ+PBDoEULLoYohD1o2JDXbhDxlHEtSdIQVqVAAUCvByZP5gV7FSsCFy68/Pt27OCpu7/9BoweDfzyixQiFPYnJIQXv2qZOGRMQ1itixe5/tTQoXz7iy/4TMvdndda3LrFi/j69eM57UOH8noMq5rTLoQJnT/PSaN0aR7fyJXLdM9tD4v7hIMrWTI1YRDxXgO7dwOJiXyfiwtPRwR4EyfpjhL2ztcXWLqUu2779+fFqpYe55CkIWyCUoDBwNcTE3nPDWdnWd0tHE/r1rxp2ejRvKHYJ59Y9vUlaQib4+Iiay6EYxs1iruqypSx/GvLv54QQtgYJyeu+vzYo0eWa3XL7CkhhLBhc+fypmVpt1E2J0kaQghhw/z8eI3Su+9yi8PcJGkIIYQNe+MN3njszz95nZK5yZiGEELYuI4deSGsJbYwlpaGEELYAUvteS9JQwghhNEkaQghhDCaJA0hhBBGk6QhhBDCaJI0hBBCGE2ShhBCCKNJ0hBCCGE0SRpCCCGMZnc79ymlogH8q3UcWeAFwEIlx6yGvGfHIO/ZNpQiokIvO8jukoatUkpFGLPVoj2R9+wY5D3bF+meEkIIYTRJGkIIIYwmScN6zNM6AA3Ie3YM8p7tiIxpCCGEMJq0NIQQQhhNkoYVUkoNUUqRUspL61jMTSn1jVLqhFLqH6XUWqVUfq1jMgelVHOl1Eml1Bml1Bdax2NuSqlXlFIGpdRxpVSkUmqQ1jFZilLKWSl1SCm1XutYzEGShpVRSr0CoAmAi1rHYiFbAVQmoioATgEYoXE8JqeUcgYwC0ALABUBdFFKVdQ2KrNLBPA5EVUAUBvAxw7wnh8bBOC41kGYiyQN6zMFwDAADjHYRES/E1Fiys29ALy1jMdMagI4Q0TniCgewE8A2mgck1kR0X9EdDDleiz4Q7SEtlGZn1LKG8DbABZoHYu5SNKwIkqp1gAuE9HfWseikd4ANmkdhBmUAHApze0oOMAH6GNKKR8ArwPYp20kFjEVfNKXrHUg5uKidQCORim1DUDRdB4aCeBLAE0tG5H5veg9E9G6lGNGgrs0llsyNgtR6dznEC1JpVQeAGsAfEpEd7WOx5yUUq0AXCeiA0qphlrHYy6SNCyMiN5K736llB8AXwB/K6UA7qY5qJSqSURXLRiiyWX0nh9TSvUE0ApAY7LPOeBRAF5Jc9sbwBWNYrEYpZQrOGEsJ6KftY7HAuoCaK2UagnADUA+pdQPRNRd47hMStZpWCml1AUA/kRka0XPMkUp1RzAtwDeJKJoreMxB6WUC3iQvzGAywDCAXQlokhNAzMjxWc+SwDcIqJPtY7H0lJaGkOIqJXWsZiajGkIrc0EkBfAVqXUYaXUHK0DMrWUgf4BALaAB4TD7DlhpKgL4D0AjVJ+r4dTzsCFjZOWhhBCCKNJS0MIIYTRJGkIIYQwmiQNIYQQRpOkIYQQwmiSNIQQQhhNkoYQZpZS8fW8Uqpgyu0CKbdLaR2bEJklSUMIMyOiSwBmA5iQctcEAPOI6F/tohIia2SdhhAWkFJS4wCAhQD6Ang9peKtEDZFak8JYQFElKCUGgpgM4CmkjCErZLuKSEspwWA/wBU1joQIbJKkoYQFqCUqgbekbE2gM+UUsU0DkmILJGkIYSZpVR8nQ3eU+IigG8ATNI2KiGyRpKGEObXF8BFItqacvs7AOWVUm9qGJMQWSKzp4QQQhhNWhpCCCGMJklDCCGE0SRpCCGEMJokDSGEEEaTpCGEEMJokjSEEEIYTZKGEEIIo0nSEEIIYbT/ByixNxS6Up4ZAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot(xt, f_mean[:,0], 'b-', label='mean')\n", + "plot(xt, f_mean[:,0]-2*np.sqrt(f_var), 'b--', label='2 x std')\n", + "plot(xt, f_mean[:,0]+2*np.sqrt(f_var), 'b--')\n", + "plot(X, Y, 'rx', label='data points')\n", + "ylabel('F')\n", + "xlabel('X')\n", + "_=legend()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Posterior samples of Gaussian process\n", + "\n", + "Apart from getting the mean and variance at every location, we may need to draw samples from the posterior GP. As the output variables at different locations are correlated with each other, each sample gives us some idea of a potential function from the posterior GP distribution.\n", + "\n", + "To draw samples from the posterior distribution, we need to change the prediction inference algorithm attached to the GP module. The default prediction function estimate the mean and variance of the output variable as shown above. We can attach another inference algorithm as the prediction algorithm. In the following code, we attach the ```GPRegressionSamplingPrediction``` algorithm as the prediction algorithm. The ```targets``` and ```conditionals``` arguments specify the target variables of the algorithm and the conditional variables of the algorithm. After spcifying a name in the ```alg_name``` argument such as ```gp_predict```, we can access this inference algorithm with the specified name like ```gp.gp_predict```. In following code, we set the ```diagonal_variance``` attribute to be ```False``` in order to draw samples from a full covariace matrix. To avoid numerical issue, we set a small jitter to help matrix inversion. Then, we create the inference body in the same way as the above example." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from mxfusion.inference import TransferInference, ModulePredictionAlgorithm\n", + "from mxfusion.modules.gp_modules.gp_regression import GPRegressionSamplingPrediction\n", + "\n", + "gp = m.Y.factor\n", + "gp.attach_prediction_algorithms(targets=gp.output_names, conditionals=gp.input_names,\n", + " algorithm=GPRegressionSamplingPrediction(\n", + " gp._module_graph, gp._extra_graphs[0], [gp._module_graph.X]), \n", + " alg_name='gp_predict')\n", + "gp.gp_predict.diagonal_variance = False\n", + "gp.gp_predict.jitter = 1e-8\n", + "infr_pred = TransferInference(ModulePredictionAlgorithm(model=m, observed=[m.X], target_variables=[m.Y], num_samples=5), \n", + " infr_params=infr.params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We draw five samples on the 100 evenly spanned input locations." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "xt = np.linspace(-5,5,100)[:, None]\n", + "y_samples = infr_pred.run(X=mx.nd.array(xt, dtype='float64'))[0].asnumpy()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We visualize the individual samples each with a different color." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzs3XWYlFX7wPHvM7Xd3cEmu0t3SXeDKAICdteLor7mT3197UBCRBQFpaQbFOmOpWNhu7t3Z2fm+f2x+EosEjs7s3E+18Ulu3P2OffI7D3PnLiPJMsygiAIQtOiMHcAgiAIgumJ5C8IgtAEieQvCILQBInkLwiC0ASJ5C8IgtAEieQvCILQBInkLwiC0ASJ5C8IgtAEieQvCILQBKnMHcDNuLq6yoGBgeYOQxAEoUE5cuRIjizLbrdqV2+Tf2BgIIcPHzZ3GIIgCA2KJEmJt9NODPsIgiA0QSL5C4IgNEFGSf6SJM2XJClLkqRTN3lckiTpa0mS4iRJOiFJUhtj9CsIgiDcHWPd+f8IDPyHxwcBoVf+PAbMNlK/giAIwl0wSvKXZXknkPcPTUYAP8nV9gOOkiR5GaNvQRAE4c6ZaszfB0i+6uuUK98TBEEQzMBUyV+q4Xs3HCEmSdJjkiQdliTpcHZ2tgnCEgRBaJpMtc4/BfC76mtfIO36RrIszwXmArRr106cLykItaDTG7iUXcLOhNOczD6JlUaBg5UFbrY2DArpiK+9+PDdlJkq+a8BnpEkaTHQESiUZTndRH0LQpMSl1XCJ9u3cyJlLU6KfCz1tlhV2VJomUOedRrFFvl8fVLGUeFBV78ePN56EkEOQeYOWzAxoyR/SZJ+BXoCrpIkpQBvA2oAWZbnABuAwUAcUAZMNUa/giD8LSGnlPc3HCA7aTMxhc2YWHx/je2UhjJsFHs4GfgHW3TL2JCwnEGBI3ix/dN42niaOGrBXCRZrp+jK+3atZNFeQdBuD3bzmTyxfJf6ZTvinO5N2ptHn4pO7ErScVGXYmtvxclsjVFOhtyZFcyHWJAAveiY5S4LuOrNuXICjXPt3meKdGTkaSapumEhkCSpCOyLLe7Vbt6W9tHEIRb0xtkvthyjtjdGxiU0wKLylyaXf4JP+ss3B5/GOuOHVH7+FyTzGWDgdyDpzi+7jxxcguk8uZ8sXQRh2IO87n8KQfTD/NRz/9gr7E34zMT6pq48xeEBspgkHnp1yOoYg8SVBSBa84JWpZsxmfav7Dt3RtJcevFfAWZJWz+Yh85BUq80vfhWrWI6cMVWDt5MavfN4Q7h5vgmQjGdLt3/qK2jyA0UB+sPY3T4eMEFUUQFL+ebkEJhK1aiV3fvreV+AEcPWwZ+0Ef2g7wI92rE0kOL/LtfAV+F9KZvGEyZ3LP1PGzEMxFJH9BaIC+3RFH2Y6duJYHExa3iK6PtMPnky9RWFre8bWUSgWdRoUy8LEYip2acTzyZV5cakH//UVM2TCVk9kn6+AZCOYmkr8gNDCrj6dycs0mfMsi8EvaSJd3JuBw7/haX7dZG3cGPdmCMjtfjnd+jdE77Rizs5iHNj3MiewTRohcqE9E8heEBiQhp5QVv6wnsrQ57pkH6P5Me2y69Tba9QNjXBn2bCsqLF052Xkag/dqmLS1hMc3P05ycfKtLyDUmrZCR1Wlvs77EclfEBqIKr2B1xbsomOBD/aFcXS51w6nweOM3o9PuBMDH4+hWOnC2Xteps8hiXGbC3l00xMUaYuM3p/wN73OwKa5p1j79XEMhrpdjCOSvyA0EDO2XaDLuSQUBonw8JP4PPBMnfUVEOVCzwkRZBu8uNTzWQYckmn9ZwLPbnuBKkNVnfXblMmyzPafz5F8Jo+ILl4oFHW710Ikf0FoAA4n5JG9Zg0KZQhuZWtp/9ZXdd5n867etBscSLIcRkq7sUz6XY961wE+2PdhnffdFO1fdYnzBzLoMCyI5l2967w/kfwFoZ6r0hv47Met+GtjsC06w6BPX7rtpZy11WFYECHt3Imz60VxcDQvrNFzavtSNlzeYJL+m4oT21M4ujmJqO7Vb7im0KR2+FYZqriQd4GjWUc5nnWcIm0RKoUKlUKFr60v3X27086jHRqlxtyhCsL/LNibQO/4PPRqG6KHVmLtHWqyviVJotfECHKSSzht/SztC15h+ooiXnN6k+YTmxPoEGiyWBqrxFO57F56gcAWrvQYH26y0hpNYodvsbaYxecWs/DsQvIqqg8c87H1wdXKFb1BT5WhivjCeLQGLVYqK/r69+WZ1s/gbVv3H70E4Z/kllTyxbSP8KIb9tI6Js3+3DxxpJWw/L+HcXWqIuK357jkLjHvsTCWjl2MperO9xYI1XLTSvjt4yM4uFkxelpb1BbKWl9T1PYBtHot807OY+GZhRRXFdPNpxsjmo2gtXtrPGw8rmlbrivnUMYhdiTvYM2lNWxO2MzE5hN5JOYR7DR2ZnoGQlP35ZpjBJUHoVdkMeSDJ8wWh4u3LT0fCGfbj2dxGvYcoSu/pufaC3zg8xHvdX/bbHE1ZOXFWjbMOoFao2Twky2MkvjvRKMd87+Qf4Hx68czO3Y2nbw7sXToUmb3nc3AoIE3JH4AK5UVPXx78GbnN1k7ai0Dgwbyw6kfGLl6pNjiLpjFmbQifNYvo8LKB/vQkzh7h5k1nvBOXkR28eJsQTiVHVoy6IhM7pql7E7Zbda4GiK9zsDGb09SWqhl8JMtsHM2/aenRpf8DbKBn8/8zPh148kpz2Fmn5l83vNzIl0ib/sanjaefNDtA34Z8gsKScHkjZPZlritDqMWhBvN/2EdSk1XlFXxjJn2hrnDAaDbuFDsXa044f0UkqeaRzcZ+Gjla2L9/x3aszyO9LhCek+KwCPIPNVTG13yTypK4osjX9DFpwsrhq+gh2+Pu75WtGs0vw75lTDnMF7880W+P/m9ESMVhJs7HJ9L26PH0Fo4EjigCrXGytwhAaCxVNFvanNKivQkDngfCww8tDqHN7a/Z+7QGoyze9M5+WcKrfr6EdbBfIfnNLrkH+gQyOKhi/m619e4WLnU+nquVq7MHzCfQUGD+PLolyw6u8gIUQrCP9v03UKKHO8B6RQD7n3c3OFcwzPYgXaDA4lLtKWi7yAiUsB+5QY2x4tPx7eSmVDEjl/O4xPuROdRzcwaS6NL/gBhTmFGXS5lobTgw24f0se/Dx8d/IjNCZuNdm1BuN6xxDzCz1xGp7YmeoxvvTxVq92gADyC7DkqjUIRbMV9Ow18t+xNMfzzD8pLtGz69iTW9hoGPBqFQnmT9HthC5xcXufxNMrkXxeUCiX/7f5fWrm34rVdr3Eo45C5QxIaqd9nzafA6R70iovc03e0ucOpkUKpoM/kSHRVMnFd3kVhoefRNQW89bvY/VsTg0Fm6/wzlBdXMfDxaKxsb7KX6MRSWDwe9s8GQ90WdxPJ/w5YqiyZ0XsGfnZ+PL/9eVJLUs0dktDInEzMI/TUebQWjoQPrf2wZV1y8rSh4/BgElJsqOo7EP8ccFyxhgNpR8wdWr1zZGMCyWfy6H5fKO4BN5ngPfAtrHgU/DvDpJWgqNulnyL53yEHCwdm9pmJLMtM3zkdnUFn7pCERuTPGXPIc+mNTplCv4H1867/ai37+OERZM9h3WikYAvG7DHw6ZLpovjbVZLP5HFwXTzhnTxp3u0mG0d3fAIbX4GIoTBhOVjW/Qogkfzvgq+dL292epPY7FjmxM4xdzhCIxGfWUSzE2cos/YgqI81ChPV76kNhUK6MvwDlzq8jqQ2MGFdKh/unGnu0OqF0oJKtsw/jbOXDffcrHTD/jmw/X1oOR7uXQBq06z5r/+vrnpqcPBgRjQbwdwTc8X4v2AUO75bSK5rL3SKHIaMqP93/X9x8rShw/AgEjJc0XftSHgqlCyfz+X8JHOHZlbV4/yn0Wn1DHwsuuYdvCeWwabp1Xf8w78BpemKLojkXwuvd3wdf3t/Xt31KsXaYnOHIzRgJZU63HZuocg+CLf2WpQ3WwlST7Xq44ebvx1HLKdg8FFx/84q/r3s3+YOy6wOb0gg9UIB94wPx8nT5sYGF7fBqicgoBuM+d6kiR9E8q8Va7U1H3b7kOyybGYeFx9zhbu3ZckmKq3aYaCS4fc2nLv+vyiUCno/GEllhURKp+ew0Mv03nSExSe3mjs0s0i9kM/h9fGEd/QkorPXjQ2yzsKyyeAWCeN/MdlQz9VE8q+lGLcYxoWP49dzv3Iu75y5wxEaIINBRr14Nhke7VAGpGJra23ukO6Kq68tbQYGEJfbDG2rMLqdkVn32zuUV1WaOzSTKi/RsnX+GezdrOgxvoZ6TGV58Ot4UFvDA0vA0sH0QSKSv1E82/pZHC0ceX//+xhkg7nDERqYvbtiUWsDkBUa+o3pa+5waqXdoECcvGw44fEUlfZqJm/NY9p685ShNgdZlvl9wVnKS7QMeCQajeV1Qzl6HSx/CApT4L6F4OBjnkARyd8oHCwceLHti8Rmx7I6brW5wxEamJzZH5Hm3R2tbRqhYYHmDqdWlGoFvSdFUFKuIbvDOHxzwWHTLxxPTzB3aCYR+3syiSdz6TomFDf/GkrB//4OXN4OQz4D/44mj+9qIvkbyfBmw2nt3prPj3xOYWWhucMRGoiU1Gycksspt/Ygul+QucMxCs9gB1r29uOCrgtFIQGM3avj7RWNv+Z/VmIR+1ZeIqilKzE9a7ijP7sW9s6Adg9D28mmD/A6jS75a8t17F5+kYKsMpP2q5AUvN7xdQorC/nh1A8m7VtouI7PnEOmZw90ilJ69epk7nCMpuOIYOzdrLgQ/Dgqg4p+Ow8xZ3/jLfymLdex+btTWDto6P1g5I3r+fMTYPXT4N0aBtaPEhiNLvlXafWc2Z3G7qUXMfURlRHOEQwOHsyis4vIKssyad9Cw6PXG7DfvYVs1xgcW2hRaUx7klNdUmuU9J4YQZHWgbQ2g+kVK7Nx24cUVWjNHZrRybLM9kXnKM6rpP9DUVjaqK9toNPCsqkgyzD2B1BZmCfQ6zS65G/jYEGHoUEknsol4USOyft/utXT6Aw6vo391uR9Cw3LofV/Uq6JAUlJvyE9zR2O0fmEOxHVw4fL1v0pcA1g0s40nl8719xhGd3ZPenEHc6i4/AgvEIcb2yw7R1IOwojvgHn+jO01+iSP0BML1+cvW3YtfQiOm3dVsa7np+dH2PDxvLbxd9ILEo0ad9Cw1K64CvSvDpRZZ+Nt1/9LuJ2t7qMboaNkwWXoiYTkaJGOvwDBxPTzB2W0eSmlbBryQV8I5xo0z/gxgYXtsD+mdDhMWg+wvQB/oNGmfyVSgU97g+jOLeCI5tNn4Afb/k4GqWGmcfExi+hZvkZudgkl1Jm40t4Vz9zh1NnNJYqej/YnGLJg8sRg5m0o4zp6z9BbzDtkGxdqNLq2TLvNGpLJX2nNkdSXDfOX5INq58C9yjoV/9OOmuUyR/AJ8yJ0PYeHNucRGG2aSd/Xa1cmRg5kY0JGzmfd96kfQsNw7FvZpLl1hEDOnr2bmfucOqUX6QzUd29SXHvi5UhiI6ntjJr9wFzh1VrOxdfIC+9lL5Tm2PjcN04vixXT/BWFMGYeWbZwXsrjTb5A3QdE4JCKbFz8QWTT/5OiZ6CjdpGnPsr3ECWZWy3rSbVqz3qgBKs7G5ysEcj0mVMCDYOas5GT2DkfgWLDn1BVnGFucO6a2f3pnFubzrtBgXi37yGIbtD8+DiZuj/Hng0N32At6FRJ38bRws6jggm6XQeFw9nmrRve40948LHsTlxM0lFTbu6oXCt83/spVIRhEFlT+c+Lc0djkloLFX0mRpDmcaLDJ9RjI49yb/WrDR3WHclN7WEnb9ewCfcifZDa5jAzT4PW96AkL7VY/31VKNO/gAxPX1xD7Bj99KLVJSa9oCJB5s/iEpSMf/UfJP2K9Rv2T/OIM2rEzp1OdFt6s/qj7rmG+FMq16epPrcQ4fLzUlJmceOC6a9KastbYWOTXNPobFW0f/hKBTXj/PrtNWncWlsYMQsqIfnL/+l0Sd/hUKi16QIKkp17F0RZ9K+Xa1cGRU6ijWX1pBZ2rBe5ELdMFRWYn36AjmuMbi10KBUNfpfwWt0Gh2Bs6OWC2ETmXCgmOmbf6CiyrQr8u6WbJD5/cezFGaXM+CRKKztaxiu2/FfSI+FYV+DnYfpg7wDRnnlSZI0UJKk85IkxUmS9GoNj0+RJClbkqTjV/48Yox+b5errx2t+/lxdk86qefzTdk1U6KmYJAN/HTmJ5P2K9RPJ5esoNAuBiQ13Xu1Nnc4JqdUK+j/TFf0aitslRPwyl/OjO2nzB3WbTmyOZHLx7PpMroZ3qFONzZI2g+7v4DWEyFyqOkDvEO1Tv6SJCmBmcAgoDkwXpKkmmY4lsiy3OrKn3m17fdOtRsShL2rJX8sPEdVpenuNHztfBkUNIhlF5ZRUFFgsn6F+qli2XxSvdqityrFt1njXNt/Ky6+dnTuZ0+uSwyjTnRk/snvic8pNXdY/yjxdC4H1lwmtL0HLfvUsDS3shhWPg4OfjDwv6YP8C4Y486/AxAny/JlWZa1wGKgfu1m4Mp28wcjKcouZ//qSybt+6HohyjXlbPswjKT9ivUL5W5eVgk5lPoEIlPa8eaz3NtIlqO6YSvRQIlLiPpk5nIq6t3mHxF3u0qzC5j6/encfG2pdekiJr/3Ta+CgVJMOpbsKihmmc9ZIzk7wMkX/V1ypXvXW+MJEknJElaLkmSWXa1+IQ5EdPTlxPbU0i7aLq78FCnUDp7dWbx+cVUGUw76SzUH6fnzSXXpSWSpKRbjxbmDsesJEliwJsjsdTmEZ07mcv5y1gTW/92/laUVrHumxMgwaAnYlDXVH/pzGo4vhC6vQQBnU0f5F0yRvKv6fbl+rfwtUCgLMstgG3AghovJEmPSZJ0WJKkw9nZ2XcfUeYZMNR8qErnUc2wd7Hk95/OUmXC0g8TIieQVZbF74m/m6xPoZ7ZsIJEnzYYbMtxD7A3dzRmZ+nqTO+22cgKa+693J7/27yewrL6c3OkrzKwcc5JinLLGfxECxzcrG5sVJQGa5+vrtbZ84bpznrNGMk/Bbj6Tt4XuOYtXJblXFmW/zrL7TugbU0XkmV5rizL7WRZbufm5nZ30WRfgLn3VB+aUAO1xd/DP/tWmm74p7tvd/zs/Fh4dqHJ+hTqj+JLl1HmyZTZhhPUzq1JD/lcLeDpp2meswobQwRdCvP5cMNpc4cEVG/E+2PhWdIuFtBnciTeoTUUbDMYYNWToKuE0fNAqb6xTT1mjOR/CAiVJClIkiQNcD+w5uoGkiRdfYLxcOCsEfqtmWsotHkQ9nwFh2reXesT5kSL3r6c3J5C0uncOgvlagpJwQMRDxCbHcupnIaxukEwnri535Dl1hoJBZ26RZo7nHpDkiTaPT8K/6RtROS1J/7IQQ4n5Jk1JlmW2b/qEhcOZNJxeDBh7T1rbrjvG7j8Jwz4D7iGmDRGY6h18pdlWQc8A2ymOqkvlWX5tCRJ/ydJ0vArzZ6TJOm0JEmxwHPAlNr2e1OSBAM/gtABsGFadVW9GnQe1Qxnb5vq8zaLTVNjfGTISGzUNiw6u8gk/Qn1h7T7TxJ824JjBS4+tuYOp16x6TmAFg6ncM49Svf8YL766RBanfnOwj6yMYGjm5OI6uFD20E1VOoESD0Cv78LkcOg7RSTxmcsRlnnL8vyBlmWw2RZbibL8gdXvveWLMtrrvz9NVmWo2RZbinLci9Zls8Zo9+bUqpg7HzwiIZlU6o3XVxHpVbS76EoKsqq+OPncyZZaWCrsWVkyEg2JWwiu6wWcxpCg1J8/jwUq6m0bkZoOy8x5FMDj3c+Ifr0TxgMibRLl/h2mXmGf45tTeLAmnjCO3lyz/1hNf9bVRRVH8Ju5wXDZ9TrXbz/pPFuL7SwhQeWgpUTLBpXvQzrOq6+tnQe2YyEEzmc3mWalQbjI8ajM+hYfnG5SfoTzC9+3gyyXVsgoaBtl1Bzh1MvWYaH4zKoF90OzqZMk41+RyYH96eaNIbj25LY+1sczdq403tSxI0lmqG6Wue6F6Egubpap1UNm70aiMab/AHsvWDicqgqh4VjofzG3b0te/vhF+nE7mUXyU0tqfOQAuwD6OTViRUXV6A3NIxt7ULtyLt3k+DbChwqcfayMXc49Zbbv6Zjpa/AJe1Lyi2z2b/gPIkmmJOTDTK7l11kz/I4glu70e+h5iiUN0mNR36EU8uh12vg37DPXG7cyR/APRLuXwT58bB4QvXM/FUkhUTfqVForFRs/u4U2gpdnYd0b9i9ZJRmsCdtT533JZhXyelTSMVKKq3DCG7tLoZ8/oHa2xvnSQ/S7WQJBzy/okRVxLpZJ+q0Iq+uSs/meaeJ/T2ZFr18GfBo9M3rLSUfgg0vQ7M+1Wv6G7jGn/wBgrrDyNmQuKd6C/Z1ewCs7TX0e6g5+Zll7Fp8oc7D6eXfCxdLF7HjtwlI+n4G2a7RSChp00kM+dyK62OPobSzZcrOQnZFfUiWQsuWeafZuyIOg5FP/yrILGPFJ0e5dDSLrmND6DYu9MYqnX8pyYKlD4K9d/Vwj6KGzV4NTNNI/gAxY6uPUju9Eja/Vj12dxW/CGfaDQ7k3P4Mzu5Nr9NQ1Ao1I0NGsjNlJxmlGXXal2A+siyj37ufy36tkG20uAc0jG3/5qR0dMT1iScIvwwOmSXkB79PjoeaY1uSWDfjOKWFlbe+yC3IsszZveks+c+h6g1cT8bQqq//zT+V6atg2dTqYeP7F4G1c61jqA+aTvIH6PIsdHoaDsyBPV/e8HD7IUH4hDmy89fz5KQU12koY8LGYJANrLzYMA+0EG6t/PQJKJaosIkkoIWzGPK5TU4TJ6Ly9uKZHWr2OOZRKC3ApZcXaRcLWfT2fo5tTUKvv7uloLmpJWyYdYI/fjqLR4Ad97/RgaCW/7ChVJarl4wn7oZhX4FnzF0+q/qnaSV/SYL+70P0GNj2Dhz/5ZqHFQqJ/o9EY2GtYuOck3V6+IufnR9dvLvw28XfxMRvI5X0w0xynSNRoKF1p4a3CchcFBYWuD//PC5plXQ/J1HguYM951czaFprvEMc2ftbHEveO8jZvem3NUcnyzI5KcVs+f40i98/SFpcIZ1HN2P4C62xdbrF2bq7Pque5O32IrS8zzhPsJ6Q6mslvXbt2smHDx+um4vrKuGXcRC/C+77GSKGXPNwxuVCVn52FL9IZ4Y81aLmJV9GsDVxKy/9+RLf9P6Ge/zuqZM+BPM52bUVhwLHU+bWmqc+63vzFSTCDWSDgfjRYyjOy2DK5GJezSsmwe1zpk8ZQ8LJHPb+Fkd+RhkqtYKgVm54hzri4G6Fg2t1/Z2yIi2lBZWkXSwg/kQOxbkVqCyUtOzlS6t+/lja3EYphtjF1XOEMeNg9NwGs55fkqQjsiy3u1U7lSmCqXdUFnDfQvhpRPVY3sTlENTjfw97BjvQ7d5Qdi6+wKH18XQYFlwnYfT064mLpQsr41aK5N/IVF6+gCJPT2lMNL7R9iLx3yFJocD95WlUPvwID531YkZz+DbxFXYcCeOetjEERLuQGV/E+f0ZXDycycVDNa8IUqoV+EU40XZgAMGt3LCyq+H0rZpc3Aarn67OCyNmNpjEfyeaZvKH6prbE5bDD4Ph1/EweQ34/F1vLvoeH7ISiji0PgEXX1uatXY3eghqhZqhwUNZdHYReRV5OFs2jokkAVIXzCbfKQwFVrTq0Mzc4TRItl27YtO9O313HGNJqJKfXSSeWDuOfN/NOHn44xnsgGewAz3uD6OkoJLCrDIKs8tRKCWs7S2wttfg6GGN2uIOV+acW19dGcA9svomUXWbbxgNTNO+HbF2hkkrwdoFFo6pLgV9hSRJ3DMhHI8ge7b9cKbOJoBHhIxAJ+tYf3l9nVxfMI/ynTuJ941GVurxixRv6nfL/eVpUFrGG+cjWW9nxWXLYrTzBiEX/r37V1JI2Dlb4hvhTFR3HyK7eBMQ7YKbv92dJ/5Tv1Uv6fSMgclrwdLByM+o/mjayR+qdwE/uApUltXDQLl/l3lWqZUMeiIGC2s162edoKzI+AXgQp1CiXKJYnXcaqNfWzCPqvRUpPQy8p1a4BxqgUrd8NeEm4tlWBiOY0YTsOU0bap8eMs7AENVHqVzB9ZYsuWuyTIc/A5+ewR8O8CkVQ26dMPtEMkfwDkYHlwNsh4WDL/mRWXjYMHgJ2MoL65i07cn0VcZv9rgyJCRnM8/z9ncuqt0LZhO1qLZlNj6opScaNG+buaLmhLXZ59FUqt55YgXxZTwkHd35NJsDLO7w/lNte+gorB6mGfDNAjpVz0HaNn4D9sRyf8vbuHV7/ba4uo3gKs+VroH2NPnwUjSLxWyfaHxK4AOChqEWqFmVdwqo15XMI+irVtJ8G2BjExwjPHnipoatbs7Lg8/jGL7fl7SDCHF4ixDLR4hyeACv94HW96o3oh1N5IPwrf3wNm10PcdGL8YNOatv3Q29yy7U3fXeT8i+V/NqwVMXAllubBgaPURbVeEtvegw7Agzh/I4MjGBKN262DhQG//3qyPX49Wb5qzBYS6oc/LRU4qJMM9BmsfCWv7xjlZaGouD01F5eFB92XniXQMp9JvBwPKXuKUz72wdwbMaAtHFoDuNn9/ss5V1/r6vl/10u8p66vX8ivMlxJ1Bh3fxn7LA+sf4LPDn2GQ6/ZMA5H8r+fbFiaugJJs+PHaN4B2gwMJ6+jBgTXxN11adrdGhoyksLKQHSk7jHpdwbTyls+jUuOEUuFP87b+5g6n0VBYW+M+bRqVp8/wfwW90MqleIZvYmT8KC4PWFC9aGPtczCjDfzxAVzeUV3N92p58XBoHvxyP8zuXN2m17/hmUNmP3g9vjCeBzc+yDfHv6FfQD9+HPgjCqlu03PT3OR1O5IPws+jwNajetbfwQeoPtR59VfHyEooZvgLrfAOqeFsz7ugN+jpv7w/zV2bM6P3DKNcUzC9C6Pv4XxpGElB4xn/VkecvUUJZ2ORZZnECRPRJiRw9OuH+OD0l1gUjsC2sg/rn+2GTfIO2P05JO0D2QBKDVg5g15b/Ud7pWS7gz9EjYRxJm3zAAAgAElEQVSuL4CNi3mfFLD+8nre3fcuGqWGNzq9wcDAgbW6ntjkVVt+HaqXgS4cAz8Mqn4DcApAqVYw6IkYfvv4CBtmn2DMy21x8qz9L7hSoWRw8GAWnllIfkU+TpaNe6VBYyRXVqK7mElih7GoHPQ4eVmbO6RGRZIkPN/4N/FjxtJzSxb7OvVmh7SelHhf3ljtxOfj+iCF9q2ewE3aX13Ftzy/+k1AqQFHfwjpCy4h9WLTllav5eNDH7Pk/BLauLfhk3s+wd3adHNEYtjnn/h1qF4FVFFYvRnsyjJQK1sNw55tiUIhse6bWKMtAR0aPBSdrGNLQs3nDgv1W8nmpegNlsjqMEJaeYpCbnXAsnlzHMeNI/+XX3jTcyoe1u64N1vOytg4Fh9KvtLIAcIGQL//qz5mcchnMPBD6PQkuIbWi8SfWZrJlE1TWHJ+CVOjpjJvwDyTJn4Qyf/WfNrAlHWgK6/+BJBVvRzTwc2aIU+1pKxQy/qZsVRV1r44W5hTGCGOIay7vK7W1xJML3vVErJdwlGgJqKNr7nDabTcXngehY0Npf/9go97fES5IRef0JW8veYkp1ILzR3eLcVmx3L/+vu5VHCJL3t+yUvtXkKtuI1aQ0Ymkv/t8IyBKRsAqfoNIPUoAB5B9vR/JIrspGI2f3cKw12Wmf2LJEkMDR7K8ezjJBcnGyFwwVRkWaby5CXi/GOQ1Xq8QhrvzlBzUzk54f7SS5QdPEjgvkRe7fAqRYoT2Hht5OlfjlJUUXfVeGtrVdwqpm6aipXKikWDF9EnoI/ZYhHJ/3a5R8BDm6prAi0YDgnVRzAGtXSjx/hwEk/l8uei87XeAzAkuLrCqCj30LBUHv4DuRjKbaPwjLBDKQq51SnHe8di1bIlmR99zFivgUyInIDOdgcZhu28tCTW6Kd+1ZbOoOOjgx/x5p43aevRll+H/EqIk3nLfItX6J1wDoKHNleXhFg4Gi5uBSC6hw/tBgdydm86B9fF16oLTxtP2nu2Z/3l9UbfTCbUndxlCyix9UOFA1FtAswdTqMnKRR4vvsO+sJCsj7/gmntptHNpxsWnqvZnrSbL7bV/XGst6uwspAntz3JwrMLmRg5kdl9Z+NgYf5PhiL534ZyrZ7UgnISckop0rghT9kArmHV1UBPV5/E1WFYEBFdvDi8PoFTO1NvccV/NjR4KAlFCZzOPW2M8AUTKDlwjAsB0cjIBEabf/lgU2AZEYHzxIkULF1K1YlTfNLjE0Icm2Hnv5BZ+7awNjbt1hepYxfyLzB+/XiOZB7h/7r8H9M7TEelqB+LLOtHFPVIYVkVey/lcDAhj4PxeVzKLqHiuno+GpWCMIdXmWH5XwKXPURxQS72XR+h54Rwyou07Pz1PNb2GoJb/cPxcP+gb0BfPtj/AesuryPaNdoYT0uoQ7qEs+gzq8iJiMbBR3H7NeOFWnN99lmKNm8m7Y03CPrtN+b2n8vUTQ+RGLCAV9apCHJ9gGgf89xlr7u8jnf3voudxo75A+bTyr3Vbf1c4dp1yDodjqNG1ml8YpMX1ZN1x5ILWLQ/iXUn0qjUGbBUK2jj70SUtz3ONhY426hRKRTklWrJKa3kcnYpsZfT+Fj/KT2VsSx1egy3AS/TOcCZtV8dJze1hBHPt8LrLjeBvbj9RY5nH2fb2G0oFaIqZH2W++lLpCzYza6u/6H90CA6DhX1+02pZNdukh99FJdHH8H9X/8iqyyLyRunklKUiUXOE6x65AF8nUy350Kr1/LZ4c/45dwvtHFvw2c9P8PVyvW2frb81GkSJ0zAqkUL/Bf8iHQX5SbEJq/bdDQpn/fWneFYUgE2GiVj2/oyuo0PMT6OaFT//D9eb5A5l9KZ86ufYFzuXGYuzOBtu6m8cE8zbDdWsX7WCUa/3BZnrzvfBDYoaBDbkrZxKPMQnbw63e3TE0ygYMcukryjkVAQ3EIUcjM12+7dcBg7htzv52PXty/uLVvyw8Dvmbh+MpnM4v5FJax76Ckcrev+E9mlgktM3zmd8/nnmRg58Y6Wcepyc0l59lmULs74fPXlXSX+O9Fkx/zTC8t5YfExRs/aS2p+Oe+NiOLAv/vywagY2gY43zLxAygVElH+boQ/vRR9m8k8rVrDdMNcpq0+wTonPQYJ1s2IpbSw8o7j6+HbAxu1DRvjN97N0xNMRC4tpDKhmESfKBQ2Blz9bM0dUpPkMX06Kg8P0l57HUNlJZ42niwetohmDmEU2n3PyEXvUq699WHvd8sgG1hybgn3rbuPrLIsvun9DdM7TL/txC9XVZH6/Avo8/LwnTEDlXPdHwDUJJP/xpPp9P98JxtOZfBMrxC2T+vJpM6B2Frc5QchhRLlsK+g6wsMqdzIn81+IamshO+lEooKK1k7IxZt+Z298CxVlvT2683WxK2i0mc9Vrb+R2SdCr1FJEEt3MSuXjNR2tnh9d57aC9fJvvzLwBwtXJlyYgFtHbqQ65mDQN+fZiMkmyj93069zSTNk7i/QPv086jHStGrLjjM7kzP/yQssOH8Xr/PayiooweY02aVPKv1Ol5Z81pnlx0lGB3W7a9eA/TBoRjc7dJ/2qSBP3ehT5v45+6gZ2BPzCkqwe/WVaQnVLCqpmx6HV3tglsUNAgirXF7EndU/v4hDqRt2kdeU7BqGQrwlp6mzucJs22W1ecHhhP3oIFFP/5JwAWSgsWDPuCnq5TyTPEMui3Yay4sMooy6gzSzN5d9+7jF83npTiFN7r+h6z+s667fH9v+T99DP5v/yK80MP4TBsWK3jul1NJvnnlFQy7tv9/Lg3gUe6BbHs8c74u9TBJFD3l2Dwp6gubuKtwrd5fVIoO+x1ZMcVsnh27B296Dp5d8LRwlEM/dRXBgNlp5Kql3gqDPhGiGJ85uY+fToW4eGkv/Y6VZnVZdclSWLGkJd40P9rKspceXvfm0zZNIXtSdvRG+68LEtycTLv7nuXQSsGsfLiSiZETmDdqHWMDBl5x2WYi//4g8wPP8SuX1/cp/3rjmOpjSYx4ZuSX8aD3x8krbCcORPbMjDas2477PBo9U7gVU/ST/80kS/9zCffnCHsdD4/zjvB1Edb3tZl1Ao1/QP6s/byWsqqyrBWiyqR9Yn20AYMRQqKHKLwCLJEY9kkfp3qNYWFBT5ffE78mLGkvfwK/j/MR1JWr5Z7pU8P7JRefL7/J06zk+eynsPX1pcxYWPo5NWJCOeIGtfgy7JMSkkKfyb/yR9Jf3A06yhKScmokFFMjZ6Kr93d1XEqP3Wa1H9NwzI6Gu+PP67zCd7rNfqlnhczi5n0/UHKtDrmT2lPu8C6n0j5n9MrYfnD4NOW4rGL+eyTU7jl6qnq4MzzU1ve1vjw4YzDTN08lY+6f8Tg4MEmCFq4XTn/nkDS+gT2dXqPLmOb0bqv2NlbXxSsWEn666/j+tSTuD333DWPzdwexyebz9AqIhkbt32cyIkFwEZtQ7hTOPYae2w0NsiyTHJxMolFiRRpiwAIcQyht39v7gu/r1ZVOLVJSSRMmICkVhO0ZAkqt7vbE1QTsdQTiMsqYdy3+1ApFSx5vDORXiY+lDlqFEhKWD4Vu2X38uory5j54WlUB3P5j/I4rz/Y6pZvAG082uBh7cHG+I0i+dczhQdiifftAkBQjPF+eYXacxg1krJDh8iZNRtNs2Y4DBnyv8ee7hWCg5WaN1craVfRmtXjgrhQFMvhjMPEFcSRUZZBaWEpBtmAr50vg4IGEeQQRHef7vjb1/50tqrMTJKmPgRVOvx/+MGoif9ONNrkn1lUweT5B1EqJJY+3pkgVzOdqNR8OIz7GZY+iOXScTz18lK++88p1Pvz+NzhDP8a+c8z+wpJQf/A/vx67leKtEXYa0z8BibUyJB6lso0HaldorBwlHFwtzJ3SMJVJEnC89130CYnkf7a66i9vbFu3fp/j0/sFICTtYYXlhzjiQUX+O7B7rU+Qet26PLzSXr4YfQFBfj/+CMWIeYr7tYoJ3wLy6uYPP8gBWVafpzawXyJ/y8Rg2HcAkg/jtWqB5j4fCRWSgVFW9KZtfXiLX98UOAgdAYdfyT9YYJghdtRumY+BtSgCqdZCw+xxLMeUmg01WvmPT1JefoZtCkp1zw+pIUXC6Z2IKu4kmEzdvPn+aw6jUdfUEDyo49RlZSM76xZWMWYt3RLo0v+FVV6HvvpMJeyS/h2Ujuz1fW4QcQQGPM9pBzC+fepDHskDDeDgvNrEvj1QOI//mi0azQ+tj5sSthkomCFWyn4cydZLqEo0RDawsvc4Qg3oXJywm/OHGSdjuSHH6Eq89oE3yXElbXPdMPb0YqpPx5i5vY49HVQDlqXnU3ipAepvHABn6+/wqZjB6P3cacaXfLPLq4kJb+cT+9tSbfQO1tvW+eiRsLouZC0l8BTz9NlVBBhVUo2/Hqe/Zdzb/pjkiQxIHAAB9IOUFBRYMKAhZrI5YWUxuUTFxCFrDTgE3Z39ZsE07AIDsJvzhx02dkkTZlCVda1bwB+ztaseKoLw1p488nm84yfu5/kvDKj9V+VmkrCxIloU1Px+3YOdj17Gu3atdHokr+fszXbXrqHEa18zB1KzWLGwtAvIG4rrYs/ILi9O50qVPz3u6P/+IIbGDgQnaxjW9I2EwYr1ET752LkUiWldlG4hVij0ojCe/WddZvW+H03t3qydcpUdNnX7vS11qj46v5WfHZvS86mFzHwy538ciCp1ofClJ8+TcIDE9DnF+D//TxsOneu1fWMySjJX5KkgZIknZckKU6SpFdreNxCkqQlVx4/IElSoDH6vRmr+v7L2HYK9HkL6dQy+rkvxMHHhnsKFLww9xAllTWXgYhwjiDAPkAM/dQDRZtXU2blhkZ2J7KVn7nDEW6Tddu2+H87h6r0dBImTqTy0qVrHpckiTFtfdn0Yg9a+jny+sqTjJy1h0MJeXfVX+G69SQ+MAEUCgJ+WnDNhHN9UOvkL0mSEpgJDAKaA+MlSWp+XbOHgXxZlkOAL4CPattvg9ftJej8DKojsxnR/gDWVmpaJVXx2uKadwH/NfRzKOMQOeU5ZghYAMCgp/h4HHH+1au0AqLr2dCi8I+s27fH//vvMZSUkjDuPoq3b7+hjY+jFQsf7sgX97Ukq6iSe+fs4+lFRzmbXnRbfchaLVmffUbatOoNXEHLl2EZEWHsp1Jrxrjz7wDEybJ8WZZlLbAYGHFdmxHAgit/Xw70kZr68ghJgn7vQfRY7Pa/ybCBhTjLChSH8/jlQFKNPzIwcCAG2cC2RDH0Yy7687uoyFSQ4R6F2lnGwU0s8WxorNu0Jmj5MjQBAaQ89TTZM77BoL22eKJCITGqtS9/TLuH5/uEsv18FoO+2sWUHw6y91LOTcu0lMfGEj9mLLnfzcNx3DgCfpiPyqV+nuxmjOTvAyRf9XXKle/V2EaWZR1QCNzwf0SSpMckSTosSdLh7GzjV9+rdxQKGDETfDvgffAROve2JKxKyapl5zifUXxD81CnUJo5NBO1fsyodMMiDJIGhTKUsJZilU9DpfbyImDRQuyHDSVn5kzihw2nZOfOG9pZa1S82C+Mva/2Zlr/ME6mFPLAdwfo/vF2Ptl8jnMZRciyjC47m4z33ifh/vHoi4vxnTUTr/97F0lzZ2cIVOkNnEot5MA/LAAxllqXd5Ak6V5ggCzLj1z5ehLQQZblZ69qc/pKm5QrX1+60uamz9CUJ3mZXUk2zOuNXFXJKsUCki+WsdNfyc/Tut8wfzH7+Gxmx85m273barW9XLg7KaNbcD43jHORTzH8uVb4NTdhuRChTpTs2kXmB/9Bm5CATdfqyqC2PXogqW+sxV9RpWf9iXRWx6axJy4Hz8JMJiTupnv8IZQGHRWDR+H6wvP4+LihVNx8cKNMqyOjsILE3DIuZZdwOaeU02lFnE0vQqszEOVtz/rnut/V8zFleYcU4OpZL1/g+pOT/2qTIkmSCnAA7m4WpTGydYMHliJ9359B9u/wk+2/aZ2i5YNVp3h/3LVF4PoH9mdW7Cy2Jm5lQuQEMwXcNMn5iZQkVHAxKhpZZcA7VCzxbAxsu3fHZs1q8n5eSO6PP5Dy9DMoXVywHzAAi4hwLIKDUXt7Y6ioRC4tpX9hFl0z9lN0fC/6y5fQqdTsCenIT35dSVO7wcxDaJQKHK3V2FupsbNUYZChskpPRZWe3BItxdct7HC0VhPuYceULoHE+DjQyq/uX1vGSP6HgFBJkoKAVOB+4IHr2qwBJgP7gLHAH3J9rShnLu6RMOpbLBePZ1jUVn472Iv0XZnsaZNN15C/a380c2xGiGMIWxK2iORvYpW/L8JQpqTcNgrfUBuU6ka3UrrJkjQaXB5+COfJD1KyaxeFK1ZSsGIFckXFTdtbt2uLzZhROIwYQYyrK2OKK7mcXUJ8TikJuWUUlGkpqqiiuEKHQpKwsLPAUq3E2UaDh70lHvYW+DtbE+xmi7NN3R8xeb1aJ39ZlnWSJD0DbAaUwHxZlk9LkvR/wGFZltcA3wM/S5IUR/Ud//217bdRihgMPV7Ba+fHtG/VEumYM3MWnKT1mz2w1vz9TzUgcACzjs8iszQTDxsPMwbctBRv20SZtQca2YUIscSzUZJUKux69cKuVy9kg4GqtHS08ZepyshAYWmFwsYGpaMDls2bo7C0vOZn3ewscLOzoGNw/ZzgvZ5RCrvJsrwB2HDd99666u8VwL3G6KvR6/kqpB2j/aUnueixmNaZ8PnKM7xxX4v/Nekf2J+Zx2eyLWmbuPs3lcoSSs6kcdG/uvhXQHTD+AUX7p6kUKDx9UHjW083jNaS+Nxa3yiUMHouCgcvhtu8g0YlUbYzk8Pxf8+NBzsEE+oUyuaEzWYMtGnRn9xIebaKLPfmqF3B3kUs8RQaNpH86yNrZxg7H/uKM/QN3YGPXsmP805Qpf/7DOABAQM4lnWMzNJMMwbadJRuXIpeYYlCEUK4WOIpNAIi+ddXvu2g1+tE5H+Jh3chYbkG5q+78L+H+wf2B2Br4lZzRdh0GAwUHzxBhls4ClQ0a1HHx4AKggmI5F+fdX0BgnowlFdAJZO+NYW0K8XfghyCCHcKF7V+TEBOO0pxiswl/yhktQGvZvWkTLgg1IJI/vWZQgmj5mJpoWOAzxLcdArmzj3+v4f7B/YnNjuWjNIMMwbZ+FX+sRhDuZIKmyg8w21RqsSvjdDwiVdxfWfvBUO/ILxyKU4u6TgnlPP7vupqGv0DxNCPKZT8uZ1SG2/UshPNW9f+DFdBqA9E8m8IokZC1ChGqf+NQWFg/+KLaLU6Ah0CCXcKZ0vCFnNH2HgVpVF8MZ+LAdVH7gVEiSWeQuMgkn9DMfhTrKyV9PZcimMl/LzwNFA99HM8+7gY+qkj+mOrKc/RkOUahcYdbBwtzB2SIBiFSP4NhY0rDP2cGP1S1NYZFB/KIT29WAz91LHSravQKa1RKYKIbNM4N/sITZNRdvgK1zJotVSeO0fFuXPoCwsxFBVjKCur3hru4IDSyQnLyAgswsKQlHdw6ljzEUjRo7jv1Pv8WPY1y+ac4Nl3uvxv6GdS80l196SaIm0ZJccuku7eGQklIWKJp9CIiORvJNqEBIo2baJk+59UnDmDXFX194NqNQprawylpaD7u5qfwsYGq1atsOvXD/shg1Ha2d26o4H/xSGuA82dt3Ehsz+Hd6XSP7A/M47NIKM0A08bkaCMRb60naJUFZdbRiNb6HEPtDd3SIJgNCL514JBq6Vw1SryFy+m8sxZAKxatsTpwUlYtWiJZVRzVC4uSJaWSJKELMsYSkvRZWdTceoUZUePUnbgIBnvvEPmhx9i178/zpMnYxUddfNO7Tyh79v0WTeNk6pO7F1+kcH/7ssMZrA1cau4+zeiiu3LMFSo0Fo1xz/SAcU/1GcXhIZGJP+7YCgtJX/xYvJ+XIAuOxuL5pF4vPYqdgMGoPa8+Z23JEkobW1R2tpiERSEw7BhyLJMxanTFK5cQeHadRStXYv90KG4vfA8Gl/fmi/UdiqK2F8Zbfia9dlvcH5zBeFO4WxO2CySv7EYDJTs2UehXRAq7IhqI5Z4Co2LmPC9A7LBQOHq1VwaOIisTz7FIjQE/x/mE/TbbzhPnvyPif9mJEnCKiYaz7feImT7H7g88TjF27ZxedBgsr/++trho78oFDD0SwJUx7G0PE3Cvgz62w8VG76MKSOW4kQdcQHRyMj4NxdLPIXGRST/21Rx5gyJ4x8gbfqrqDw9CfjlF/znz8emc2eMdRa90tYW9xdeoNnmTdgNGkjOrNkkTJiINjHxxsae0Uidn2K8/Sfo0KPa0wxkxJp/I9EdWUVFjpo8l2isfCQsbW880k8QGjKR/G9B1unImfMt8ePuQ5uaitd//kPgksVYt2ldZ32qPTzw+fhjfL78Am1iIpdHjaZwzZobG/Z4BUtbDeG2aylPrqK7dogo82wkpX9sRKtxRCX5E9MuwNzhCILRieT/D7TJySROepDsL7/Erl9fmq1bi+PoUUgK0/xvsx84kODVq7CKiiLtlelkfz2Da06/tLRH0fdtBlj/TIWymIhzvTiTeZbUklSTxNdoFaVRciaTFM/qifdmLcRpaULjI5L/TZTs2kX8mLFUxsXh/ckn+Hz+OUpH0x/Yrfb0xP/7eTiMHk3OrFmkvTIdg1b7d4NWE8CrBQMdZqIsU9MivRdbE8SGr9qQz22gKMOSRJ8YZFsdzt425g5JEIxOJP/ryLJMztzvSH7scdReXgStXIHDsKFGG9e/G5JGg9cH7+P2wgsUrV1L8uOPY/jrYGmFAsXgj4jSHMCgSaBNan+2n91ltlgbg4rtq9BXWaC3CCe4hatZ/+0Foa6I5H8Vg1ZL2rSXyf78c+wHDSLw119uvtzSxCRJwvWJx/H674eU7T9AyrPP/f0JwL8T+uajuc/+MxSyEocTIaQUp5g34IZKW0rxoZPkOYWixIKotmK8X2icRPK/Ql9cTPKjj1G0fj1uL72E92eforC2NndYN3AcORKv9/6P0l27SH3xpf8tBVX2ewcXdSZK68OEZ3dk08E/zRtoQ3XpD4pT1Vzyi8Gg1OMTZvqhPkEwBZH8garMLBInTqLsyBG8P/4I18cerdcf9R3HjsXjzTco+f130qa/imwwgFMAcodHmWrzNRWqUrK2SddODgu3pergKirz1RQ5RuMQrEalvoPaS4LQgDT55K9NSSXxgQeoSk7G79s5OAwfbu6QbovzhAm4vfQSRRs2kP311wCo7nkZhUZBpfNmHPK9OLjvrJmjbGAMekp27KDUxhsVzrTu0MzcEQlCnWnSyV+blETipEnoS0rwX7AA265dzR3SHXF59BEcxo4hd863FKxaBdbOKLr/iymqxeRbZXB4TRIGvcHcYTYcKYcoSdCT6B0DQHCMu5kDEoS602STf+XleBInTkIuLyfgxx+wiok2d0h3TJIkvN56C+uOHUl/8y3KDh9G3eUJ7NUuJHmuhQINZ/akmzvMBsNwYg3FmRake8aAa5U4uEVo1Jpk8tcmJ5M0eTKyXo//TwuwjIw0d0h3TdJo8P36KzQ+PqQ8+xxVuYWo+v6bHqqdpNtdYt+ai2grdLe+kEDZ9g1olQ4olEFEt68fq7wEoa40ueRflZlJ0pSpyFotAT/+gGVYmLlDqjWlgwO+s2ZiqKwk9V/TsIi5l/YGF/b7r0ZbYiD292Rzh1j/5Vyk+Hwe6e4tAIgWJR2ERq5JJX9dXh5JDz2MvqAAv3nzsAgNNXdIRmMRHIzXu+9SfuQI2d/MxKfPm/ipz5PidJqjW5IoK9Le+iJNmHxuPUVpliT6tqDKplLs6hUavSaT/A2lpSQ/+hhVKSn4zZndIMf4b8Vh2FAc77uP3HnfIxe40LnKll0BK9Fp9RzdVENlUOF/KnetRltpTZVlGAExTvV6qa8gGEOTSP6yTkfKiy9Sce4cPl99iXX79uYOqc54vP4aFpGRpL32Or3DnqTYMosC17Oc3JlCcV6FucOrn4ozKT56kTzn5ihQ06FLuLkjEoQ61+iTvyzLZLz7LqU7d+H59lvY9exp7pDqlMLCAp/PP0OurESz7BAxlSoOeP0CMhxaH2/u8Oqn8+spTrUg3rcFVWotXs0czB2RINS5Rp/8c+fMoWDZclyeeByncePMHY5JWAQF4f7yNEp37+b+9BbE2xShcIvj3N508jNKzR1evVN1cAXl+ZYU20djF6JEoWz0vxaC0LiTf9GGDWR/9TUOI4bj9vzz5g7HpJweeACbLl0IWXUS31yZDPs5KDUKDq4Vd//XKM+nZN8xChxCUUjWdOnacJf9CsKdaLTJv/zkSdJeex2rtm3xfO+9JjeBJ0kSXh/+B0mjYdpGC3bZleHinkjckSyyk4rNHV79cWELxSkakrxboFfoCInxMndEgmASjTL5V2VmkvLU06hcXPCd8TUKjcbcIZmF2sMDzzffxDu5jDaxCmz5BAtrJQfXibv/v+iPr6Q405Ic15bgo0VtIQq5CU1Do0v+hvJyUp56GkNpKb6zZ6NydjZ3SGZlP3QIlt26Mn6HgROU4umVQMKJHDLiC80dmvlpyyjdvYdi20AUCic6dWs8+z4E4VYaXfLX5+djKC/H+9NPsQxv+Lt3a0uSJHzffRelQon/HitaFf8HSxsVB9dcNndo5nfpd4qTFKR4tkEv6WkpqngKTUijS/5qb2+CV6/Crncvc4dSb6h9fKh4aDQtL8ukZJbh759A8tl80i7mmzs0s5JPraEw3YpMj1ZoPUqwsFKZOyRBMJlaJX9JkpwlSdoqSdLFK/91ukk7vSRJx6/8WVObPm8rLrW6rrtocFo9+Trx3krUB22JyfoQa3s1+1dfbroHvugqKdu1lWKNPyhdaNVZ1PIRmpba3vm/Cvwuy3Io8PuVr2tSLsv/396dR0dZ5/kef39TVVkJCWRfISEBWcIakKVJUJYB3MD6L+EAABXeSURBVBC0tV0accEZdWbU6VanOfY4ffW21+5rq2jrYE8r2vQA2iCIIE0U2bcQSCTsJBASsofsVLb63T9Ie+wre0ieJPV9nZNTVcmT+n1+cM4nv3rqqecxw1u/usbVUroZT09vjs6fjJcTmjLrSIg/TeHxKvIPuenq/8TX1OS2UBQ2EhcuJozvfqf7UOpS2lr+dwCLW+8vBma18flUO5ow6QG+HCXUHPcj/sgr9Ojlxa7P3XP1b75dSVWBLwXhw6kNrsTH3z2PCFPuq63lH2aMKQRovb3YpY+8RSRdRHaKiP6BsMiosFFsnh5JjZ+Nxl1OBiYWUJxbzakD5VZH61hNTpzb11Mt0RhHKImj9Ipdyv1ctvxFJE1EDlzg646rGCfWGJMM3Ae8ISIXPKxCROa3/pFILy0tvYqnV1fCQzyYPOg2Ft8MzgpPwrf8ip7B3uz+PNe9Vv8nvqb6hIui0BG4cPEPNydbnUipDnfZ8jfGTDHGDLnA1yqgWEQiAFpvSy7yHGdab3OAb4ARF9lukTEm2RiTHBISco1TUpdyW/xtbBkM+TE9ce5tIik+j9K8GnL3l1kdrcOY7JVU5vuSHzmK6l7l+Af4Wh1JqQ7X1t0+q4G5rffnAqv+/w1EpJeIeLXeDwYmAAfbOK66RvGB8QwKHsyy2cG0NHnQc/UrBIb6nN/373KD1X+TE+e29VRJH1yOMGJGXvAANaW6vbaW/6vAVBE5BkxtfYyIJIvIH1q3GQiki0gmsBF41Rij5W+h2+JvY5dvHseGxNFwpImhUSeoOFPH8YwLvnDrXk58RXWOoTAsmRZp5o5p46xOpJQl2lT+xphyY8xkY0xi621F6/fTjTGPtt7fboxJMsYMa7397+sRXF27GXEzsImNbfeNAgf4LPs/9I7wZffnubhaXFbHa1fmwAqq8v0oiEimPKiYgAB/qyMpZYlu9wlfdXlBPkGMjxxPRvNOskaOpOlMM0P8MqksrufonmKr47Wfxjqc29dTZu+PsQfSZ4R7n/dJuTctfzd1W7/bKKovIvf+OXgEuPBctpDgKD/2rMmlpbuu/g+vpSYXikKTafJo4K7pKVYnUsoyWv5u6qaYm/B3+FPhuYfNyeNx1bQwsGEz1WVOjuwosjpeuzCZSzlb0IuisBGUhBTQ0093+Sj3peXvprzt3syIm8HXp9Oouf0JHFEtOFZ9QGi0D3vW5tLS1M1W/7UlnNu9lWLHQIzNj9gRQVYnUspSWv5u7M7EO2loaSAo8gRrhk6EFhcJRWuprWgge+sZq+NdXwdWUJ3rRWH4GJz2Wu6Zpmd9Ve5Ny9+NDQ4aTEJgAn/N+5zalH/CJ7EJr7RPCY9ykL7uJE0NLVZHvG7MvqWUFYZQFpxEYWgevXwDrY6klKW0/N2YiDArYRZZZVncNNKHTwfchN2rhfhD/8O56kayNp62OuL1UXaM2oyDFAaOBnGQODbc6kRKWU7L383dEn8LNrGxq3Q9Z4bOw29wE94ZG4kKN+z7ax4N9U1WR2y7rOVUnfQlP3I8pb6nuX/SdKsTKWU5LX83F+wTTEp0CqtPrObBlBv4c5/JePZsInbXIhrqm9m3Ic/qiG3jctGyZylnqvpT7xdNWXQxPTx7WJ1KKctp+StmJcyi3FnOWTI53Pcn9BjRiE9uFrG968j8Op/66karI167k5upyS7lTNg4WmhiwqRBVidSqlPQ8ldMjJ5IiE8Inxz9hAcnJfFx8DT8IpxEffM2LU0tpK89aXXEa5fxERWngygMH0NO0AHuHDrF6kRKdQpa/gqHh4PZibPZVrCNhMhGdoTcjd/wZnzO5tHXu4jszQVUltRbHfPq1VfQuPsLTrWMwmXz4VxcHd52b6tTKdUpaPkrAOYkzkFEWHFsBfNuTuKPvtMJ7FdH5NcL8bDBrlU5Vke8elnLqDzm4EzkOGodZdye+iOrEynVaWj5KwAiekQwMWoiK46tYMqgYL4JvBPvJIOPqSau8VuO7y2h+GS11TGvnDGYPYvJLxlAZeAADoVnMK3fRKtTKdVpaPmr7/x4wI8pd5azOf8b5k5K4n2P6QQPPEvE1g/w8oIdK493ncs9FuylZn8up4JScdGI90BPHB4Oq1Mp1Wlo+avvTIicQIRfBJ8c/YRZI6JY73cH9v42fANaiD+TRsGRyq5zsfeMxZScDKcw/EaOhKTz2PjZVidSqlPR8lffsXnYmJM4h52FOymsP839KYN5q+U2woYUEXZgNT28mtj+l+Od/5TPzioat6/gmC0V4+EgJ/owI8OGWZ1KqU5Fy1/9ndmJs7GLnaWHl3LvmBjWec2kMcYf/7524rP/zNmierI3F1gd89IyPqLiqJ2CyFSK/Y4waegERMTqVEp1Klr+6u+E+IYwre80Vh5fiQsnc1MH8ppzFmEDTxNUmE6Io4Ldn+firOukp31oacZs/y+OVqfQ4N2LjKgtzBt2l9WplOp0tPzVDzww8AHqmupYdWIVD47rw1deUznbO4ygYQ7itr9L47lmdq/JtTrmhR1eQ1VWGXkhKZzzKMcR60OIb4jVqZTqdLT81Q8khSQxPGQ4Sw4twcsuPJySyMv1swmOO0mgZw0xdQc4sCmfijN1Vkf9AbPjHY6duZGqgH7sjd3Iw8PvtjqSUp2Slr+6oPsH3c/pmtNsKdjCT8f1YbvXjzjtl0DoiFpi932M3cPF5mVHO9ehn/l7qdudydHgGTRRy/Gwb5keN9nqVEp1Slr+6oKmxE4h3C+cPx38E35edh5NTeD5mnvpGVJAYLw//XJWUXDkLEd3daLr/e78PUfyR1MZ2J89MX8lNXYKDpse26/UhWj5qwuye9i5d8C97CraxZGKI/x0XF+O+Q4j3WcC4f2PEnHqG3rbKtn66XHO1XaCs35W5FK/ZQ2He95Oi6kjO3IbTyQ/aHUqpTotLX91UXf1vwsfuw8fZn9IDy87T96UwM+rZuPwdxKcGk3CzndoqGti+4oTVkeFzb/l6MmRVAYmsidyI/EBQ4kPiLc6lVKdlpa/uqgArwDu7n8363LXcbrmNA+MjaUpIJ7VnrcQFLSD3kFC3/JtHN5eSMGRs9YFLT+Bc/Nysn1mgauOb2M38uSoudblUaoL0PJXlzR38Fw8xIMPDnyAl93Gs1P788vKmbT49CTiR83EZn+Kn83J1x8fotHZbE3ITa+RlZNKZWAiu0J34ucZxKSYVGuyKNVFaPmrSwr1DWVWwiw+O/4ZxXXFzBoRRURYBK/zU3xd+wienMSA9N9TU+5ky/JjHR+w9ChVX60lK/BehGr2J6zhgUE/weZh6/gsSnUhWv7qsh4e8jAu4+Kjgx9h8xCenzGA96rHUtQrmdDgzYT41RFXsZ3D2ws5kVHSodnMN79mR9E9NHj3ZnX0djzEzn2D9BO9Sl2Olr+6rGj/aGbGzeSTo59w1nmWmwaEMjY+iPmVDyLiJHJmb2IPLCPQVsXGJYepPdvQMcEKMihYn0FO0DQ8HKXkR6UxNXYGAV4BHTO+Ul2Ylr+6Io8kPYKz2cni7MWICP95+xCyG0JJC3kQ39o0Qu+ezA3b36DZ2UTaB9ntf+ZPVwuuz55ma/1j2Ewzf4jajXg0808jH27fcZXqJrT81RXpF9iPGXEzWHJoCSX1JQwI92fuuL48eSoFZ6/+BPuuI6h/GDccW0rB0Uq2tvf+//Q/snNHHOU9B3GudxUNoZtIjZqsh3cqdYW0/NUVe2rEUzSbZt7NfBeAp6cm0tPPlxf4F2g4S1RKIxFnM4mvy+DApgIOtNepn2uKyfufJez3vZ9gKebd4N2IrYF/HvmP7TOeUt2Qlr+6YjH+Mdwz4B5WHltJTlUOPb0dvDBjIJ8V9mbfDf+Go3QjkY9Mok/6B4RJEVuWHm2X4//PrXqJtMon8GquZc/wIBxB2xgfkcKA3gOu+1hKdVda/uqqPJb0GF42LxZmLARg9ogoRvftxdzsYTjjp+Ff+gHhTzzAgM2/wc9Wz9p3syg8XnndxjcH1/DlV/E4HYHcMMGb1dVpiO0cT43QVb9SV0PLX12VIJ8gHhryEGl5aWSWZuLhIfzmrmE0t8AzzscwvkH0al5C8KxpJG1+GS+PBla/tZ/ThyvaPLar9DgbFqZzxmsUg32O8XKdJ15BWxgTPpakkKTrMDul3IeWv7pqcwfNJcg7iFd3vUqLq4W+wX4suGUg63KaWDf4t0h9OeGRWwkadQPD0l7Ez9HIF29ncTKr7JrHdJ2rZf0rqzhmUuhXvYMTM1M40fg52Gr55xFPXsfZKeUetPzVVfN1+PLc6Oc4UH6A5UeXA3D/jbGk9A/h2W02iqYvQsoOET2+lF43DmPo+l8Q4OXki3ez2LHyOC3NV3cYaFNDM+teWk6OcwSJJRsY/KsH+e2ub/EO3sLMuJkMDx3eHtNUqlvT8lfXZEbcDMZFjOPNjDcpqS9BRHhtzlC87DYe3R5I4y1v4pG/hZhJdfS+aSxJ654nPqCCjPV5/OW1vVQUXtlVwHL3l/Dn59ZxsqovA05/Rur/nc9/bitGgr7A027jmVHPtPNMleqe2lT+InK3iGSLiEtEki+x3XQROSIix0XkhbaMqToHEeHFsS/S7Grm1d2vAhAe4M3rPx5G9plqnjkyGNe0V5Bja4gakk3vmTfT97MXGVX7JdWldSz91S7WvJ1Jzr7SH3wgrNHZzMlvy/jinf2sfe8ApqKK5INvMeHXj/KXEhsbT+3Ao0cWjyY9QrhfuBXTV6rLs7fx9w8As4H/utgGImID3gGmAvnAHhFZbYw52MaxlcViesYwf+h8Fu5byKbTm0iNSWXywDBemH4Dv153mMSwaTx9Tx9kxeNEhBfht+BJbO8sYfTBbZTP/BdOnbKz7kA5dk8PfHt64uPviTFQmleDcRlsrgb65a5lgN9hopd/TKbTk/9Yto2gxHUE+kXy0OCHrP4nUKrLatPK3xhzyBhz5DKbjQGOG2NyjDGNwFLgjraMqzqPeYPnkRCYwC+3/5LS+lIA5qfEM2dkNG+kHePzxlHwyHrEZiMg50XinxhI0NhBRHz6EmPWPMHohq+I71VJL49KpKocU1pIfPk3DN//Jj/a9nNGT2im74r1lPkE8I9/yqB31FbOST4/H/0zvO3eFs9eqa6rrSv/KxEFnP7e43zgxg4YV3UAh83Baymvcd8X9/H8ludZNHURdg87/3v2EPIq6nh62X6a7hrK7Mc3w9bfYd+9iOjwZhqfnU5Vjp3q3Vvw37Hi757TJ6iRnmOi6fn477EPTqG+sZnHP96L0+MEth7ruSXuFqb2mWrRjJXqHi5b/iKSBlxox+oCY8yqKxhDLvA9c5Gx5gPzAWJjY6/gqVVnkNgrkQVjF/Dithd5L/M9nhrxFF52G398aDSPf7yXZ5dnUj5zII9N+18w9gnY/Bs8D64ixLuM4InQ0uiBcQE2HyQyCfv05yBhMgDF1U4eWbyHg0UlxCR9io8jnAU3LrB2wkp1A5ctf2PMlDaOkQ/EfO9xNHDmImMtAhYBJCcnX/APhOqcZiXMYm/xXhZlLWJk6EjGR43H39vBB/NG88yy/byy9hAFled4fvoN+Nz6Otz6OtSVISWHsJ+rgNBB0DsevncRluwzVTzyYTo1ziZSx21lX0Upb978If6e/hbOVKnuoSMO9dwDJIpInIh4AvcCqztgXNXBfnHjL+gX2I9nNz1LZmkmAF52Gwt/MpKHxvflw+0nmfL6Jr48UIQxBvyCIW4iDLoDghO/K/4aZxPvbDzOj9/bgQjMnZFHenkajw97XI/pV+o6EWOufYEtIncCC4EQoBLYb4z5BxGJBP5gjJnZut1M4A3ABvzRGPPK5Z47OTnZpKenX3M2ZY3iumLmrZ/HWedZ3p/2PkOCh3z3s1055fzH6mwOF9Uwum8vJg0IZWx8bxJC/SmrbaCw0smu3HI+3H6SGmczqf1DmDjqGG/uf5VpfabxWsprenlGpS5DRPYaYy566P1327Wl/NuTln/XVVRXxENfPkR1YzXvT3ufwUGDv/tZc4uLj3acYumePI4W117w96cPDufJmxLIbdjEgq0LSI1O5XeTfofD5uioKSjVZWn5K0udqT3DvC/nUeGs4PkxzzMncQ4if//ef3ltA3tOVnCyvJ6wnl5EBPjQN8iPEH8HH2Z/yFv73mJM+Bjenvw2XjYvi2aiVNei5a8sV1JfwoKtC9hZuJPJsZN5adxLBHoHXvJ38qrzWLB1AftL9zO1z1RenvAyvg7fDkqsVNen5a86BZdxsTh7MW/tewsfmw/T46Zze7/bGRYy7LtXAk2uJvYU7mFD3ga+yPkCu4edBTcuYGbczB+8WlBKXZqWv+pUDlccZnH2YtJOpeFscRLoFYiP3QeHh4OzDWepaazBx+7DzbE388zIZwjzC7M6slJdkpa/6pRqG2vZcGoDWWVZNLU00ehqxNfuS2p0KuMix+kpG5RqIy1/pZRyQ1da/no+f6WUckNa/kop5Ya0/JVSyg1p+SullBvS8ldKKTek5a+UUm5Iy18ppdyQlr9SSrmhTvshLxEpBU5ZneMaBANlVofoYDpn96Bz7hr6GGNCLrdRpy3/rkpE0q/k03Xdic7ZPeicuxfd7aOUUm5Iy18ppdyQlv/1t8jqABbQObsHnXM3ovv8lVLKDenKXyml3JCWfzsSkZ+JiBGRYKuztDcR+Y2IHBaRLBFZKSKXvlhvFyUi00XkiIgcF5EXrM7T3kQkRkQ2isghEckWkX+1OlNHERGbiOwTkTVWZ2kPWv7tRERigKlAntVZOsgGYIgxZihwFPh3i/NcdyJiA94BZgCDgJ+IyCBrU7W7ZuDfjDEDgbHAk24w57/5V+CQ1SHai5Z/+/kd8BzgFm+qGGP+aoxpbn24E4i2Mk87GQMcN8bkGGMagaXAHRZnalfGmEJjTEbr/RrOl2GUtanan4hEA7cAf7A6S3vR8m8HInI7UGCMybQ6i0UeBtZZHaIdRAGnv/c4Hzcowr8Rkb7ACGCXtUk6xBucX7y5rA7SXuxWB+iqRCQNCL/AjxYAvwCmdWyi9nepORtjVrVus4DzuwqWdGS2DiIX+J5bvLITkR7AX4CnjTHVVudpTyJyK1BijNkrIpOsztNetPyvkTFmyoW+LyJJQByQKSJwfvdHhoiMMcYUdWDE6+5ic/4bEZkL3ApMNt3zGOJ8IOZ7j6OBMxZl6TAi4uB88S8xxqywOk8HmADcLiIzAW+gp4j8yRjzgMW5ris9zr+dichJINkY09VODnVVRGQ68DqQaowptTpPexARO+ffzJ4MFAB7gPuMMdmWBmtHcn4FsxioMMY8bXWejta68v+ZMeZWq7Ncb7rPX10vbwP+wAYR2S8i71kd6HprfUP7KWA959/4XN6di7/VBOBB4ObW/9f9rSti1cXpyl8ppdyQrvyVUsoNafkrpZQb0vJXSik3pOWvlFJuSMtfKaXckJa/Ukq5IS1/pZRyQ1r+Sinlhv4fC0rLwvqcHgkAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "for i in range(y_samples.shape[0]):\n", + " plot(xt, y_samples[i,:,0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Gaussian process with a mean function\n", + "\n", + "TBA" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Variational sparse Gaussian process regression\n", + "\n", + "TBA" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}