Hierarchical Dirichlet Model

#1

Hello Edwardians and @dustin !

I am trying to implement a simplified Hierarchical Dirichlet Process Model from (Teh et. al 2006, https://www.stat.berkeley.edu/~aldous/206-Exch/Papers/hierarchical_dirichlet.pdf). I’m generally a PyMC3 user but was excited when I saw the DirichletProcess as a random variable. The model described in the paper is below:

42%20PM

My naive implementation abusing the DirichletProcess RV is below: `H = Dirichlet([0.5])
gamma = Gamma(5.0,1.0)

alpha_0 = Gamma(5.0,1.0)
G_0dp = DirichletProcess(gamma, H)

alpha_1 = Gamma(.1,.1)
G_1dp = DirichletProcess(alpha_0,G_0dp.base)

G_jdp = DirichletProcess(alpha_1,G_1dp.base,sample_shape=1000)
T = 1000

qH = Dirichlet(tf.Variable([0.5]))
qGamma = Gamma(tf.Variable(5.0),tf.Variable(1.0))
qalpha_0 = Gamma(tf.Variable(5.0),tf.Variable(1.0))
qG_0dp = DirichletProcess(qGamma,qH)
qalpha_1 = Gamma(tf.Variable(.1),tf.Variable(.1))
qG_1dp = DirichletProcess(qalpha_0,qG_0dp.base)
#qG_jdp = DirichletProcess(qalpha_1,qG_1dp.base,sample_shape=1000)

#xH = Empirical(tf.Variable())

latent_vars = {H: qH, gamma: qGamma,alpha_0: qalpha_0,G_0dp: qG_0dp, alpha_1: qalpha_1,G_1dp: qG_1dp}
data = {G_jdp: np.random.normal(6.0,1.5,[1000,1])}

inference = ed.KLqp(latent_vars,data=data)
inference.run(n_samples=5, n_iter=2500)`

where I get an error that ‘prob is not implemented.’ I know people were interested in the topic before (#232), but that issue was never resolved.

I’d really appreciate any pointers in designing such a solution! I also think other Edwardians would appreciate an HDP as well.

Thank you!
pindapuj

#2

Full Error

NotImplementedError Traceback (most recent call last)
/Users/Puja/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/distributions/distribution.py in _call_log_prob(self, value, name, **kwargs)
693 try:
–> 694 return math_ops.log(self._prob(value, **kwargs))
695 except NotImplementedError:

/Users/Puja/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/distributions/distribution.py in _prob(self, value)
711 def _prob(self, value):
–> 712 raise NotImplementedError(“prob is not implemented”)
713

NotImplementedError: prob is not implemented

During handling of the above exception, another exception occurred:

NotImplementedError Traceback (most recent call last)
in ()
16
17 inference = ed.KLqp(latent_vars,data=data)
—> 18 inference.run(n_samples=5, n_iter=2500)

/Users/Puja/anaconda/lib/python3.6/site-packages/edward/inferences/inference.py in run(self, variables, use_coordinator, *args, **kwargs)
123 Passed into initialize.
124 “”"
–> 125 self.initialize(*args, **kwargs)
126
127 if variables is None:

/Users/Puja/anaconda/lib/python3.6/site-packages/edward/inferences/klqp.py in initialize(self, n_samples, kl_scaling, *args, **kwargs)
108 self.n_samples = n_samples
109 self.kl_scaling = kl_scaling
–> 110 return super(KLqp, self).initialize(*args, **kwargs)
111
112 def build_loss_and_gradients(self, var_list):

/Users/Puja/anaconda/lib/python3.6/site-packages/edward/inferences/variational_inference.py in initialize(self, optimizer, var_list, use_prettytensor, global_step, *args, **kwargs)
66 var_list = list(var_list)
67
—> 68 self.loss, grads_and_vars = self.build_loss_and_gradients(var_list)
69
70 if self.logging:

/Users/Puja/anaconda/lib/python3.6/site-packages/edward/inferences/klqp.py in build_loss_and_gradients(self, var_list)
158 # return build_score_entropy_loss_and_gradients(self, var_list)
159 # else:
–> 160 return build_score_rb_loss_and_gradients(self, var_list)
161
162

/Users/Puja/anaconda/lib/python3.6/site-packages/edward/inferences/klqp.py in build_score_rb_loss_and_gradients(inference, var_list)
1065 q_log_probs[s][qz] = tf.reduce_sum(
1066 inference.scale.get(z, 1.0) *
-> 1067 qz_copy.log_prob(tf.stop_gradient(dict_swap[z])))
1068
1069 for z in six.iterkeys(inference.latent_vars):

/Users/Puja/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/distributions/distribution.py in log_prob(self, value, name)
707 values of type self.dtype.
708 “”"
–> 709 return self._call_log_prob(value, name)
710
711 def _prob(self, value):

/Users/Puja/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/distributions/distribution.py in _call_log_prob(self, value, name, **kwargs)
694 return math_ops.log(self._prob(value, **kwargs))
695 except NotImplementedError:
–> 696 raise original_exception
697
698 def log_prob(self, value, name=“log_prob”):

/Users/Puja/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/distributions/distribution.py in _call_log_prob(self, value, name, **kwargs)
689 value = ops.convert_to_tensor(value, name=“value”)
690 try:
–> 691 return self._log_prob(value, **kwargs)
692 except NotImplementedError as original_exception:
693 try:

/Users/Puja/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/distributions/distribution.py in _log_prob(self, value)
683
684 def _log_prob(self, value):
–> 685 raise NotImplementedError(“log_prob is not implemented”)
686
687 def _call_log_prob(self, value, name, **kwargs):

NotImplementedError: log_prob is not implemented