Feature selection in Bayesian linear Regreesion


#1

I want to try the Bayesian linear Regreesion with feature selection, this is my code, is it right? I hope the parameter z can select the true features in the model.

def build_toy_dataset(N, w, noise_std=0):
    D = len(w)
    x = np.random.randn(N, D).astype(np.float32)
    y = np.dot(x, w) + np.random.normal(0, noise_std, size=N)
    return x, y

### Data
N = 500
D = 8
w_true = (-1.25, 4.1, 2.32, 0.99, -3.44, 0.0, 0.0, 0.0)
X_train, y_train = build_toy_dataset(N, w_true, 0.1)
X_test, y_test = build_toy_dataset(N, w_true, 0.1)

### Model
X = tf.placeholder(tf.float32, [N, D])
z = Bernoulli(probs=tf.ones(D))
w = Normal(loc=tf.zeros(D), scale=tf.ones(D))
b = Normal(loc=tf.zeros(1), scale=tf.ones(1))
y = Normal(loc=ed.dot(X,  tf.to_float(z)*w) + b, scale=tf.ones(N))

qw = Normal(loc=tf.Variable(tf.random_normal([D])), 
            scale=tf.nn.softplus(tf.Variable(tf.random_normal([D]))))
qz = Bernoulli(probs=tf.nn.softplus(tf.Variable(tf.random_normal([D]))))
qb = Normal(loc=tf.Variable(tf.random_normal([1])), 
            scale=tf.nn.softplus(tf.Variable(tf.random_normal([1]))))
inference = ed.KLqp({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.run()