Learning from Data: Regularization with Weight Decay Exercise

regularization.py

import numpy as np

# Read in the data
data = []
with open('in.dta', 'r') as input:
    row = [float(x) for x in input.readline().strip().split(' ') if x != '']
    while row:
        data.append(row)
        row = [float(x) for x in input.readline().strip().split(' ') if x != '']
training_x = np.array([p[:2] for p in data])
training_y = np.array([p[2] for p in data])

data = []
with open('out.dta', 'r') as input:
    row = [float(x) for x in input.readline().strip().split(' ') if x != '']
    while row:
        data.append(row)
        row = [float(x) for x in input.readline().strip().split(' ') if x != '']
test_x = np.array([p[:2] for p in data])
test_y = np.array([p[2] for p in data])

# x1, x2 -> 1, x1, x2, x1^2, x2^2, x1*x2, |x1-x2|, |x1+x2|
def transformation(x):
    x1 = x[0]
    x2 = x[1]
    return [1, x1, x2, x1**2, x2**2, x1*x2, abs(x1-x2), abs(x1+x2)]

training_x_transformed = np.apply_along_axis(transformation, arr=training_x, axis=1)
test_x_transformed = np.apply_along_axis(transformation, arr=test_x, axis=1)

# With no regularization (lambda = 0)
w = np.dot(np.dot(np.linalg.inv(np.dot(training_x_transformed.T, 
                                       training_x_transformed)), 
                                training_x_transformed.T), 
           training_y)
Ein = np.sum(np.sign(np.dot(training_x_transformed, w)) != training_y)/training_y.size
Eout = np.sum(np.sign(np.dot(test_x_transformed, w)) != test_y)/test_y.size
print("with no regularization, Ein is {:f} and Eout is {:f}".format(Ein, Eout))

# With regularization
for k in [-3, -2, -1, 0, 1, 2, 3]:
    w = np.dot(np.dot(np.linalg.inv(np.dot(training_x_transformed.T, training_x_transformed) + 
                                    10**k*np.identity(8)), training_x_transformed.T),
               training_y)
    Ein = np.sum(np.sign(np.dot(training_x_transformed, w)) != training_y)/training_y.size
    Eout = np.sum(np.sign(np.dot(test_x_transformed, w)) != test_y)/test_y.size
    print("with k = {:n}, Ein is {:f} and Eout is {:f}".format(k, Ein, Eout))