SVM with L^1 regularization

# Generate data for SVM classifier with L1 regularization.
using Random
Random.seed!(3);
n = 20;
m = 1000;
TEST = m;
DENSITY = 0.2;
beta_true = randn(n, 1);
idxs = randperm(n)[1:round(Int, (1 - DENSITY) * n)];
beta_true[idxs] .= 0
offset = 0;
sigma = 45;
X = 5 * randn(m, n);
Y = sign.(X * beta_true .+ offset .+ sigma * randn(m, 1));
X_test = 5 * randn(TEST, n);

# Form SVM with L1 regularization problem.
using Convex, SCS, ECOS

beta = Variable(n);
v = Variable();
loss = sum(pos(1 - Y .* (X * beta - v)));
reg = norm(beta, 1);

# Compute a trade-off curve and record train and test error.
TRIALS = 100
train_error = zeros(TRIALS);
test_error = zeros(TRIALS);
lambda_vals = exp10.(range(-2, stop = 0, length = TRIALS);)
beta_vals = zeros(length(beta), TRIALS);
for i in 1:TRIALS
    lambda = lambda_vals[i]
    problem = minimize(loss / m + lambda * reg)
    solve!(problem, SCS.Optimizer; silent_solver = true)
    train_error[i] =
        sum(
            float(
                sign.(X * beta_true .+ offset) .!=
                sign.(evaluate(X * beta - v)),
            ),
        ) / m
    test_error[i] =
        sum(
            float(
                sign.(X_test * beta_true .+ offset) .!=
                sign.(evaluate(X_test * beta - v)),
            ),
        ) / TEST
    beta_vals[:, i] = evaluate(beta)
end

Plot the train and test error over the trade-off curve.

using Plots
plot(lambda_vals, train_error, label = "Train error");
plot!(lambda_vals, test_error, label = "Test error");
plot!(xscale = :log, yscale = :log, ylabel = "errors", xlabel = "lambda")

Plot the regularization path for beta.

plot()
for i in 1:n
    plot!(lambda_vals, vec(beta_vals[i, :]), label = "beta$i")
end
plot!(xscale = :log, ylabel = "betas", xlabel = "lambda")

This page was generated using Literate.jl.