[WIP] Equivalence tests

test/Project.toml

@@ -0,0 +1,7 @@
+[deps]
+AbstractGPs = "99985d1d-32ba-4be9-9821-2ec096f28918"
+Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
+Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+IterTools = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/equivalences.jl

@@ -0,0 +1,119 @@
+@testset "equivalences" begin
+    rng, N = MersenneTwister(654321), 20
+    x = rand(rng, N) * 10
+    y = sin.(x) + 0.9 * cos.(x * 1.6) + 0.4 * rand(rng, N)
+
+    z = copy(x) # Set inducing inputs == training inputs
+
+    # Create a kernel from parameters k
+    kernel(k) = softplus(k[1]) * (SqExponentialKernel() ∘ ScaleTransform(softplus(k[2])))
+    k_init = [0.2, 0.6] # initial kernel parameters
+
+    lik_noise = 0.1 # The (fixed) Gaussian likelihood noise
+    jitter = 1e-5
+
+    ## FIRST - define the models
+    # GPR - Exact GP regression
+    struct GPRModel
+        k # kernel parameters
+    end
+    @Flux.functor GPRModel
+
+    function (m::GPRModel)(x)
+        f = GP(kernel(m.k))
+        fx = f(x, lik_noise)
+        return fx
+    end
+
+    # # SGPR - Sparse GP regression (Titsias 2009)
+    # struct SGPRModel
+    #     k # kernel parameters
+    #     z # inducing points
+    # end
+    # @Flux.functor SGPRModel (k,) # Don't train the inducing inputs
+
+    # function (m::SGPRModel)(x)
+    #     f = GP(kernel(m.k))
+    #     fx = f(x, lik_noise)
+    #     fz = f(m.z, lik_noise)
+    #     return fx, fz
+    # end
+
+    # SVGP - Sparse variational GP regression (Hensman 2014)
+    struct SVGPModel
+        k # kernel parameters
+        z # inducing points
+        m # variational mean
+        A # variational covariance sqrt (Σ = A'A)
+    end
+    @Flux.functor SVGPModel (k, m, A,) # Don't train the inducing inputs
+
+    function (m::SVGPModel)(x)
+        f = GP(kernel(m.k))
+        q = MvNormal(m.m, m.A'm.A)
+        fx = f(x, lik_noise)
+        fz = f(m.z, jitter)
+        return fx, fz, q
+    end
+
+    ## SECOND - create the models and associated training losses
+    gpr = GPRModel(copy(k_init))
+    function GPR_loss(x, y)
+        fx = gpr(x)
+        return -logpdf(fx, y)
+    end
+
+    # sgpr = SGPRModel(copy(k_init), copy(z))
+    # function SGPR_loss(x, y)
+    #     fx, fz = sgpr(x)
+    #     return -AbstractGPs.elbo(fx, y, fz)
+    # end
+
+    m, A = rand(rng, N), rand(rng, N, N)/2 # initialise the variational parameters
+    svgp = SVGPModel(copy(k_init), copy(z), m, A)
+    function SVGP_loss(x, y)
+        fx, fz, q = svgp(x)
+        return -SparseGPs.elbo(fx, y, fz, q)
+    end
+
+    ## THIRD - train the models
+    data = [(x, y)]
+    opt = ADAM(0.01)
+
+    svgp_ps = Flux.params(svgp)
+    delete!(svgp_ps, svgp.k) # Don't train the kernel parameters
+
+    # Flux.train!((x, y) -> GPR_loss(x, y), Flux.params(gpr), ncycle(data, 3000), opt)
+    # Flux.train!((x, y) -> SGPR_loss(x, y), Flux.params(sgpr), ncycle(data, 3000), opt)
+    Flux.train!((x, y) -> SVGP_loss(x, y), svgp_ps, ncycle(data, 9000), opt)
+
+    ## FOURTH - construct the posteriors
+    function posterior(m::GPRModel, x, y)
+        f = GP(kernel(m.k))
+        fx = f(x, lik_noise)
+        return AbstractGPs.posterior(fx, y)
+    end
+
+    # function posterior(m::SGPRModel, x, y)
+    #     f = GP(kernel(m.k))
+    #     fx = f(x, lik_noise)
+    #     fz = f(m.z)
+    #     return AbstractGPs.approx_posterior(VFE(), fx, y, fz)
+    # end
+
+    function posterior(m::SVGPModel)
+        f = GP(kernel(m.k))
+        fz = f(m.z, jitter)
+        q = MvNormal(m.m, m.A'm.A)
+        return SparseGPs.approx_posterior(SVGP(), fz, q)
+    end
+    gpr_post = posterior(gpr, x, y)
+    # sgpr_post = posterior(sgpr, x, y)
+    svgp_post = posterior(svgp)
+
+    ## FIFTH - test equivalences
+    @test all(isapprox.(mean(gpr_post, x), mean(svgp_post, x), atol=1e-3))
+    @test all(isapprox.(cov(gpr_post, x), cov(svgp_post, x), atol=1e-3))
+
+end
+

test/runtests.jl

@@ -0,0 +1,22 @@
+using Random
+using Test
+using SparseGPs
+using Flux
+using IterTools
+using AbstractGPs
+using Distributions
+
+const GROUP = get(ENV, "GROUP", "All")
+const PKGDIR = dirname(dirname(pathof(SparseGPs)))
+
+include("test_utils.jl")
+
+@testset "SparseGPs" begin
+    include("svgp.jl")
+    println(" ")
+    @info "Ran svgp tests"
+
+    include("equivalences.jl")
+    println(" ")
+    @info "Ran equivalences tests"
+end

test/svgp.jl

@@ -0,0 +1,4 @@
+@testset "svgp" begin
+    x = 4
+    @test x == 4
+end