Gaussian processes kernlab::gausspr implementation

[bhvieira]

EDIT: should I have created this as a PR?

Hey everyone, mlR currently lacks some learners and methods from kernlab, and as I'm currently working with both I thought it would be a good idea to share what I have. Here's the script for gaussian processes classification. I just edited the "classif.ksvm" one, and kept all the comments too.

#' @export
makeRLearner.classif.gausspr = function() {
  makeRLearnerClassif(
    cl = "classif.gausspr",
    package = "kernlab",
    # FIXME: stringdot pars and check order, scale and offset limits
    par.set = makeParamSet(
      makeLogicalLearnerParam(id = "scaled", default = TRUE),
      makeDiscreteLearnerParam(id = "kernel", default = "rbfdot",
        values = c("vanilladot", "polydot", "rbfdot", "tanhdot", "laplacedot", "besseldot", "anovadot", "splinedot")),
      makeNumericLearnerParam(id = "sigma",
        lower = 0, requires = quote(kernel %in% c("rbfdot", "anovadot", "besseldot", "laplacedot"))),
      makeIntegerLearnerParam(id = "degree", default = 3L, lower = 1L,
        requires = quote(kernel %in% c("polydot", "anovadot", "besseldot"))),
      makeNumericLearnerParam(id = "scale", default = 1, lower = 0,
        requires = quote(kernel %in% c("polydot", "tanhdot"))),
      makeNumericLearnerParam(id = "offset", default = 1,
        requires = quote(kernel %in% c("polydot", "tanhdot"))),
      makeIntegerLearnerParam(id = "order", default = 1L,
        requires = quote(kernel == "besseldot")),
      makeNumericLearnerParam(id = "tol", default = 0.001, lower = 0),
      makeLogicalLearnerParam(id = "fit", default = TRUE)
    ),
    par.vals = list(fit = FALSE),
    properties = c("twoclass", "multiclass", "numerics", "factors", "prob"),
    name = "Gaussian Processes",
    short.name = "gausspr",
    note = "Kernel parameters have to be passed directly and not by using the `kpar` list in `gausspr`. Note that `fit` has been set to `FALSE` by default for speed."
  )
}

#' @export
trainLearner.classif.gausspr = function(.learner, .task, .subset, .weights = NULL, degree, offset, scale, sigma, order, length, lambda, normalized,  ...) {

  # FIXME: custom kernel. freezes? check mailing list
  # FIXME: unify cla + regr, test all sigma stuff

#     # there's a strange behaviour in r semantics here wgich forces this, see do.call and the comment about substitute
#     if (!is.null(args$kernel) && is.function(args$kernel) && !is(args$kernel,"kernel")) {
#       args$kernel = do.call(args$kernel, kpar)
#     }
  kpar = learnerArgsToControl(list, degree, offset, scale, sigma, order, length, lambda, normalized)
  f = getTaskFormula(.task)
  pm = .learner$predict.type == "prob"
  if (base::length(kpar) > 0L)
    kernlab::gausspr(f, data = getTaskData(.task, .subset), kpar = kpar, prob.model = pm, ...)
  else
    kernlab::gausspr(f, data = getTaskData(.task, .subset), prob.model = pm, ...)
}

#' @export
predictLearner.classif.gausspr = function(.learner, .model, .newdata, ...) {
  type = switch(.learner$predict.type, prob = "probabilities", "response")
  kernlab::predict(.model$learner.model, newdata = .newdata, type = type, ...)
}

Here is the regression one. kernlab manual says it can predict sdeviation but I wasn't able to make it work with the bh.task dataset so it might be better to leave it out for now. EDIT: I double checked the kernlab manual and that seems like the correct behavior, so the code below now includes se estimation.

#' @export
makeRLearner.regr.gausspr = function() {
  makeRLearnerRegr(
    cl = "regr.gausspr",
    package = "kernlab",
    # FIXME: stringdot pars and check order, scale and offset limits
    par.set = makeParamSet(
      makeLogicalLearnerParam(id = "scaled", default = TRUE),
      makeDiscreteLearnerParam(id = "kernel", default = "rbfdot",
        values = c("vanilladot", "polydot", "rbfdot", "tanhdot", "laplacedot", "besseldot", "anovadot", "splinedot")),
      makeNumericLearnerParam(id = "sigma",
        lower = 0, requires = quote(kernel %in% c("rbfdot", "anovadot", "besseldot", "laplacedot"))),
      makeIntegerLearnerParam(id = "degree", default = 3L, lower = 1L,
        requires = quote(kernel %in% c("polydot", "anovadot", "besseldot"))),
      makeNumericLearnerParam(id = "scale", default = 1, lower = 0,
        requires = quote(kernel %in% c("polydot", "tanhdot"))),
      makeNumericLearnerParam(id = "offset", default = 1,
        requires = quote(kernel %in% c("polydot", "tanhdot"))),
      makeIntegerLearnerParam(id = "order", default = 1L,
        requires = quote(kernel == "besseldot")),
      makeNumericLearnerParam(id = "var", default = 0.001),
      makeNumericLearnerParam(id = "tol", default = 0.001, lower = 0),
      makeLogicalLearnerParam(id = "fit", default = TRUE)
    ),
    par.vals = list(fit = FALSE),
    properties = c("numerics", "factors", "se"),
    name = "Gaussian Processes",
    short.name = "gausspr",
    note = "Kernel parameters have to be passed directly and not by using the `kpar` list in `gausspr`. Note that `fit` has been set to `FALSE` by default for speed."
  )
}

#' @export
trainLearner.regr.gausspr = function(.learner, .task, .subset, .weights = NULL, degree, offset, scale, sigma, order, length, lambda, normalized,  ...) {

  # FIXME: custom kernel. freezes? check mailing list
  # FIXME: unify cla + regr, test all sigma stuff

#     # there's a strange behaviour in r semantics here wgich forces this, see do.call and the comment about substitute
#     if (!is.null(args$kernel) && is.function(args$kernel) && !is(args$kernel,"kernel")) {
#       args$kernel = do.call(args$kernel, kpar)
#     }
  kpar = learnerArgsToControl(list, degree, offset, scale, sigma, order, length, lambda, normalized)
  f = getTaskFormula(.task)
  vm = .learner$predict.type == "se"
  if (base::length(kpar) > 0L)
    kernlab::gausspr(f, data = getTaskData(.task, .subset), kpar = kpar, variance.model = vm, type = "regression", ...)
  else
    kernlab::gausspr(f, data = getTaskData(.task, .subset), variance.model = vm, type = "regression", ...)
}

#' @export
predictLearner.regr.gausspr = function(.learner, .model, .newdata, ...) {
  if(.learner$predict.type != "se"){
    matrix(kernlab::predict(.model$learner.model, newdata = .newdata, ...))
  }else{
    pred = matrix(kernlab::predict(.model$learner.model, newdata = .newdata, ...)) 
    pred.se = matrix(kernlab::predict(.model$learner.model, newdata = .newdata, type = "sdeviation", ...))
    cbind(pred,pred.se)   
  }
}

Keep in mind gaussian processes classification is computationally costly and scale O(n^3). I'm also working with probabilities outputs for native multiclass ksvm types ("kbb-svc" and "spoc-svc" return only decisions) and lssvm probability outputs outside kernlab and also an alternative prob.model through isotonic regression instead of Platt Scaling (scikit users might expect to find it here, dunno). If I'm sucessful I'll share it here.

I'm new to Git, so if I posted this in the wrong area or the wrong way please excuse me. Thanks!

[berndbischl]

Hi @catastrophic-failure

Thanks! Sharing this code with others is exactly what we wish for and what the package should "motivate" you to do.

Even better would be submitting a pull request for this. But of course this is issue is also good. We will take this over now, Stefan = @Coorsaa is working on this now on our side to integrate this.

[bhvieira]

Ok :) Also thanks for mlr, it's a great tool. I have coded subselect::trim.matrix as a Preprocess Wrapper. Will try a PR this time. I'm new to Git so please forgive me if I do anything wrong!

[berndbischl]

I have coded subselect::trim.matrix as a Preprocess Wrapper. Will try a PR this time. I'm new to Git so please forgive me if I do anything wrong!

Ok. Simply try your best and we will help you. For new code, where it is a bit harder to decide "how" we should integrate it (eg possibly new preproc stuff and not a simple learner), please open up a issue first.

This way we can discuss what i best, and we dont waste your time when we think it should be handled differently than in your code ;-)

[Coorsaa]

Done in PR #930