ModelLogregNnBatch_test.lua

-- ModelLogregNnbatch_test.lua
-- unit test
-- ISSUE: For some unknown reason, the error rate on the fitted model is high

require 'ConfusionMatrix'
require 'ModelLogregNnbatch'
require 'makeVp'
require 'ObjectivefunctionLogregNnbatch'
require 'printAllVariables'
require 'printTableVariable'
require 'printVariable'
require 'Random'
require 'torch'

torch.manualSeed(123)

-------------------------------------------------------------------------------
-- make test objects (fixtures in unit test frameworks)
-------------------------------------------------------------------------------

-- create random data about the same size as the problem of most interest
-- RETURNS
-- X, y, s, nCLasses : synthetic data
-- actualTheta       : actual parameters used to generate y from X
local function makeTrainingData(dataSize)
   local vp, verboseLevel = makeVp(0, 'makeTrainingData')

   local nSamples = 60  
   local nFeatures = 8
   local nClasses = 14

   if dataSize == 'small' then
      nSamples = 5
      nFeatures = 2
      nClasses = 3
   end
   
   local L2 = 0       -- arbitrary value needed for APIs

   -- randomly generate data
   local X = torch.rand(nSamples, nFeatures)
   local y = Random():integer(nSamples, 1, nClasses)  -- class numbers not derived from weights
   local s = torch.abs(torch.rand(nSamples)) -- saliences must be non-negative

   -- get random weights from the corresponding optimization function
   local opfunc = ObjectivefunctionLogregNnbatch(X, y, s, nClasses, L2)
   local initialTheta = opfunc:initialTheta()  -- random weights
   vp(2, 'initialTheta', initialTheta)
   
   return X, y, s, nClasses, initialTheta
end

-- return model
local function makeModel(dataSize)
   local X, y, s, nClasses, initialTheta = makeTrainingData(dataSize)

   local model = ModelLogregNnbatch(X, y, s, nClasses)
   return model
end

-------------------------------------------------------------------------------
-- TEST PUBLIC METHODS
-------------------------------------------------------------------------------

-------------------------------------------------------------------------------
-- test construction
-------------------------------------------------------------------------------

local function testConstruction()
   local function testFields(m)
      assert(m.X)
      assert(m.y)
      assert(m.s)
      assert(m.nClasses)
   end
   local zeroSaliences = true
   local lambda = 0.0001
   local model = makeModel()
   assert(model ~= nil)
   testFields(model)

   local model = makeModel('small')
   assert(model ~= nil)
   testFields(model)
end

testConstruction()

-------------------------------------------------------------------------------
-- test method fit
-------------------------------------------------------------------------------

-- converge based only on toleranceLoss
local function fitModelBottouEpoch(model, toleranceLoss, printLoss)
   local vp = makeVp(0, 'fitModelBottouEpoch')
   vp(1, 'model', model)
   assert(model)
   assert(toleranceLoss)
   assert(printLoss ~= nil)

   local function nextStepSizes(currentStepSize)
      return {currentStepSize, 0.5 * currentStepSize, 1.5 * currentStepSize}
   end

   local convergence = {
      toleranceLoss = toleranceLoss}

   local regularizer = {
      L2 = .001}

   local bottouEpoch = {
      initialStepSize = 1,
      nEpochsBeforeAdjustingStepSize = 10,
      nEpochsToAdjustStepSize = 2,
      nextStepSizes = nextStepSizes}

   local fittingOptions = {
      method = 'bottouEpoch',
      convergence = convergence,
      regularizer = regularizer,
      printLoss = printLoss,
      bottouEpoch = bottouEpoch}

   local optimalTheta, fitInfo = model:fit(fittingOptions)
   return optimalTheta, fitInfo
end

-- converge based only on toleranceLoss
local function fitModelGradientDescent(model, toleranceLoss, printLoss)
   local vp = makeVp(0, 'fitModelGradientDescent')
   vp(1, 'model', model)
   assert(model)
   assert(toleranceLoss)
   assert(printLoss ~= nil)

   local convergence = {
      toleranceLoss = toleranceLoss}

   local regularizer = {
      L2 = .001}

   local gradientDescent = {
      stepSize = 1}
      
   local fittingOptions = {
      method = 'gradientDescent',
      convergence = convergence,
      regularizer = regularizer,
      printLoss = printLoss,
      gradientDescent = gradientDescent}

   local optimalTheta, fitInfo = model:fit(fittingOptions)
   return optimalTheta, fitInfo
end

local function fitModelLbfgs(model, toleranceLoss, printLoss)
   local vp = makeVp(0, 'fitModelLbfgs')
   vp(1, 'model', model)
   assert(model)
   assert(toleranceLoss)
   assert(printLoss ~= nil)

   local convergence = {
      maxEpochs = 1,
      toleranceLoss = toleranceLoss}

   local regularizer = {
      L2 = .001}

   local lbfgs = {
      lineSearch = 'wolf'}
      
   local fittingOptions = {
      method = 'lbfgs',
      convergence = convergence,
      regularizer = regularizer,
      printLoss = printLoss,
      lbfgs = lbfgs}

   local optimalTheta, fitInfo = model:fit(fittingOptions)
   return optimalTheta, fitInfo
end

local function testFitInfo(fitInfo)
   local vp = makeVp(0, 'testFitInfo')
   assert(type(fitInfo) == 'table')
   
   local function testFieldType(fieldName, expectedType)
      local fieldValue = fitInfo[fieldName]
      assert(fieldValue)
      assert(type(fieldValue) == expectedType)
      vp(2, fieldName, fieldValue)
   end
   
   testFieldType('convergedReason', 'string')
   testFieldType('finalLoss', 'number')
   testFieldType('nEpochsUntilConvergence', 'number')
   
   assert(fitInfo.optimalTheta:nDimension() == 1)

   assert(fitInfo.convergedReason == 'toleranceLoss' or
          fitInfo.convergedReason == 'maxEpochs')
end

local function makeConfusionMatrix(actuals, predictions)
   local vp = makeVp(0, 'makeConfusionMatrix')
   vp(1, 'actuals', actuals, 'predictions', predictions)
   local cm = ConfusionMatrix()
   for i = 1, actuals:size(1) do
      cm:add(actuals[i], predictions[i])
   end
   return cm
end

local function determineErrorRate(actuals, predictions)
   local vp = makeVp(1, 'determineErrorRate')
   local cm = makeConfusionMatrix(actuals, predictions)
   return cm:errorRate()
end

local function testOptimalTheta(optimalTheta, model, newX, expectedY)
   local vp = makeVp(0, 'testOptimalTheta')
   vp(1, 'model', model)
   assert(optimalTheta:nDimension() == 1)
   assert(model)
   assert(newX)
   assert(expectedY)


   -- test supposed optimalTheta by checking the predictions
   local probabilities, predictInfo = model:predict(newX, optimalTheta)
   local errorRate = determineErrorRate(expectedY, predictInfo.mostLikelyClasses)
   vp(1, 'errorRate', errorRate)
   -- NOTE: for the BottouEpoch method, the lowest error rate was 0.62 for toleranceLoss = 1e-6
   assert(errorRate <= 1) -- for now
end

local function testFitDriver(howToFitModel, toleranceLoss, printLoss, dataSize)
   local vp = makeVp(2, 'testFit')
   local model = makeModel(dataSize)
   local optimalTheta, fitInfo = howToFitModel(model, toleranceLoss, printLoss)
   testOptimalTheta(optimalTheta, model, model.X, model.y)
   testFitInfo(fitInfo)
end

local function testFit(dataSize)
   local toleranceLoss = 1e-6
   local printLoss = false
   testFitDriver(fitModelLbfgs, toleranceLoss, printLoss, dataSize)
   testFitDriver(fitModelBottouEpoch, toleranceLoss, printLoss, dataSize)
   testFitDriver(fitModelGradientDescent, toleranceLoss, printLoss, dataSize)
end

if true then
   local dataSize = 'small'
   testFit(dataSize)
else
   print('did not run testFit')
end

-------------------------------------------------------------------------------
-- test method predict
-------------------------------------------------------------------------------

local function testPredictValues(toleranceLoss)
   local vp, verbose = makeVp(0, 'testPredictValues')

   assert(toleranceLoss ~= nil)
   
   -- make the model
   local model = makeModel()
   local actualX = model.X
   local actualY = model.y
   vp(2, 'actualY', actualY)
   
   -- fit the model
   local printLoss = false
   local optimalTheta, fitInfo = fitModelBottouEpoch(model, toleranceLoss, printLoss)
   -- check that we actually converged
   assert(fitInfo.convergedReason ~= 'maxEpochs', fitInfo.convergedReason)

   -- predict each X used as training data
   local probabilities, predictInfo = model:predict(actualX, optimalTheta)
   vp(2, 'probabilities', probabilities)
   assert(probabilities:nDimension() == 2)
   assert(probabilities:size(1) == model.nSamples)
   assert(probabilities:size(2) == model.nClasses)

   assert(type(predictInfo) == 'table')
   local predictedClasses = predictInfo.mostLikelyClasses
   vp(2, 'predictedClasses', predictedClasses)
   assert(predictedClasses:size(1) == model.nSamples)

   -- see how we did
   local cm = makeConfusionMatrix(actualY, predictedClasses)
   local errorRate = cm:errorRate()
   vp(1, 'errorRate', errorRate)
   if verbose > 0 then
      cm:printTo(io.stdout,'confusion matrix')
   end
   assert(errorRate < .90)  -- I don't know why the predictions are not more accurate
end

local toleranceLoss = 1e-6  -- tests show that this toleranceLoss value lead to lowest overall Loss
testPredictValues(toleranceLoss)

print('ok ModelLogregNnbatch')