SpatialPooler with log boosting

CommonCompilerConfig.cmake

@@ -350,7 +350,7 @@ else()
         set(optimization_flags_cc ${optimization_flags_cc} -pipe -O3)
         set(optimization_flags_lt ${optimization_flags_lt} -O3)
         if(NOT ${CMAKE_SYSTEM_PROCESSOR} STREQUAL "armv7l")
-                set(optimization_flags_cc ${optimization_flags_cc} -mtune=generic)
+                set(optimization_flags_cc ${optimization_flags_cc} -march=native)
         endif()
         if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" AND NOT MINGW)
                 # NOTE -flto must go together in both cc and ld flags; also, it's presently incompatible

src/examples/mnist/MNIST_SP.cpp

@@ -75,7 +75,7 @@ class MNIST {
 
 void setup() {
 
-  input.initialize({28, 28,1}); 
+  input.initialize({28, 28, 1}); 
   columns.initialize({28, 28, 8}); //1D vs 2D no big difference, 2D seems more natural for the problem. Speed-----, Results+++++++++; #columns HIGHEST impact. 
   sp.initialize(
     /* inputDimensions */             input.dimensions,
@@ -90,7 +90,7 @@ void setup() {
     /* synPermConnected */            0.5f, //no difference, let's leave at 0.5 in the middle
     /* minPctOverlapDutyCycles */     0.2f, //speed of re-learning?
     /* dutyCyclePeriod */             1402,
-    /* boostStrength */               2.0f, // Boosting does help, but entropy is high, on MNIST it does not matter, for learning with TM prefer boosting off (=0.0), or "neutral"=1.0
+    /* boostStrength */               7.0f, // Boosting does help, but entropy is high, on MNIST it does not matter, for learning with TM prefer boosting off (BOOSTING_DISABLED), or "neutral"=1.0
     /* seed */                        4u,
     /* spVerbosity */                 1u,
     /* wrapAround */                  true); // does not matter (helps slightly)

src/htm/algorithms/SpatialPooler.cpp

@@ -71,7 +71,7 @@ SpatialPooler::SpatialPooler() {
 }
 
 SpatialPooler::SpatialPooler(
-    const vector<UInt> inputDimensions, const vector<UInt> columnDimensions,
+    const vector<UInt>& inputDimensions, const vector<UInt>& columnDimensions,
     UInt potentialRadius, Real potentialPct, bool globalInhibition,
     Real localAreaDensity,
     UInt stimulusThreshold, Real synPermInactiveDec, Real synPermActiveInc,
@@ -401,7 +401,6 @@ void SpatialPooler::initialize(
   // 1D input produces 1D output; 2D => 2D, etc. //TODO allow nD -> mD conversion
   NTA_CHECK(inputDimensions_.size() == columnDimensions_.size()); 
 
-  NTA_CHECK(localAreaDensity > 0 && localAreaDensity <= MAX_LOCALAREADENSITY);
   setLocalAreaDensity(localAreaDensity); 
 
   rng_ = Random(seed);
@@ -742,7 +741,8 @@ void SpatialPooler::updateDutyCyclesHelper_(vector<Real> &dutyCycles,
 
 
 void SpatialPooler::updateBoostFactors_() {
-  if (globalInhibition_) {
+  if(boostStrength_ == SpatialPooler::BOOSTING_DISABLED) return;
+  else if (globalInhibition_) {
     updateBoostFactorsGlobal_();
   } else {
     updateBoostFactorsLocal_();
@@ -755,8 +755,12 @@ void applyBoosting_(const UInt i,
 		    const vector<Real>& actualDensity,
 		    const Real boost,
 	            vector<Real>& output) {
-  if(boost < htm::Epsilon) return; //skip for disabled boosting
-  output[i] = exp((targetDensity - actualDensity[i]) * boost); //TODO doc this code
+  if(boost == SpatialPooler::BOOSTING_DISABLED) output[i] = actualDensity[i]; //no change
+  else if(boost == SpatialPooler::BOOSTING_LOG) { //logarithmic boosting
+    output[i] = log2(actualDensity[i]) / log2(targetDensity);
+  } else if(boost >= SpatialPooler::BOOSTING_EXP) { //exponential boost
+    output[i] = exp((targetDensity - actualDensity[i]) * boost);
+  } else NTA_THROW << "Invalid boost mode! " << boost; 
 }
 
 
@@ -775,12 +779,12 @@ void SpatialPooler::updateBoostFactorsLocal_() {
     Real localActivityDensity = 0.0f;
 
     if (wrapAround_) {
-      for(auto neighbor: WrappingNeighborhood(i, inhibitionRadius_, columnDimensions_)) {
+      for(const auto neighbor: WrappingNeighborhood(i, inhibitionRadius_, columnDimensions_)) {
         localActivityDensity += activeDutyCycles_[neighbor];
         numNeighbors += 1;
       }
     } else {
-      for(auto neighbor: Neighborhood(i, inhibitionRadius_, columnDimensions_)) {
+      for(const auto neighbor: Neighborhood(i, inhibitionRadius_, columnDimensions_)) {
         localActivityDensity += activeDutyCycles_[neighbor];
         numNeighbors += 1;
       }

src/htm/algorithms/SpatialPooler.hpp

@@ -61,12 +61,16 @@ using namespace std;
 class SpatialPooler : public Serializable
 {
 public:
-
-  const Real MAX_LOCALAREADENSITY = 0.5f; //require atleast 2 areas
+  //boosting modes:
+  static const constexpr Real BOOSTING_DISABLED = 0.0f;
+  static const constexpr Real BOOSTING_LOG      = -1.0f * htm::Epsilon; //negative value closest to 0, so it's possible to search <BOOSTING_LOG, (DISABLED=0.0),..10.0> in parameter optimization to try all 3 combinations of boosting. 
+  static const constexpr Real BOOSTING_EXP      = 1.0f * htm::Epsilon; //any value > BOOSTING_DISABLED enables the exponential boosting mode
 
   SpatialPooler();
-  SpatialPooler(const vector<UInt> inputDimensions, const vector<UInt> columnDimensions,
-                UInt potentialRadius = 16u, Real potentialPct = 0.5f,
+  SpatialPooler(const vector<UInt>& inputDimensions, 
+		const vector<UInt>& columnDimensions,
+                UInt potentialRadius = 16u, 
+		Real potentialPct = 0.5f,
                 bool globalInhibition = true, 
 		Real localAreaDensity = 0.05f, //5%
                 UInt stimulusThreshold = 0u, 
@@ -75,7 +79,7 @@ class SpatialPooler : public Serializable
 		Real synPermConnected = 0.1f,
                 Real minPctOverlapDutyCycles = 0.001f,
                 UInt dutyCyclePeriod = 1000u, 
-		Real boostStrength = 0.0f,
+		Real boostStrength = BOOSTING_DISABLED,
                 Int seed = 1, 
 		UInt spVerbosity = 0u, 
 		bool wrapAround = true);
@@ -176,13 +180,24 @@ class SpatialPooler : public Serializable
         boost. Shorter values make it potentially more unstable and
         likely to oscillate.
 
-  @param boostStrength A number greater or equal than 0, used to
-        control boosting strength. 
-	No boosting is applied if it is set to 0.0, (runs faster due to skipped code).
-        The strength of boosting increases as a function of boostStrength.
-        Boosting encourages columns to have similar activeDutyCycles as their
-        neighbors, which will lead to more efficient use of columns. However,
-        too much boosting may also lead to instability of SP outputs.
+  @param boostStrength A number used to control boosting strength coeficient, 
+	 and mode of operation based on (reserved) values:
+	- `== SpatialPooler::BOOSTING_DISABLED`:
+	No boosting is applied if it is set, (runs faster due to skipped code).
+	- `== SpatialPooler::BOOSTING_LOG`:
+	Logarithmic boosting is used.
+	- `> 0.0`: 
+	Exponential boosting (default in Numenta) selected. `boostStrength` is 
+	used as a multiplacation constant.
+        The strength of boosting increases as a function of `boostStrength`.
+
+        Boosting encourages columns to have similar `activeDutyCycles` (aka. activation
+        frequencies) as their neighbors, which will lead to more efficient use of columns. 
+	This helps achieving the target sparsity of the output.
+	However, too much boosting may also lead to instability of SP outputs.
+
+	Notes:
+	- Log boosting does not require a parameter, but it is slower than exp.
 
 
   @param seed Seed for our random number generator. If seed is < 0
@@ -203,11 +218,16 @@ class SpatialPooler : public Serializable
 	     Real potentialPct = 0.5f,
              bool globalInhibition = true, 
 	     Real localAreaDensity = 0.05f,
-             UInt stimulusThreshold = 0u,
-             Real synPermInactiveDec = 0.01f, Real synPermActiveInc = 0.1f,
-             Real synPermConnected = 0.1f, Real minPctOverlapDutyCycles = 0.001f,
-             UInt dutyCyclePeriod = 1000u, Real boostStrength = 0.0f,
-             Int seed = 1, UInt spVerbosity = 0u, bool wrapAround = true);
+	     UInt stimulusThreshold = 0u,
+             Real synPermInactiveDec = 0.008f, 
+	     Real synPermActiveInc = 0.05f,
+             Real synPermConnected = 0.1f, 
+	     Real minPctOverlapDutyCycles = 0.001f,
+             UInt dutyCyclePeriod = 1000u, 
+	     Real boostStrength = BOOSTING_DISABLED,
+             Int seed = 1, 
+	     UInt spVerbosity = 0u, 
+	     bool wrapAround = true);
 
 
   /**

src/htm/types/Types.hpp

@@ -130,7 +130,7 @@ typedef std::size_t Size;
  *     numeric_limits<float>::epsilon()  == 1.19209e-7
  *     numeric_limits<double>::epsilon() == 2.22045e-16
  */
-static const htm::Real32 Epsilon = htm::Real(1e-6);
+static const constexpr htm::Real32 Epsilon = htm::Real(1e-6);
 
 /**
  * Represents a signed integer.

src/test/unit/algorithms/SpatialPoolerTest.cpp

@@ -239,7 +239,10 @@ void setup(SpatialPooler& sp, UInt numIn, UInt numCols, Real sparsity = 0.5f) {
 TEST(SpatialPoolerTest, testUpdateInhibitionRadius) {
   SpatialPooler sp;
   vector<UInt> colDim, inputDim;
-  colDim.push_back(57);
+
+  //test for global inhibition, this is trivial
+  {
+  colDim.push_back(57); //max SP dimension
   colDim.push_back(31);
   colDim.push_back(2);
   inputDim.push_back(1);
@@ -248,9 +251,13 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) {
 
   EXPECT_NO_THROW(sp.initialize(inputDim, colDim));
   sp.setGlobalInhibition(true);
-  ASSERT_EQ(sp.getInhibitionRadius(), 57u);
+  ASSERT_TRUE(sp.getInhibitionRadius() == 57) << "In global inh radius must match max dimension";
+  }
 
-  //test 2 - local inhibition radius
+  //tests for local inhibition
+  UInt numInputs = 3;
+  UInt numCols = 12;
+  {
   colDim.clear();
   inputDim.clear();
   // avgColumnsPerInput = 4
@@ -266,12 +273,12 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) {
     Real permArr[] = {1, 1, 1};
     sp.setPermanence(i, permArr);
   }
-  UInt trueInhibitionRadius = 6;
   // ((3 * 4) - 1)/2 => round up
   sp.updateInhibitionRadius_();
-  ASSERT_EQ(trueInhibitionRadius, sp.getInhibitionRadius());
+  ASSERT_EQ(6u,  sp.getInhibitionRadius());
+  }
 
-  //test 3
+  {
   colDim.clear();
   inputDim.clear();
   // avgColumnsPerInput = 1.2
@@ -289,12 +296,11 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) {
     }
     sp.setPermanence(i, permArr);
   }
-  trueInhibitionRadius = 1;
   sp.updateInhibitionRadius_();
-  ASSERT_EQ(trueInhibitionRadius, sp.getInhibitionRadius());
-
+  ASSERT_EQ(1u, sp.getInhibitionRadius());
+  }
 
-  //test 4
+  {
   colDim.clear();
   inputDim.clear();
   // avgColumnsPerInput = 2.4
@@ -309,10 +315,10 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) {
     Real permArr[] = {1, 1, 0, 0, 0};
     sp.setPermanence(i, permArr);
   }
-  trueInhibitionRadius = 2;
   // ((2.4 * 2) - 1)/2 => round up
   sp.updateInhibitionRadius_();
-  ASSERT_EQ(trueInhibitionRadius, sp.getInhibitionRadius());
+  ASSERT_EQ(2u, sp.getInhibitionRadius());
+  }
 }
 
 TEST(SpatialPoolerTest, testUpdateMinDutyCycles) {
@@ -985,24 +991,11 @@ TEST(SpatialPoolerTest, testUpdateBoostFactors) {
   sp.getBoostFactors(resultBoostFactors1.data());
   ASSERT_TRUE(check_vector_eq(trueBoostFactors1, resultBoostFactors1));
 
-  Real32 initActiveDutyCycles2[] = {0.1f, 0.3f, 0.02f, 0.04f, 0.7f, 0.12f};
-  Real32 initBoostFactors2[] = {0, 0, 0, 0, 0, 0};
-  vector<Real32> trueBoostFactors2 = {3.10599f, 0.42035f,    6.91251f,
-                                      5.65949f, 0.00769898f, 2.54297f};
-  vector<Real32> resultBoostFactors2(6, 0);
-  sp.setGlobalInhibition(false);
-  sp.setBoostStrength(10);
-  sp.setBoostFactors(initBoostFactors2);
-  sp.setActiveDutyCycles(initActiveDutyCycles2);
-  sp.updateBoostFactors_();
-  sp.getBoostFactors(resultBoostFactors2.data());
-
-  ASSERT_TRUE(check_vector_eq(trueBoostFactors2, resultBoostFactors2));
 
   Real32 initActiveDutyCycles3[] = {0.1f, 0.3f, 0.02f, 0.04f, 0.7f, 0.12f};
   Real initBoostFactors3[] = {0, 0, 0, 0, 0, 0};
-  vector<Real32> trueBoostFactors3 = {1.25441f, 0.840857f, 1.47207f,
-                                      1.41435f, 0.377822f, 1.20523f};
+  vector<Real32> trueBoostFactors3 = {1.49044f, 0.779321f, 2.53222f,
+	                              2.08355f, 0.230873f, 1.37243f};
   vector<Real32> resultBoostFactors3(6, 0);
   sp.setWrapAround(true);
   sp.setGlobalInhibition(false);
@@ -1015,10 +1008,11 @@ TEST(SpatialPoolerTest, testUpdateBoostFactors) {
 
   ASSERT_TRUE(check_vector_eq(trueBoostFactors3, resultBoostFactors3));
 
+
   Real32 initActiveDutyCycles4[] = {0.1f, 0.3f, 0.02f, 0.04f, 0.7f, 0.12f};
   Real32 initBoostFactors4[] = {0, 0, 0, 0, 0, 0};
-  vector<Real32> trueBoostFactors4 = {1.94773f, 0.263597f,   4.33476f,
-                                      3.549f,   0.00482795f, 1.59467f};
+  vector<Real32> trueBoostFactors4 = {1.2851f, 0.67195f, 2.18334f,
+	                              1.79649f, 0.199064f, 1.18334f};
   vector<Real32> resultBoostFactors4(6, 0);
   sp.setGlobalInhibition(true);
   sp.setBoostStrength(10);
@@ -1028,7 +1022,7 @@ TEST(SpatialPoolerTest, testUpdateBoostFactors) {
   sp.updateBoostFactors_();
   sp.getBoostFactors(resultBoostFactors4.data());
 
-  ASSERT_TRUE(check_vector_eq(trueBoostFactors3, resultBoostFactors3));
+  ASSERT_TRUE(check_vector_eq(trueBoostFactors4, resultBoostFactors4));
 }