Add KL divergence between Deterministic + others for pytorch

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.1.0
+current_version = 2.1.1
 
 [bumpversion:file:setup.py]
 

.flake8

@@ -1,2 +1,4 @@
 [flake8]
-ignore = E203,E501,W503,E731,E741,F401,F403,F405
+ignore = E203,E501,W503,E731,E741
+per-file-ignores =
+    src/probflow/__init__.py:F401,F403

Makefile

@@ -21,6 +21,7 @@ test-pytorch:
 
 format:
 	. venv/bin/activate; \
+        autoflake -r --in-place --remove-all-unused-imports --ignore-init-module-imports src/probflow tests; \
         isort src/probflow tests; \
 	black src/probflow tests; \
 	flake8 src/probflow tests

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name="probflow",
-    version="2.1.0",
+    version="2.1.1",
     author="Brendan Hasz",
     author_email="winsto99@gmail.com",
     description="A Python package for building Bayesian models with TensorFlow or PyTorch",
@@ -41,6 +41,7 @@
             "torch >= 1.5.0",
         ],
         "dev": [
+            "autoflake >= 1.4",
             "black >= 19.10b0",
             "bumpversion",
             "flake8 >= 3.8.3",

src/probflow/__init__.py

@@ -9,4 +9,4 @@
 from probflow.utils.io import *
 from probflow.utils.settings import *
 
-__version__ = "2.1.0"
+__version__ = "2.1.1"

src/probflow/applications/linear_regression.py

@@ -1,5 +1,3 @@
-from typing import Callable, List, Union
-
 import probflow.utils.ops as O
 from probflow.distributions import Normal
 from probflow.models import ContinuousModel

src/probflow/callbacks/monitor_parameter.py

@@ -8,7 +8,7 @@ class MonitorParameter(Callback):
 
     """
 
-    def __init__(self, x, y=None, params=None):
+    def __init__(self, params):
 
         # Store metrics and epochs
         self.params = params

src/probflow/models/categorical_model.py

@@ -139,5 +139,4 @@ def calibration_curve(
         TODO: Docs...
 
         """
-        pass
         # TODO

src/probflow/models/model.py

@@ -895,5 +895,4 @@ def summary(self):
         model would have to add to it the observation dist
 
         """
-        pass
         # TODO

src/probflow/modules/dense.py

@@ -1,4 +1,3 @@
-import probflow.utils.ops as O
 from probflow.parameters import DeterministicParameter, Parameter
 from probflow.utils.settings import get_backend, get_flipout
 

src/probflow/modules/dense_network.py

@@ -1,4 +1,4 @@
-from typing import Callable, List, Union
+from typing import Callable, List
 
 import probflow.utils.ops as O
 

src/probflow/modules/embedding.py

@@ -1,10 +1,7 @@
-from typing import Callable, Dict, List, Type, Union
+from typing import List, Union
 
 import probflow.utils.ops as O
-from probflow.distributions import Deterministic, Normal
 from probflow.parameters import DeterministicParameter, Parameter
-from probflow.utils.base import BaseDistribution
-from probflow.utils.initializers import xavier
 
 from .module import Module
 

src/probflow/parameters/bounded_parameter.py

@@ -1,6 +1,5 @@
 import probflow.utils.ops as O
 from probflow.distributions import Normal
-from probflow.utils.casting import to_numpy
 from probflow.utils.initializers import scale_xavier, xavier
 
 from .parameter import Parameter

src/probflow/parameters/deterministic_parameter.py

@@ -1,4 +1,3 @@
-import probflow.utils.ops as O
 from probflow.distributions import Deterministic, Normal
 from probflow.utils.initializers import xavier
 

src/probflow/parameters/multivariate_normal_parameter.py

@@ -1,7 +1,6 @@
 import probflow.utils.ops as O
 from probflow.distributions import MultivariateNormal
 from probflow.utils.initializers import xavier
-from probflow.utils.settings import get_backend
 
 from .parameter import Parameter
 

src/probflow/utils/ops.py

@@ -253,8 +253,6 @@ def abs(val):
 def square(val):
     """Power of 2"""
     if get_backend() == "pytorch":
-        import torch
-
         return val ** 2
     else:
         import tensorflow as tf

src/probflow/utils/plotting.py

@@ -10,7 +10,6 @@
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-from matplotlib.colors import to_rgba
 
 COLORS = plt.rcParams["axes.prop_cycle"].by_key()["color"]
 

src/probflow/utils/torch_distributions.py

@@ -7,7 +7,7 @@ def get_TorchDeterministic():
 
     import torch
     from torch.distributions import constraints
-    from torch.distributions.distribution import Distribution
+    from torch.distributions.kl import register_kl
     from torch.distributions.utils import broadcast_all
 
     class TorchDeterministic(torch.distributions.distribution.Distribution):
@@ -77,4 +77,10 @@ def icdf(self, value):
         def entropy(self):
             return torch.log(torch.zeros([1]))
 
+    @register_kl(
+        TorchDeterministic, torch.distributions.distribution.Distribution
+    )
+    def kl_deterministic_continuous(p, q):
+        return -q.log_prob(p.mean)
+
     return TorchDeterministic

tests/examples/test_example_correlation.py

@@ -3,7 +3,6 @@
 
 import matplotlib.pyplot as plt
 import numpy as np
-import pytest
 import tensorflow as tf
 import tensorflow_probability as tfp
 

tests/examples/test_example_gan.py

@@ -1,9 +1,7 @@
 """Tests example GAN model"""
 
 
-import matplotlib.pyplot as plt
 import numpy as np
-import pytest
 import tensorflow as tf
 import tensorflow_probability as tfp
 

tests/examples/test_example_gmm.py

@@ -1,10 +1,7 @@
 """Tests exmaple gaussian mixture model"""
 
 
-import matplotlib.pyplot as plt
 import numpy as np
-import pytest
-import tensorflow as tf
 import tensorflow_probability as tfp
 
 import probflow as pf

tests/examples/test_example_heteroscedastic.py

@@ -1,9 +1,7 @@
 """Tests example robust heteroscedastic regression"""
 
 
-import matplotlib.pyplot as plt
 import numpy as np
-import pytest
 import tensorflow as tf
 import tensorflow_probability as tfp
 

tests/examples/test_example_linear_regression.py

@@ -3,7 +3,6 @@
 
 import numpy as np
 import pandas as pd
-import tensorflow as tf
 
 import probflow as pf
 

tests/examples/test_example_ppca.py

@@ -1,9 +1,7 @@
 """Tests example probabilistic PCA"""
 
 
-import matplotlib.pyplot as plt
 import numpy as np
-import pytest
 import tensorflow as tf
 import tensorflow_probability as tfp
 

tests/stats/test_LogisticRegression.py

@@ -2,7 +2,6 @@
 
 
 import numpy as np
-import tensorflow as tf
 
 import probflow as pf
 

tests/unit/pytorch/applications/test_dense_classifier.py

@@ -0,0 +1,42 @@
+import numpy as np
+
+from probflow.applications import DenseClassifier
+
+
+def test_DenseClassifier():
+    """Tests probflow.applications.DenseClassifier"""
+
+    # Data
+    x = np.random.randn(100, 5).astype("float32")
+    w = np.random.randn(5, 1).astype("float32")
+    y = x @ w + 1
+    y = np.round(1.0 / (1.0 + np.exp(-y))).astype("float32")
+
+    # Create the model
+    model = DenseClassifier([5, 20, 15, 2])
+
+    # Fit the model
+    model.fit(x, y, batch_size=10, epochs=3)
+
+    # Predictive functions
+    model.predict(x)
+
+
+def test_MultinomialDenseClassifier():
+    """Tests probflow.applications.DenseClassifier w/ >2 output classes"""
+
+    # Data
+    x = np.random.randn(100, 5).astype("float32")
+    w = np.random.randn(5, 3).astype("float32")
+    b = np.random.randn(1, 3).astype("float32")
+    y = x @ w + b
+    y = np.argmax(y, axis=1).astype("int32")
+
+    # Create the model
+    model = DenseClassifier([5, 20, 15, 3])
+
+    # Fit the model
+    model.fit(x, y, batch_size=10, epochs=3)
+
+    # Predictive functions
+    model.predict(x)

tests/unit/pytorch/applications/test_dense_regression.py

@@ -0,0 +1,103 @@
+import numpy as np
+
+from probflow.applications import DenseRegression
+
+
+def test_DenseRegression():
+    """Tests probflow.applications.DenseRegression"""
+
+    # Data
+    x = np.random.randn(100, 5).astype("float32")
+    w = np.random.randn(5, 1).astype("float32")
+    y = x @ w + 1
+
+    # Create the model
+    model = DenseRegression([5, 20, 15, 1])
+
+    # Fit the model
+    model.fit(x, y, batch_size=10, epochs=3)
+
+    # Predictive functions
+    preds = model.predict(x[:11, :])
+    assert isinstance(preds, np.ndarray)
+    assert preds.ndim == 2
+    assert preds.shape[0] == 11
+    assert preds.shape[1] == 1
+
+    # predictive interval
+    lb, ub = model.predictive_interval(x[:12, :], ci=0.9)
+    assert isinstance(lb, np.ndarray)
+    assert lb.ndim == 2
+    assert lb.shape[0] == 12
+    assert lb.shape[1] == 1
+    assert isinstance(ub, np.ndarray)
+    assert ub.ndim == 2
+    assert ub.shape[0] == 12
+    assert ub.shape[1] == 1
+
+
+def test_DenseRegression_heteroscedastic():
+    """Tests probflow.applications.DenseRegression w/ heteroscedastic"""
+
+    # Data
+    x = np.random.randn(100, 5).astype("float32")
+    w = np.random.randn(5, 1).astype("float32")
+    y = x @ w + 1
+    y = y + np.exp(y) * np.random.randn(100, 1).astype("float32")
+
+    # Create the model
+    model = DenseRegression([5, 20, 15, 1], heteroscedastic=True)
+
+    # Fit the model
+    model.fit(x, y, batch_size=10, epochs=3)
+
+    # Predictive functions
+    preds = model.predict(x[:11, :])
+    assert isinstance(preds, np.ndarray)
+    assert preds.ndim == 2
+    assert preds.shape[0] == 11
+    assert preds.shape[1] == 1
+
+    # predictive interval
+    lb, ub = model.predictive_interval(x[:12, :], ci=0.9)
+    assert isinstance(lb, np.ndarray)
+    assert lb.ndim == 2
+    assert lb.shape[0] == 12
+    assert lb.shape[1] == 1
+    assert isinstance(ub, np.ndarray)
+    assert ub.ndim == 2
+    assert ub.shape[0] == 12
+    assert ub.shape[1] == 1
+
+
+def test_DenseRegression_multivariate():
+    """Tests probflow.applications.DenseRegression w/ >1 output dims"""
+
+    # Data
+    N = 256
+    Di = 7
+    Do = 3
+    x = np.random.randn(N, Di).astype("float32")
+    w = np.random.randn(Di, Do).astype("float32")
+    y = x @ w + 0.1 * np.random.randn(N, Do).astype("float32")
+
+    # Create the model
+    model = DenseRegression([Di, 16, Do])
+
+    # Fit the model
+    model.fit(x, y, batch_size=128, epochs=3)
+
+    # Predictive functions
+    preds = model.predict(x[:11, :])
+    assert isinstance(preds, np.ndarray)
+    assert preds.ndim == 2
+    assert preds.shape[0] == 11
+    assert preds.shape[1] == Do
+
+    # Predictive functions
+    preds = model.predictive_sample(x[:11, :], n=13)
+    assert isinstance(preds, np.ndarray)
+    assert preds.ndim == 3
+    assert preds.shape[0] == 13
+    assert preds.shape[1] == 11
+    assert preds.shape[2] == Do