diff --git a/pandas/plotting/_core.py b/pandas/plotting/_core.py
index c3476d1443fc3a..a7ee4897c020c2 100644
--- a/pandas/plotting/_core.py
+++ b/pandas/plotting/_core.py
@@ -774,6 +774,11 @@ def __init__(self, data, x, y, **kwargs):
             x = self.data.columns[x]
         if is_integer(y) and not self.data.columns.holds_integer():
             y = self.data.columns[y]
+        if len(self.data[x]._get_numeric_data()) == 0:
+            raise ValueError(self._kind + ' requires x column to be numeric')
+        if len(self.data[y]._get_numeric_data()) == 0:
+            raise ValueError(self._kind + ' requires y column to be numeric')
+
         self.x = x
         self.y = y
 
diff --git a/pandas/tests/plotting/test_frame.py b/pandas/tests/plotting/test_frame.py
index 03bc477d6f852e..607f1f1cbfe8c8 100644
--- a/pandas/tests/plotting/test_frame.py
+++ b/pandas/tests/plotting/test_frame.py
@@ -909,6 +909,24 @@ def test_plot_scatter(self):
         axes = df.plot(x='x', y='y', kind='scatter', subplots=True)
         self._check_axes_shape(axes, axes_num=1, layout=(1, 1))
 
+    @slow
+    def test_plot_scatter_with_categorical_data(self):
+        # GH 16199
+        df = pd.DataFrame({'x': [1, 2, 3, 4],
+                           'y': pd.Categorical(['a', 'b', 'a', 'c'])})
+
+        with pytest.raises(ValueError) as ve:
+            df.plot(x='x', y='y', kind='scatter')
+        ve.match('requires y column to be numeric')
+
+        with pytest.raises(ValueError) as ve:
+            df.plot(x='y', y='x', kind='scatter')
+        ve.match('requires x column to be numeric')
+
+        with pytest.raises(ValueError) as ve:
+            df.plot(x='y', y='y', kind='scatter')
+        ve.match('requires x column to be numeric')
+
     @slow
     def test_plot_scatter_with_c(self):
         df = DataFrame(randn(6, 4),