diff --git a/doc/source/whatsnew/v1.5.1.rst b/doc/source/whatsnew/v1.5.1.rst
index a8ec836ba93df..4f4898fe64e35 100644
--- a/doc/source/whatsnew/v1.5.1.rst
+++ b/doc/source/whatsnew/v1.5.1.rst
@@ -91,7 +91,7 @@ Bug fixes
 - Bug in :func:`assert_index_equal` for extension arrays with non matching ``NA`` raising ``ValueError`` (:issue:`48608`)
 - Bug in :meth:`DataFrame.pivot_table` raising unexpected ``FutureWarning`` when setting datetime column as index (:issue:`48683`)
 - Bug in :meth:`DataFrame.sort_values` emitting unnecessary ``FutureWarning`` when called on :class:`DataFrame` with boolean sparse columns (:issue:`48784`)
--
+- Bug in :class:`.arrays.ArrowExtensionArray` with a comparison operator to an invalid object would not raise a ``NotImplementedError`` (:issue:`48833`)
 
 .. ---------------------------------------------------------------------------
 
diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py
index 03f734730a0f9..31b34e557531c 100644
--- a/pandas/core/arrays/arrow/array.py
+++ b/pandas/core/arrays/arrow/array.py
@@ -391,7 +391,7 @@ def _cmp_method(self, other, op):
                 result[valid] = op(np.array(self)[valid], other)
                 return BooleanArray(result, mask)
         else:
-            return NotImplementedError(
+            raise NotImplementedError(
                 f"{op.__name__} not implemented for {type(other)}"
             )
 
diff --git a/pandas/tests/extension/test_arrow.py b/pandas/tests/extension/test_arrow.py
index e6bf8569ef007..0c8221cb73eee 100644
--- a/pandas/tests/extension/test_arrow.py
+++ b/pandas/tests/extension/test_arrow.py
@@ -1669,6 +1669,13 @@ def test_compare_array(self, data, comparison_op, na_value, request):
                 with pytest.raises(type(exc)):
                     ser.combine(other, comparison_op)
 
+    def test_invalid_other_comp(self, data, comparison_op):
+        # GH 48833
+        with pytest.raises(
+            NotImplementedError, match=".* not implemented for <class 'object'>"
+        ):
+            comparison_op(data, object())
+
 
 def test_arrowdtype_construct_from_string_type_with_unsupported_parameters():
     with pytest.raises(NotImplementedError, match="Passing pyarrow type"):