diff --git a/controllers/checkly/apicheck_controller.go b/controllers/checkly/apicheck_controller.go
index a3647d4..959a523 100644
--- a/controllers/checkly/apicheck_controller.go
+++ b/controllers/checkly/apicheck_controller.go
@@ -172,7 +172,7 @@ func (r *ApiCheckReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	checklyID, err := external.Create(internalCheck, r.ApiClient)
 	if err != nil {
 		logger.Error(err, "Failed to create checkly alert")
-		return ctrl.Result{}, nil
+		return ctrl.Result{}, err
 	}
 
 	// Update the custom resource Status with the returned ID
diff --git a/external/checkly/check.go b/external/checkly/check.go
index b0d291b..1315c4a 100644
--- a/external/checkly/check.go
+++ b/external/checkly/check.go
@@ -19,6 +19,7 @@ package external
 import (
 	"context"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/checkly/checkly-go-sdk"
@@ -37,7 +38,12 @@ type Check struct {
 	Muted           bool
 }
 
-func checklyCheck(apiCheck Check) (check checkly.Check) {
+func checklyCheck(apiCheck Check) (check checkly.Check, err error) {
+
+	shouldFail, err := shouldFail(apiCheck.SuccessCode)
+	if err != nil {
+		return
+	}
 
 	alertSettings := checkly.AlertSettings{
 		EscalationType: checkly.RunBased,
@@ -64,7 +70,7 @@ func checklyCheck(apiCheck Check) (check checkly.Check) {
 		MaxResponseTime:      checkValueInt(apiCheck.MaxResponseTime, 15000),
 		Activated:            true,
 		Muted:                apiCheck.Muted, // muted for development
-		ShouldFail:           false,
+		ShouldFail:           shouldFail,
 		DoubleCheck:          false,
 		SSLCheck:             false,
 		LocalSetupScript:     "",
@@ -110,7 +116,10 @@ func checklyCheck(apiCheck Check) (check checkly.Check) {
 // Create creates a new checklyhq.com check
 func Create(apiCheck Check, client checkly.Client) (ID string, err error) {
 
-	check := checklyCheck(apiCheck)
+	check, err := checklyCheck(apiCheck)
+	if err != nil {
+		return
+	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
@@ -128,7 +137,10 @@ func Create(apiCheck Check, client checkly.Client) (ID string, err error) {
 // Update updates an existing checklyhq.com check
 func Update(apiCheck Check, client checkly.Client) (err error) {
 
-	check := checklyCheck(apiCheck)
+	check, err := checklyCheck(apiCheck)
+	if err != nil {
+		return
+	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
@@ -148,3 +160,15 @@ func Delete(ID string, client checkly.Client) (err error) {
 
 	return
 }
+
+func shouldFail(successCode string) (bool, error) {
+	code, err := strconv.Atoi(successCode)
+	if err != nil {
+		return false, err
+	}
+	if code < 400 {
+		return false, nil
+	} else {
+		return true, nil
+	}
+}
diff --git a/external/checkly/check_test.go b/external/checkly/check_test.go
index fdf9c51..4c93032 100644
--- a/external/checkly/check_test.go
+++ b/external/checkly/check_test.go
@@ -26,49 +26,69 @@ import (
 
 func TestChecklyCheck(t *testing.T) {
 
-	data := Check{
+	data1 := Check{
 		Name:            "foo",
 		Namespace:       "bar",
 		Frequency:       15,
 		MaxResponseTime: 2000,
 		Endpoint:        "https://foo.bar/baz",
-		SuccessCode:     "200",
+		SuccessCode:     "403",
 		Muted:           true,
 	}
 
-	testData := checklyCheck(data)
+	testData, _ := checklyCheck(data1)
+
+	if testData.Name != data1.Name {
+		t.Errorf("Expected %s, got %s", data1.Name, testData.Name)
+	}
 
-	if testData.Name != data.Name {
-		t.Errorf("Expected %s, got %s", data.Name, testData.Name)
+	if testData.Frequency != data1.Frequency {
+		t.Errorf("Expected %d, got %d", data1.Frequency, testData.Frequency)
 	}
 
-	if testData.Frequency != data.Frequency {
-		t.Errorf("Expected %d, got %d", data.Frequency, testData.Frequency)
+	if testData.MaxResponseTime != data1.MaxResponseTime {
+		t.Errorf("Expected %d, got %d", data1.MaxResponseTime, testData.MaxResponseTime)
 	}
 
-	if testData.MaxResponseTime != data.MaxResponseTime {
-		t.Errorf("Expected %d, got %d", data.Frequency, testData.Frequency)
+	if testData.Muted != data1.Muted {
+		t.Errorf("Expected %t, got %t", data1.Muted, testData.Muted)
 	}
 
-	if testData.Muted != data.Muted {
-		t.Errorf("Expected %t, got %t", data.Muted, testData.Muted)
+	if testData.ShouldFail != true {
+		t.Errorf("Expected %t, got %t", true, testData.ShouldFail)
 	}
 
-	data = Check{
+	data2 := Check{
 		Name:        "foo",
 		Namespace:   "bar",
 		Endpoint:    "https://foo.bar/baz",
 		SuccessCode: "200",
 	}
 
-	testData = checklyCheck(data)
+	testData, _ = checklyCheck(data2)
 
 	if testData.Frequency != 5 {
-		t.Errorf("Expected %d, got %d", data.Frequency, testData.Frequency)
+		t.Errorf("Expected %d, got %d", 5, testData.Frequency)
 	}
 
 	if testData.MaxResponseTime != 15000 {
-		t.Errorf("Expected %d, got %d", data.Frequency, testData.Frequency)
+		t.Errorf("Expected %d, got %d", 15000, testData.MaxResponseTime)
+	}
+
+	if testData.ShouldFail != false {
+		t.Errorf("Expected %t, got %t", false, testData.ShouldFail)
+	}
+
+	failData := Check{
+		Name:        "fail",
+		Namespace:   "bar",
+		Endpoint:    "https://foo.bar/baz",
+		SuccessCode: "foo",
+	}
+
+	_, err := checklyCheck(failData)
+	if err == nil {
+		t.Error("Expected error, got nil")
 	}
 
 	return
@@ -199,3 +219,31 @@ func TestChecklyCheckActions(t *testing.T) {
 
 	return
 }
+
+func TestShouldFail(t *testing.T) {
+	testTrue := "401"
+	testFalse := "200"
+	testErr := "foo"
+
+	testResponse, err := shouldFail(testTrue)
+	if err != nil {
+		t.Errorf("Expected no error, got %e", err)
+	}
+	if testResponse != true {
+		t.Errorf("Expected true, got %t", testResponse)
+	}
+
+	testResponse, err = shouldFail(testFalse)
+	if err != nil {
+		t.Errorf("Expected no error, got %e", err)
+	}
+	if testResponse != false {
+		t.Errorf("Expected false, got %t", testResponse)
+	}
+
+	_, err = shouldFail(testErr)
+	if err == nil {
+		t.Errorf("Expected error, got none")
+	}
+
+}