Merge branch 'main' into feat/composition-default-authmode

algolia · Nov 18, 2024 · 0e97d38 · 0e97d38
2 parents 98eb49e + aae4ddb
commit 0e97d38
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 8 deletions.
diff --git a/specs/bundled/crawler.doc.yml b/specs/bundled/crawler.doc.yml
@@ -2881,8 +2881,8 @@ components:
     beforeIndexPublishing:
       type: object
       description: >-
-        Checks triggered after the crawl finishes and before the records are
-        added to the Algolia index.
+        These checks are triggered after the crawl finishes but before the
+        records are added to the Algolia index.
       properties:
         maxLostRecordsPercentage:
           type: number
@@ -2900,9 +2900,14 @@ components:
           minimum: 1
           maximum: 100
           default: 10
+        maxFailedUrls:
+          type: number
+          description: |
+            Stops the crawler if a specified number of pages fail to crawl.
+            If undefined, the crawler won't stop if it encounters such errors.
     safetyChecks:
       type: object
-      description: Safety checks for ensuring data integrity between crawls.
+      description: Checks to ensure the crawl was successful.
       properties:
         beforeIndexPublishing:
           $ref: '#/components/schemas/beforeIndexPublishing'

diff --git a/specs/bundled/crawler.yml b/specs/bundled/crawler.yml
@@ -2881,8 +2881,8 @@ components:
     beforeIndexPublishing:
       type: object
       description: >-
-        Checks triggered after the crawl finishes and before the records are
-        added to the Algolia index.
+        These checks are triggered after the crawl finishes but before the
+        records are added to the Algolia index.
       properties:
         maxLostRecordsPercentage:
           type: number
@@ -2900,9 +2900,14 @@ components:
           minimum: 1
           maximum: 100
           default: 10
+        maxFailedUrls:
+          type: number
+          description: |
+            Stops the crawler if a specified number of pages fail to crawl.
+            If undefined, the crawler won't stop if it encounters such errors.
     safetyChecks:
       type: object
-      description: Safety checks for ensuring data integrity between crawls.
+      description: Checks to ensure the crawl was successful.
       properties:
         beforeIndexPublishing:
           $ref: '#/components/schemas/beforeIndexPublishing'

diff --git a/specs/crawler/common/schemas/configuration.yml b/specs/crawler/common/schemas/configuration.yml
@@ -444,14 +444,14 @@ extraParameters:
 
 safetyChecks:
   type: object
-  description: Safety checks for ensuring data integrity between crawls.
+  description: Checks to ensure the crawl was successful.
   properties:
     beforeIndexPublishing:
       $ref: '#/beforeIndexPublishing'
 
 beforeIndexPublishing:
   type: object
-  description: Checks triggered after the crawl finishes and before the records are added to the Algolia index.
+  description: These checks are triggered after the crawl finishes but before the records are added to the Algolia index.
   properties:
     maxLostRecordsPercentage:
       type: number
@@ -464,6 +464,11 @@ beforeIndexPublishing:
       minimum: 1
       maximum: 100
       default: 10
+    maxFailedUrls:
+      type: number
+      description: |
+        Stops the crawler if a specified number of pages fail to crawl.
+        If undefined, the crawler won't stop if it encounters such errors.
 
 schedule:
   type: string