deploy: eb01803

cubed-dev · Nov 20, 2024 · 8b18d94 · 8b18d94
1 parent 3430729
commit 8b18d94
Show file tree

Hide file tree

Showing 55 changed files with 106 additions and 106 deletions.
diff --git a/.doctrees/array-api.doctree b/.doctrees/array-api.doctree
diff --git a/.doctrees/configuration.doctree b/.doctrees/configuration.doctree
diff --git a/.doctrees/design.doctree b/.doctrees/design.doctree
diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle
diff --git a/.doctrees/user-guide/reliability.doctree b/.doctrees/user-guide/reliability.doctree
diff --git a/_sources/configuration.md b/_sources/configuration.md
@@ -118,7 +118,7 @@ since it is deliberately designed not to have anything except the most basic fea
 | Property                     | Default | Description                                                                                        |
 |------------------------------|---------|----------------------------------------------------------------------------------------------------|
 | `retries`                    | 2       | The number of times to retry a task if it fails.                                                   |
-| `use_backups`                | `True`  | Whether to use backup tasks for mitigating stragglers.                                             |
+| `use_backups`                |         | Whether to use backup tasks for mitigating stragglers. Defaults to `True` only if `work_dir` is a filesystem supporting atomic writes (currently a cloud store like S3 or GCS). |
 | `batch_size`                 | `None`  | Number of input tasks to submit to be run in parallel. The default is not to batch.                |
 | `compute_arrays_in_parallel` | `False` | Whether arrays are computed one at a time or in parallel.                                          |
 | `max_workers`                | `None`  | The maximum number of workers to use in the `ThreadPoolExecutor`. Defaults to number of CPU cores. |
@@ -128,7 +128,7 @@ since it is deliberately designed not to have anything except the most basic fea
 
 | Property                     | Default | Description                                                                                                                                |
 |------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------|
-| `use_backups`                | `True`  | Whether to use backup tasks for mitigating stragglers.                                                                                     |
+| `use_backups`                |         | Whether to use backup tasks for mitigating stragglers. Defaults to `True` only if `work_dir` is a filesystem supporting atomic writes (currently a cloud store like S3 or GCS). |
 | `batch_size`                 | `None`  | Number of input tasks to submit to be run in parallel. `None` means don't batch.                                                           |
 | `compute_arrays_in_parallel` | `False` | Whether arrays are computed one at a time or in parallel.                                                                                  |
 | `max_workers`                | `None`  | The maximum number of workers to use in the `ProcessPoolExecutor`. Defaults to number of CPU cores.                                        |
@@ -154,7 +154,7 @@ Note that there is currently no way to set retries or a timeout for the Coiled e
 | Property                     | Default | Description                                                                                                                     |
 |------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------|
 | `retries`                    | 2       | The number of times to retry a task if it fails.                                                                                |
-| `use_backups`                | `True`  | Whether to use backup tasks for mitigating stragglers.                                                                          |
+| `use_backups`                |         | Whether to use backup tasks for mitigating stragglers. Defaults to `True` only if `work_dir` is a filesystem supporting atomic writes (currently a cloud store like S3 or GCS). |
 | `batch_size`                 | `None`  | Number of input tasks to submit to be run in parallel. The default is not to batch.                                             |
 | `compute_arrays_in_parallel` | `False` | Whether arrays are computed one at a time or in parallel.                                                                       |
 | `compute_kwargs`             | `None`  | Keyword arguments to pass to Dask's [`distributed.Client`](https://distributed.dask.org/en/latest/api.html#client) constructor. |
@@ -167,7 +167,7 @@ Note that there is currently no way to set a timeout for the Dask executor.
 |------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `retries`                    | 2       | The number of times to retry a task if it fails.                                                                                                                                                                                                                                                                                                                  |
 | `timeout`                    | `None`  | Tasks that take longer than the timeout will be automatically killed and retried. Defaults to the timeout specified when [deploying the lithops runtime image](https://lithops-cloud.github.io/docs/source/cli.html#lithops-runtime-deploy-runtime-name). This is 180 seconds in the [examples](https://github.com/cubed-dev/cubed/blob/main/examples/README.md). |
-| `use_backups`                | `True`  | Whether to use backup tasks for mitigating stragglers.                                                                                                                                                                                                                                                                                                            |
+| `use_backups`                |         | Whether to use backup tasks for mitigating stragglers. Defaults to `True` only if `work_dir` is a filesystem supporting atomic writes (currently a cloud store like S3 or GCS). |
 | `compute_arrays_in_parallel` | `False` | Whether arrays are computed one at a time or in parallel.                                                                                                                                                                                                                                                                                                         |
 | Other properties             | N/A     | Other properties will be passed as keyword arguments to the [`lithops.executors.FunctionExecutor`](https://lithops-cloud.github.io/docs/source/api_futures.html#lithops.executors.FunctionExecutor) constructor.                                                                                                                                                  |
 

diff --git a/_sources/user-guide/reliability.md b/_sources/user-guide/reliability.md
@@ -23,4 +23,4 @@ A few slow running tasks (called stragglers) can disproportionately slow down th
 
 When a backup task is launched the original task is not cancelled, so it is to be expected that both tasks will complete and write their (identical) output. This only works since tasks are idempotent and each write a single, whole Zarr chunk in an atomic operation. (Updates to a single key are atomic in both [Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html#ConsistencyModel) and Google Cloud Storage.)
 
-Backup tasks are enabled by default, but if you need to turn them off you can do so with ``use_backups=False``.
+Backup tasks are only enabled by default on filesystems supporting atomic writes, which currently includes cloud stores like S3 or GCS. You can turn backup tasks off completely on cloud stores by setting ``use_backups=False``.
diff --git a/_static/documentation_options.js b/_static/documentation_options.js
@@ -1,5 +1,5 @@
 const DOCUMENTATION_OPTIONS = {
-    VERSION: '5879d09',
+    VERSION: 'eb01803',
     LANGUAGE: 'en',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',

diff --git a/api.html b/api.html
@@ -39,7 +39,7 @@
   <link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
 <link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
 
-    <script src="_static/documentation_options.js?v=29f738cd"></script>
+    <script src="_static/documentation_options.js?v=ce75a5e6"></script>
     <script src="_static/doctools.js?v=9bcbadda"></script>
     <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
@@ -50,7 +50,7 @@
     <link rel="prev" title="Diagnostics" href="user-guide/diagnostics.html" />
   <meta name="viewport" content="width=device-width, initial-scale=1"/>
   <meta name="docsearch:language" content="en"/>
-  <meta name="docsearch:version" content="5879d09
+  <meta name="docsearch:version" content="eb01803
 " />
   </head>
 

diff --git a/array-api.html b/array-api.html
@@ -39,7 +39,7 @@
   <link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
 <link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
 
-    <script src="_static/documentation_options.js?v=29f738cd"></script>
+    <script src="_static/documentation_options.js?v=ce75a5e6"></script>
     <script src="_static/doctools.js?v=9bcbadda"></script>
     <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
@@ -50,7 +50,7 @@
     <link rel="prev" title="cubed.measure_reserved_mem" href="generated/cubed.measure_reserved_mem.html" />
   <meta name="viewport" content="width=device-width, initial-scale=1"/>
   <meta name="docsearch:language" content="en"/>
-  <meta name="docsearch:version" content="5879d09
+  <meta name="docsearch:version" content="eb01803
 " />
   </head>
 

diff --git a/computation.html b/computation.html
@@ -39,7 +39,7 @@
   <link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
 <link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
 
-    <script src="_static/documentation_options.js?v=29f738cd"></script>
+    <script src="_static/documentation_options.js?v=ce75a5e6"></script>
     <script src="_static/doctools.js?v=9bcbadda"></script>
     <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
@@ -50,7 +50,7 @@
     <link rel="prev" title="Operations" href="operations.html" />
   <meta name="viewport" content="width=device-width, initial-scale=1"/>
   <meta name="docsearch:language" content="en"/>
-  <meta name="docsearch:version" content="5879d09
+  <meta name="docsearch:version" content="eb01803
 " />
   </head>
 

diff --git a/configuration.html b/configuration.html
@@ -39,7 +39,7 @@
   <link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
 <link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
 
-    <script src="_static/documentation_options.js?v=29f738cd"></script>
+    <script src="_static/documentation_options.js?v=ce75a5e6"></script>
     <script src="_static/doctools.js?v=9bcbadda"></script>
     <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
@@ -50,7 +50,7 @@
     <link rel="prev" title="Python Array API" href="array-api.html" />
   <meta name="viewport" content="width=device-width, initial-scale=1"/>
   <meta name="docsearch:language" content="en"/>
-  <meta name="docsearch:version" content="5879d09
+  <meta name="docsearch:version" content="eb01803
 " />
   </head>
 
@@ -501,8 +501,8 @@ <h4><code class="docutils literal notranslate"><span class="pre">threads</span><
 <td><p>The number of times to retry a task if it fails.</p></td>
 </tr>
 <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">use_backups</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p></td>
-<td><p>Whether to use backup tasks for mitigating stragglers.</p></td>
+<td><p></p></td>
+<td><p>Whether to use backup tasks for mitigating stragglers. Defaults to <code class="docutils literal notranslate"><span class="pre">True</span></code> only if <code class="docutils literal notranslate"><span class="pre">work_dir</span></code> is a filesystem supporting atomic writes (currently a cloud store like S3 or GCS).</p></td>
 </tr>
 <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">batch_size</span></code></p></td>
 <td><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p></td>
@@ -531,8 +531,8 @@ <h4><code class="docutils literal notranslate"><span class="pre">processes</span
 </thead>
 <tbody>
 <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">use_backups</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p></td>
-<td><p>Whether to use backup tasks for mitigating stragglers.</p></td>
+<td><p></p></td>
+<td><p>Whether to use backup tasks for mitigating stragglers. Defaults to <code class="docutils literal notranslate"><span class="pre">True</span></code> only if <code class="docutils literal notranslate"><span class="pre">work_dir</span></code> is a filesystem supporting atomic writes (currently a cloud store like S3 or GCS).</p></td>
 </tr>
 <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">batch_size</span></code></p></td>
 <td><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p></td>
@@ -594,8 +594,8 @@ <h4><code class="docutils literal notranslate"><span class="pre">dask</span></co
 <td><p>The number of times to retry a task if it fails.</p></td>
 </tr>
 <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">use_backups</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p></td>
-<td><p>Whether to use backup tasks for mitigating stragglers.</p></td>
+<td><p></p></td>
+<td><p>Whether to use backup tasks for mitigating stragglers. Defaults to <code class="docutils literal notranslate"><span class="pre">True</span></code> only if <code class="docutils literal notranslate"><span class="pre">work_dir</span></code> is a filesystem supporting atomic writes (currently a cloud store like S3 or GCS).</p></td>
 </tr>
 <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">batch_size</span></code></p></td>
 <td><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p></td>
@@ -633,8 +633,8 @@ <h4><code class="docutils literal notranslate"><span class="pre">lithops</span><
 <td><p>Tasks that take longer than the timeout will be automatically killed and retried. Defaults to the timeout specified when <a class="reference external" href="https://lithops-cloud.github.io/docs/source/cli.html#lithops-runtime-deploy-runtime-name">deploying the lithops runtime image</a>. This is 180 seconds in the <a class="reference external" href="https://github.com/cubed-dev/cubed/blob/main/examples/README.md">examples</a>.</p></td>
 </tr>
 <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">use_backups</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p></td>
-<td><p>Whether to use backup tasks for mitigating stragglers.</p></td>
+<td><p></p></td>
+<td><p>Whether to use backup tasks for mitigating stragglers. Defaults to <code class="docutils literal notranslate"><span class="pre">True</span></code> only if <code class="docutils literal notranslate"><span class="pre">work_dir</span></code> is a filesystem supporting atomic writes (currently a cloud store like S3 or GCS).</p></td>
 </tr>
 <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">compute_arrays_in_parallel</span></code></p></td>
 <td><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p></td>

diff --git a/contributing.html b/contributing.html
@@ -39,7 +39,7 @@
   <link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
 <link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
 
-    <script src="_static/documentation_options.js?v=29f738cd"></script>
+    <script src="_static/documentation_options.js?v=ce75a5e6"></script>
     <script src="_static/doctools.js?v=9bcbadda"></script>
     <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
@@ -49,7 +49,7 @@
     <link rel="prev" title="Computation" href="computation.html" />
   <meta name="viewport" content="width=device-width, initial-scale=1"/>
   <meta name="docsearch:language" content="en"/>
-  <meta name="docsearch:version" content="5879d09
+  <meta name="docsearch:version" content="eb01803
 " />
   </head>
 

diff --git a/design.html b/design.html
@@ -39,7 +39,7 @@
   <link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
 <link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
 
-    <script src="_static/documentation_options.js?v=29f738cd"></script>
+    <script src="_static/documentation_options.js?v=ce75a5e6"></script>
     <script src="_static/doctools.js?v=9bcbadda"></script>
     <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
@@ -50,7 +50,7 @@
     <link rel="prev" title="Related Projects" href="related-projects.html" />
   <meta name="viewport" content="width=device-width, initial-scale=1"/>
   <meta name="docsearch:language" content="en"/>
-  <meta name="docsearch:version" content="5879d09
+  <meta name="docsearch:version" content="eb01803
 " />
   </head>
 

diff --git a/generated/cubed.Array.compute.html b/generated/cubed.Array.compute.html
@@ -39,7 +39,7 @@
   <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
 <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
 
-    <script src="../_static/documentation_options.js?v=29f738cd"></script>
+    <script src="../_static/documentation_options.js?v=ce75a5e6"></script>
     <script src="../_static/doctools.js?v=9bcbadda"></script>
     <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="../_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
@@ -50,7 +50,7 @@
     <link rel="prev" title="cubed.Array" href="cubed.Array.html" />
   <meta name="viewport" content="width=device-width, initial-scale=1"/>
   <meta name="docsearch:language" content="en"/>
-  <meta name="docsearch:version" content="5879d09
+  <meta name="docsearch:version" content="eb01803
 " />
   </head>
 

diff --git a/generated/cubed.Array.html b/generated/cubed.Array.html
@@ -39,7 +39,7 @@
   <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
 <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
 
-    <script src="../_static/documentation_options.js?v=29f738cd"></script>
+    <script src="../_static/documentation_options.js?v=ce75a5e6"></script>
     <script src="../_static/doctools.js?v=9bcbadda"></script>
     <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="../_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
@@ -50,7 +50,7 @@
     <link rel="prev" title="API Reference" href="../api.html" />
   <meta name="viewport" content="width=device-width, initial-scale=1"/>
   <meta name="docsearch:language" content="en"/>
-  <meta name="docsearch:version" content="5879d09
+  <meta name="docsearch:version" content="eb01803
 " />
   </head>