add jobInfoLight API endpoint that excludes attempt information (#16682)

* add jobInfoLight API endpoint that excludes attempt information, which can be enormous as it includes all log lines * update replication activity to call new light endpoint
airbytehq · Sep 14, 2022 · 1d29672 · 1d29672
1 parent a0555ae
commit 1d29672
Show file tree

Hide file tree

Showing 8 changed files with 163 additions and 11 deletions.
diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml
@@ -2113,6 +2113,29 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+  /v1/jobs/get-light:
+    post:
+      tags:
+        - jobs
+      summary: Get information about a job excluding attempt info and logs
+      operationId: getJobInfoLight
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/JobIdRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JobInfoLightRead"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
+        "422":
+          $ref: "#/components/responses/InvalidInputResponse"
   /v1/jobs/cancel:
     post:
       tags:
@@ -4017,6 +4040,13 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/AttemptInfoRead"
+    JobInfoLightRead:
+      type: object
+      required:
+        - job
+      properties:
+        job:
+          $ref: "#/components/schemas/JobRead"
     JobDebugInfoRead:
       type: object
       required:

diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java
@@ -45,6 +45,7 @@
 import io.airbyte.api.model.generated.InternalOperationResult;
 import io.airbyte.api.model.generated.JobDebugInfoRead;
 import io.airbyte.api.model.generated.JobIdRequestBody;
+import io.airbyte.api.model.generated.JobInfoLightRead;
 import io.airbyte.api.model.generated.JobInfoRead;
 import io.airbyte.api.model.generated.JobListRequestBody;
 import io.airbyte.api.model.generated.JobReadList;
@@ -779,6 +780,11 @@ public JobInfoRead getJobInfo(final JobIdRequestBody jobIdRequestBody) {
     return execute(() -> jobHistoryHandler.getJobInfo(jobIdRequestBody));
   }
 
+  @Override
+  public JobInfoLightRead getJobInfoLight(final JobIdRequestBody jobIdRequestBody) {
+    return execute(() -> jobHistoryHandler.getJobInfoLight(jobIdRequestBody));
+  }
+
   @Override
   public JobDebugInfoRead getJobDebugInfo(final JobIdRequestBody jobIdRequestBody) {
     return execute(() -> jobHistoryHandler.getJobDebugInfo(jobIdRequestBody));

diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java
@@ -16,6 +16,7 @@
 import io.airbyte.api.model.generated.DestinationDefinitionRead;
 import io.airbyte.api.model.generated.JobConfigType;
 import io.airbyte.api.model.generated.JobDebugRead;
+import io.airbyte.api.model.generated.JobInfoLightRead;
 import io.airbyte.api.model.generated.JobInfoRead;
 import io.airbyte.api.model.generated.JobRead;
 import io.airbyte.api.model.generated.JobStatus;
@@ -65,6 +66,10 @@ public JobInfoRead getJobInfoRead(final Job job) {
         .attempts(job.getAttempts().stream().map(this::getAttemptInfoRead).collect(Collectors.toList()));
   }
 
+  public JobInfoLightRead getJobInfoLightRead(final Job job) {
+    return new JobInfoLightRead().job(getJobRead(job));
+  }
+
   public static JobDebugRead getDebugJobInfoRead(final JobInfoRead jobInfoRead,
                                                  final SourceDefinitionRead sourceDefinitionRead,
                                                  final DestinationDefinitionRead destinationDefinitionRead,
@@ -80,19 +85,23 @@ public static JobDebugRead getDebugJobInfoRead(final JobInfoRead jobInfoRead,
   }
 
   public static JobWithAttemptsRead getJobWithAttemptsRead(final Job job) {
+    return new JobWithAttemptsRead()
+        .job(getJobRead(job))
+        .attempts(job.getAttempts().stream().map(JobConverter::getAttemptRead).toList());
+  }
+
+  private static JobRead getJobRead(final Job job) {
     final String configId = job.getScope();
     final JobConfigType configType = Enums.convertTo(job.getConfigType(), JobConfigType.class);
 
-    return new JobWithAttemptsRead()
-        .job(new JobRead()
-            .id(job.getId())
-            .configId(configId)
-            .configType(configType)
-            .resetConfig(extractResetConfigIfReset(job).orElse(null))
-            .createdAt(job.getCreatedAtInSecond())
-            .updatedAt(job.getUpdatedAtInSecond())
-            .status(Enums.convertTo(job.getStatus(), JobStatus.class)))
-        .attempts(job.getAttempts().stream().map(JobConverter::getAttemptRead).toList());
+    return new JobRead()
+        .id(job.getId())
+        .configId(configId)
+        .configType(configType)
+        .resetConfig(extractResetConfigIfReset(job).orElse(null))
+        .createdAt(job.getCreatedAtInSecond())
+        .updatedAt(job.getUpdatedAtInSecond())
+        .status(Enums.convertTo(job.getStatus(), JobStatus.class));
   }
 
   /**

diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/JobHistoryHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/JobHistoryHandler.java
@@ -13,6 +13,7 @@
 import io.airbyte.api.model.generated.JobDebugInfoRead;
 import io.airbyte.api.model.generated.JobDebugRead;
 import io.airbyte.api.model.generated.JobIdRequestBody;
+import io.airbyte.api.model.generated.JobInfoLightRead;
 import io.airbyte.api.model.generated.JobInfoRead;
 import io.airbyte.api.model.generated.JobListRequestBody;
 import io.airbyte.api.model.generated.JobReadList;
@@ -106,6 +107,11 @@ public JobInfoRead getJobInfo(final JobIdRequestBody jobIdRequestBody) throws IO
     return jobConverter.getJobInfoRead(job);
   }
 
+  public JobInfoLightRead getJobInfoLight(final JobIdRequestBody jobIdRequestBody) throws IOException {
+    final Job job = jobPersistence.getJob(jobIdRequestBody.getId());
+    return jobConverter.getJobInfoLightRead(job);
+  }
+
   public JobDebugInfoRead getJobDebugInfo(final JobIdRequestBody jobIdRequestBody)
       throws ConfigNotFoundException, IOException, JsonValidationException {
     final Job job = jobPersistence.getJob(jobIdRequestBody.getId());

diff --git a/airbyte-server/src/test/java/io/airbyte/server/converters/JobConverterTest.java b/airbyte-server/src/test/java/io/airbyte/server/converters/JobConverterTest.java
@@ -22,6 +22,7 @@
 import io.airbyte.api.model.generated.DestinationDefinitionRead;
 import io.airbyte.api.model.generated.JobConfigType;
 import io.airbyte.api.model.generated.JobDebugRead;
+import io.airbyte.api.model.generated.JobInfoLightRead;
 import io.airbyte.api.model.generated.JobInfoRead;
 import io.airbyte.api.model.generated.JobRead;
 import io.airbyte.api.model.generated.JobWithAttemptsRead;
@@ -210,6 +211,12 @@ void testGetJobInfoRead() {
     assertEquals(JOB_INFO, jobConverter.getJobInfoRead(job));
   }
 
+  @Test
+  void testGetJobInfoLightRead() {
+    final JobInfoLightRead expected = new JobInfoLightRead().job(JOB_INFO.getJob());
+    assertEquals(expected, jobConverter.getJobInfoLightRead(job));
+  }
+
   @Test
   void testGetDebugJobInfoRead() {
     assertEquals(JOB_DEBUG_INFO, JobConverter.getDebugJobInfoRead(JOB_INFO, sourceDefinitionRead, destinationDefinitionRead, airbyteVersion));

diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java
@@ -256,6 +256,19 @@ void testGetJobInfo() throws IOException {
     assertEquals(exp, jobInfoActual);
   }
 
+  @Test
+  @DisplayName("Should return the right job info without attempt information")
+  void testGetJobInfoLight() throws IOException {
+    when(jobPersistence.getJob(JOB_ID)).thenReturn(testJob);
+
+    final JobIdRequestBody requestBody = new JobIdRequestBody().id(JOB_ID);
+    final JobInfoLightRead jobInfoLightActual = jobHistoryHandler.getJobInfoLight(requestBody);
+
+    final JobInfoLightRead exp = new JobInfoLightRead().job(toJobInfo(testJob));
+
+    assertEquals(exp, jobInfoLightActual);
+  }
+
   @Test
   @DisplayName("Should return the right info to debug this job")
   void testGetDebugJobInfo() throws IOException, JsonValidationException, ConfigNotFoundException, URISyntaxException {

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java
@@ -241,7 +241,7 @@ private CheckedSupplier<Worker<StandardSyncInput, ReplicationOutput>, Exception>
     final JobIdRequestBody id = new JobIdRequestBody();
     id.setId(Long.valueOf(jobRunConfig.getJobId()));
 
-    final var jobScope = airbyteApiClient.getJobsApi().getJobInfo(id).getJob().getConfigId();
+    final var jobScope = airbyteApiClient.getJobsApi().getJobInfoLight(id).getJob().getConfigId();
     final var connectionId = UUID.fromString(jobScope);
 
     return () -> new ReplicationLauncherWorker(

diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html
@@ -290,6 +290,7 @@ <h4><a href="#Jobs">Jobs</a></h4>
   <li><a href="#cancelJob"><code><span class="http-method">post</span> /v1/jobs/cancel</code></a></li>
   <li><a href="#getJobDebugInfo"><code><span class="http-method">post</span> /v1/jobs/get_debug_info</code></a></li>
   <li><a href="#getJobInfo"><code><span class="http-method">post</span> /v1/jobs/get</code></a></li>
+  <li><a href="#getJobInfoLight"><code><span class="http-method">post</span> /v1/jobs/get-light</code></a></li>
   <li><a href="#listJobsFor"><code><span class="http-method">post</span> /v1/jobs/list</code></a></li>
   </ul>
   <h4><a href="#Logs">Logs</a></h4>
@@ -4739,6 +4740,78 @@ <h4 class="field-label">422</h4>
         <a href="#InvalidInputExceptionInfo">InvalidInputExceptionInfo</a>
   </div> <!-- method -->
   <hr/>
+  <div class="method"><a name="getJobInfoLight"/>
+    <div class="method-path">
+    <a class="up" href="#__Methods">Up</a>
+    <pre class="post"><code class="huge"><span class="http-method">post</span> /v1/jobs/get-light</code></pre></div>
+    <div class="method-summary">Get information about a job excluding attempt info and logs (<span class="nickname">getJobInfoLight</span>)</div>
+    <div class="method-notes"></div>
+
+
+    <h3 class="field-label">Consumes</h3>
+    This API call consumes the following media types via the <span class="header">Content-Type</span> request header:
+    <ul>
+      <li><code>application/json</code></li>
+    </ul>
+
+    <h3 class="field-label">Request body</h3>
+    <div class="field-items">
+      <div class="param">JobIdRequestBody <a href="#JobIdRequestBody">JobIdRequestBody</a> (required)</div>
+
+      <div class="param-desc"><span class="param-type">Body Parameter</span> &mdash;  </div>
+
+    </div>  <!-- field-items -->
+
+
+
+
+    <h3 class="field-label">Return type</h3>
+    <div class="return-type">
+      <a href="#JobInfoLightRead">JobInfoLightRead</a>
+
+    </div>
+
+    <!--Todo: process Response Object and its headers, schema, examples -->
+
+    <h3 class="field-label">Example data</h3>
+    <div class="example-data-content-type">Content-Type: application/json</div>
+    <pre class="example"><code>{
+  "job" : {
+    "createdAt" : 6,
+    "configId" : "configId",
+    "id" : 0,
+    "resetConfig" : {
+      "streamsToReset" : [ {
+        "name" : "name",
+        "namespace" : "namespace"
+      }, {
+        "name" : "name",
+        "namespace" : "namespace"
+      } ]
+    },
+    "updatedAt" : 1
+  }
+}</code></pre>
+
+    <h3 class="field-label">Produces</h3>
+    This API call produces the following media types according to the <span class="header">Accept</span> request header;
+    the media type will be conveyed by the <span class="header">Content-Type</span> response header.
+    <ul>
+      <li><code>application/json</code></li>
+    </ul>
+
+    <h3 class="field-label">Responses</h3>
+    <h4 class="field-label">200</h4>
+    Successful operation
+        <a href="#JobInfoLightRead">JobInfoLightRead</a>
+    <h4 class="field-label">404</h4>
+    Object with given id was not found.
+        <a href="#NotFoundKnownExceptionInfo">NotFoundKnownExceptionInfo</a>
+    <h4 class="field-label">422</h4>
+    Input failed validation
+        <a href="#InvalidInputExceptionInfo">InvalidInputExceptionInfo</a>
+  </div> <!-- method -->
+  <hr/>
   <div class="method"><a name="listJobsFor"/>
     <div class="method-path">
     <a class="up" href="#__Methods">Up</a>
@@ -10609,6 +10682,7 @@ <h3>Table of Contents</h3>
     <li><a href="#JobDebugInfoRead"><code>JobDebugInfoRead</code> - </a></li>
     <li><a href="#JobDebugRead"><code>JobDebugRead</code> - </a></li>
     <li><a href="#JobIdRequestBody"><code>JobIdRequestBody</code> - </a></li>
+    <li><a href="#JobInfoLightRead"><code>JobInfoLightRead</code> - </a></li>
     <li><a href="#JobInfoRead"><code>JobInfoRead</code> - </a></li>
     <li><a href="#JobListRequestBody"><code>JobListRequestBody</code> - </a></li>
     <li><a href="#JobRead"><code>JobRead</code> - </a></li>
@@ -11440,6 +11514,13 @@ <h3><a name="JobIdRequestBody"><code>JobIdRequestBody</code> - </a> <a class="up
       <div class="param">id </div><div class="param-desc"><span class="param-type"><a href="#long">Long</a></span>  format: int64</div>
     </div>  <!-- field-items -->
   </div>
+  <div class="model">
+    <h3><a name="JobInfoLightRead"><code>JobInfoLightRead</code> - </a> <a class="up" href="#__Models">Up</a></h3>
+    <div class='model-description'></div>
+    <div class="field-items">
+      <div class="param">job </div><div class="param-desc"><span class="param-type"><a href="#JobRead">JobRead</a></span>  </div>
+    </div>  <!-- field-items -->
+  </div>
   <div class="model">
     <h3><a name="JobInfoRead"><code>JobInfoRead</code> - </a> <a class="up" href="#__Models">Up</a></h3>
     <div class='model-description'></div>