set last_successful_status

src/main/java/org/cdlib/mrt/zk/Job.java

@@ -222,9 +222,23 @@ public static Job createJob(ZooKeeper client, String bid, int priority, JSONObje
     }
   }
 
-  public JSONObject statusObject(IngestState status) {
-    JSONObject jobj = super.statusObject(status);
-    jobj.put(MerrittJsonKey.LastSuccessfulStatus.key(), JSONObject.NULL);
+  public JSONObject statusObject(JSONObject statj, IngestState status) {
+    JSONObject jobj = super.statusObject(statj, status);
+    if (!jobj.has(MerrittJsonKey.LastSuccessfulStatus.key())) {
+      jobj.put(MerrittJsonKey.LastSuccessfulStatus.key(), JSONObject.NULL);
+    }
+
+    JobState oldstat = null;
+    if (statj.has(MerrittJsonKey.Status.key())) {
+      oldstat = JobState.valueOf(statj.get(MerrittJsonKey.Status.key()).toString());
+    }
+
+    if (oldstat == null) {
+    } else if (status == JobState.Failed || status == JobState.Deleted) {
+    } else if (status == oldstat) {
+    } else {
+      jobj.put(MerrittJsonKey.LastSuccessfulStatus.key(), statj.get(MerrittJsonKey.Status.key()));
+    }
     jobj.put(MerrittJsonKey.RetryCount.key(), retryCount);
     return jobj;
   }

src/main/java/org/cdlib/mrt/zk/QueueItem.java

@@ -233,8 +233,8 @@ public void createOrSetData(ZooKeeper client, ZKKey key, Object data) throws Kee
     }
   }
 
-  public JSONObject statusObject(IngestState status) {
-    JSONObject jobj = new JSONObject();
+  public JSONObject statusObject(JSONObject statj, IngestState status) {
+    JSONObject jobj = statj == null ? new JSONObject() : new JSONObject(statj.toString());
     jobj.put(MerrittJsonKey.Status.key(), status.name());
     jobj.put(MerrittJsonKey.LastModified.key(), QueueItemHelper.now());
     return jobj;
@@ -255,7 +255,8 @@ public void setStatus(ZooKeeper client, IngestState status, String message) thro
         }  
       }
       String statpath = makePath(ZKKey.STATUS);
-      JSONObject json = statusObject(status);
+      JSONObject oldjson = QueueItemHelper.exists(client, statpath) ? jsonProperty(client, ZKKey.STATUS) : new JSONObject();
+      JSONObject json = statusObject(oldjson, status);
       if (message == null) {
         message = "";
       }

src/main/ruby/lib/merritt_zk_job.rb

@@ -142,13 +142,21 @@ def self.create_job(zk, bid, data, priority: 5, identifiers: {}, metadata: {})
       job
     end
 
-    def status_object(status)
-      {
-        status: status.name,
-        last_successful_status: nil,
-        last_modified: Time.now.to_s,
-        retry_count: @retry_count
-      }
+    def status_object(oldstat, status)
+      oldstatus = oldstat.nil? ? nil : oldstat[:status]
+      jobj = super(oldstat, status)
+
+      jobj[:last_successful_status] = nil unless jobj.key?(:last_successful_status)
+
+      if oldstatus.nil?
+      elsif status.nil?
+      elsif status == MerrittZK::JobState::Failed || status == MerrittZK::JobState::Deleted
+      elsif status.name == oldstatus
+      else
+        jobj[:last_successful_status] = oldstatus
+      end
+      jobj[:retry_count] = @retry_count
+      jobj
     end
 
     def self.acquire_job(zk, state)

src/main/ruby/lib/merritt_zk_queue_item.rb

@@ -111,17 +111,22 @@ def self.create_id(zk, prefix)
       path.split('/')[-1]
     end
 
-    def status_object(status)
-      {
-        status: status.name,
+    def status_object(oldstat, status)
+      defs = {
         last_modified: Time.now.to_s
       }
+      stat = oldstat.nil? ? defs : oldstat
+      stat[:status] = status.name
+      stat
     end
 
     def set_status(zk, status, message = '')
       return if status == @status
 
-      json = status_object(status)
+      oldjson = nil
+      oldjson = json_property(zk, ZkKeys::STATUS) if zk.exists?("#{path}/#{ZkKeys::STATUS}")
+
+      json = status_object(oldjson, status)
       json[:message] = message
       data = QueueItem.serialize(json)
       if @status.nil?

test-cases.yml

@@ -162,7 +162,7 @@ create_job_state_change:
     /jobs/jid0/priority: 5
     /jobs/jid0/space_needed: 0
     /jobs/jid0/status: 
-      last_successful_status: null
+      last_successful_status: Pending
       status: Estimating
       last_modified: now
       retry_count: 0
@@ -187,7 +187,7 @@ load_job_state_change:
     /jobs/jid0/priority: 5
     /jobs/jid0/space_needed: 0
     /jobs/jid0/status: 
-      last_successful_status: null
+      last_successful_status: Pending
       status: Estimating
       last_modified: now
       retry_count: 0
@@ -212,7 +212,7 @@ acquire_pending_job:
     /jobs/jid0/priority: 5
     /jobs/jid0/space_needed: 0
     /jobs/jid0/status: 
-      last_successful_status: null
+      last_successful_status: Pending
       status: Estimating
       last_modified: now
       retry_count: 0
@@ -251,7 +251,7 @@ acquire_lowest_priority_job:
     /jobs/jid1/priority: 2
     /jobs/jid1/space_needed: 0
     /jobs/jid1/status: 
-      last_successful_status: null
+      last_successful_status: Pending
       status: Estimating
       last_modified: now
       retry_count: 0
@@ -301,7 +301,7 @@ job_happy_path:
     /jobs/jid1/priority: 2
     /jobs/jid1/space_needed: 0
     /jobs/jid1/status: 
-      last_successful_status: null
+      last_successful_status: Notify
       status: Completed
       last_modified: now
       retry_count: 0
@@ -341,7 +341,7 @@ batch_happy_path:
     /jobs/jid1/priority: 2
     /jobs/jid1/space_needed: 0
     /jobs/jid1/status: 
-      last_successful_status: null
+      last_successful_status: Notify
       status: Completed
       last_modified: now
       retry_count: 0
@@ -367,7 +367,7 @@ batch_failure:
     /jobs/jid1/priority: 2
     /jobs/jid1/space_needed: 0
     /jobs/jid1/status: 
-      last_successful_status: null
+      last_successful_status: Recording
       status: Deleted
       last_modified: now
       retry_count: 0
@@ -393,7 +393,7 @@ batch_recovery:
     /jobs/jid1/priority: 2
     /jobs/jid1/space_needed: 0
     /jobs/jid1/status: 
-      last_successful_status: null
+      last_successful_status: Notify
       status: Completed
       last_modified: now
       retry_count: 1