Skip to content

Commit

Permalink
Fix downstrem bug in IL
Browse files Browse the repository at this point in the history
bug: after deleting dataset, the event of deleting job can not be sent to LC.

Signed-off-by: JimmyYang20 <yangjin39@huawei.com>
  • Loading branch information
JimmyYang20 committed Mar 17, 2022
1 parent 527c574 commit 50b1d45
Showing 1 changed file with 14 additions and 8 deletions.
22 changes: 14 additions & 8 deletions pkg/globalmanager/controllers/incrementallearning/downstream.go
Original file line number Diff line number Diff line change
@@ -62,18 +62,15 @@ func (c *Controller) syncToEdge(eventType watch.EventType, obj interface{}) erro
// more details at https://github.com/kubernetes/kubernetes/issues/3030
job.Kind = KindName

jobConditions := job.Status.Conditions
if len(jobConditions) == 0 {
return nil
}

dataName := job.Spec.Dataset.Name
// LC has dataset object on this node that may call dataset node
var dsNodeName string
ds, err := c.client.Datasets(job.Namespace).Get(context.TODO(), dataName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("dataset(%s/%s) not found", job.Namespace, dataName)
klog.Errorf("not found job(name=%s/%s)'s dataset, error: %v", job.Kind, job.Name, err)
} else {
dsNodeName = ds.Spec.NodeName
}
// LC has dataset object on this node that may call dataset node
dsNodeName := ds.Spec.NodeName

var trainNodeName string
var evalNodeName string
@@ -102,6 +99,15 @@ func (c *Controller) syncToEdge(eventType watch.EventType, obj interface{}) erro
return nil
}

if dsNodeName == "" {
return nil
}

jobConditions := job.Status.Conditions
if len(jobConditions) == 0 {
return nil
}

latestCondition := jobConditions[len(jobConditions)-1]
currentType := latestCondition.Type
jobStage := latestCondition.Stage

0 comments on commit 50b1d45

Please sign in to comment.