Make sure to use the max number of failures
This commit is contained in:
@@ -88,9 +88,11 @@ func (s *scheduler) processInstance(ctx context.Context, si *scheduledInstance)
|
|||||||
log.Printf("Error retrieving instance %s from storage. Got: %v\n", si.instance.Name, err)
|
log.Printf("Error retrieving instance %s from storage. Got: %v\n", si.instance.Name, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
failed := false
|
||||||
for _, task := range s.tasks {
|
for _, task := range s.tasks {
|
||||||
err := task.Run(ctx, si.instance)
|
err := task.Run(ctx, si.instance)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
failed = true
|
||||||
log.Printf("Error running task %s on instance %s. Got: %v\n", task.Name(), si.instance.Name, err)
|
log.Printf("Error running task %s on instance %s. Got: %v\n", task.Name(), si.instance.Name, err)
|
||||||
// Since one task failed, we just assume something might be wrong with the instance, so we don't try to process the rest of the tasks.
|
// Since one task failed, we just assume something might be wrong with the instance, so we don't try to process the rest of the tasks.
|
||||||
si.fails++
|
si.fails++
|
||||||
@@ -98,10 +100,12 @@ func (s *scheduler) processInstance(ctx context.Context, si *scheduledInstance)
|
|||||||
log.Printf("Instance %s has failed to execute tasks too many times. Giving up.\n", si.instance.Name)
|
log.Printf("Instance %s has failed to execute tasks too many times. Giving up.\n", si.instance.Name)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
continue
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
si.fails = 0
|
if !failed {
|
||||||
|
si.fails = 0
|
||||||
|
}
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
log.Printf("Processing tasks for instance %s has been canceled.\n", si.instance.Name)
|
log.Printf("Processing tasks for instance %s has been canceled.\n", si.instance.Name)
|
||||||
return
|
return
|
||||||
|
|||||||
Reference in New Issue
Block a user