Fix: gc failure cause workflow restart not working properly (#5242)

Signed-off-by: Somefive <yd219913@alibaba-inc.com>

Signed-off-by: Somefive <yd219913@alibaba-inc.com>
This commit is contained in:
Somefive
2022-12-30 18:10:38 +08:00
committed by GitHub
parent c85502e54a
commit 42d6791476
3 changed files with 110 additions and 7 deletions
@@ -302,6 +302,11 @@ func (r *Reconciler) gcResourceTrackers(logCtx monitorContext.Context, handler *
}))
defer subCtx.Commit("finish gc resourceTrackers")
statusUpdater := r.patchStatus
if isUpdate {
statusUpdater = r.updateStatus
}
var options []resourcekeeper.GCOption
if !gcOutdated {
options = append(options, resourcekeeper.DisableMarkStageGCOption{}, resourcekeeper.DisableGCComponentRevisionOption{}, resourcekeeper.DisableLegacyGCOption{})
@@ -309,8 +314,10 @@ func (r *Reconciler) gcResourceTrackers(logCtx monitorContext.Context, handler *
finished, waiting, err := handler.resourceKeeper.GarbageCollect(logCtx, options...)
if err != nil {
logCtx.Error(err, "Failed to gc resourcetrackers")
r.Recorder.Event(handler.app, event.Warning(velatypes.ReasonFailedGC, err))
return r.endWithNegativeCondition(logCtx, handler.app, condition.ReconcileError(err), phase)
cond := condition.Deleting()
cond.Message = fmt.Sprintf("error encountered during garbage collection: %s", err.Error())
handler.app.Status.SetConditions(cond)
return r.result(statusUpdater(logCtx, handler.app, phase)).ret()
}
if !finished {
logCtx.Info("GarbageCollecting resourcetrackers unfinished")
@@ -319,13 +326,10 @@ func (r *Reconciler) gcResourceTrackers(logCtx monitorContext.Context, handler *
cond.Message = fmt.Sprintf("Waiting for %s to delete. (At least %d resources are deleting.)", waiting[0].DisplayName(), len(waiting))
}
handler.app.Status.SetConditions(cond)
return r.result(r.patchStatus(logCtx, handler.app, phase)).requeue(baseGCBackoffWaitTime).ret()
return r.result(statusUpdater(logCtx, handler.app, phase)).requeue(baseGCBackoffWaitTime).ret()
}
logCtx.Info("GarbageCollected resourcetrackers")
if isUpdate {
return r.result(r.updateStatus(logCtx, handler.app, phase)).ret()
}
return r.result(r.patchStatus(logCtx, handler.app, phase)).ret()
return r.result(statusUpdater(logCtx, handler.app, phase)).ret()
}
type reconcileResult struct {