Clean up PVCs when canceling populator flows

Makefile

@@ -143,6 +143,7 @@ tidy:
 # Run go mod vendor against code
 vendor:
 	go mod vendor
+	./hack/post-vendor.sh
 
 # Generate code
 generate: controller-gen

hack/post-vendor.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+rm vendor/github.com/nxadm/tail/Dockerfile
+rm vendor/github.com/pelletier/go-toml/v2/Dockerfile

pkg/controller/plan/kubevirt.go

@@ -931,6 +931,54 @@ func (r *KubeVirt) SetPopulatorPodOwnership(vm *plan.VMStatus) (err error) {
 	return
 }
 
+// Deletes PVCs that were populated using a volume populator, including prime PVCs
+func (r *KubeVirt) DeletePopulatedPVCs(vm *plan.VMStatus) error {
+	pvcs, err := r.getPVCs(vm.Ref)
+	if err != nil {
+		return err
+	}
+	for _, pvc := range pvcs {
+		if err = r.deleteCorrespondingPrimePVC(&pvc, vm); err != nil {
+			return err
+		}
+		if err = r.deletePopulatedPVC(&pvc, vm); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *KubeVirt) deleteCorrespondingPrimePVC(pvc *core.PersistentVolumeClaim, vm *plan.VMStatus) error {
+	primePVC := core.PersistentVolumeClaim{}
+	err := r.Destination.Client.Get(context.TODO(), client.ObjectKey{Namespace: r.Plan.Spec.TargetNamespace, Name: fmt.Sprintf("prime-%s", string(pvc.UID))}, &primePVC)
+	switch {
+	case err != nil && !k8serr.IsNotFound(err):
+		return err
+	case err == nil:
+		err = r.DeleteObject(&primePVC, vm, "Deleted prime PVC.", "pvc")
+		if err != nil && !k8serr.IsNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *KubeVirt) deletePopulatedPVC(pvc *core.PersistentVolumeClaim, vm *plan.VMStatus) error {
+	err := r.DeleteObject(pvc, vm, "Deleted PVC.", "pvc")
+	switch {
+	case err != nil && !k8serr.IsNotFound(err):
+		return err
+	case err == nil:
+		pvcCopy := pvc.DeepCopy()
+		pvc.Finalizers = nil
+		patch := client.MergeFrom(pvcCopy)
+		if err = r.Destination.Client.Patch(context.TODO(), pvc, patch); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Delete any populator pods that belong to a VM's migration.
 func (r *KubeVirt) DeletePopulatorPods(vm *plan.VMStatus) (err error) {
 	list, err := r.getPopulatorPods()

pkg/controller/plan/migration.go

@@ -331,28 +331,29 @@ func (r *Migration) begin() (err error) {
 // Archive the plan.
 // Best effort to remove any retained migration resources.
 func (r *Migration) Archive() {
-	err := r.init()
-	if err != nil {
+	if err := r.init(); err != nil {
 		r.Log.Error(err, "Archive initialization failed.")
 		return
 	}
 
-	if r.Plan.Provider.Source.Type() == v1beta1.Ova {
-		err = r.deletePvcPvForOva()
-		if err != nil {
+	switch r.Plan.Provider.Source.Type() {
+	case v1beta1.Ova:
+		if err := r.deletePvcPvForOva(); err != nil {
 			r.Log.Error(err, "Failed to clean up the PVC and PV for the OVA plan")
-			return
 		}
 	}
 
 	for _, vm := range r.Plan.Status.Migration.VMs {
-		err = r.cleanup(vm)
-		if err != nil {
-			r.Log.Error(err,
-				"Couldn't clean up VM while archiving plan.",
-				"vm",
-				vm.String())
+		dontFailOnError := func(err error) bool {
+			if err != nil {
+				r.Log.Error(liberr.Wrap(err),
+					"Couldn't clean up VM while archiving plan.",
+					"vm",
+					vm.String())
+			}
+			return false
 		}
+		_ = r.cleanup(vm, dontFailOnError)
 	}
 	return
 }
@@ -384,95 +385,103 @@ func (r *Migration) SetPopulatorDataSourceLabels() {
 
 // Cancel the migration.
 // Delete resources associated with VMs that have been marked canceled.
-func (r *Migration) Cancel() (err error) {
-	err = r.init()
-	if err != nil {
-		err = liberr.Wrap(err)
-		return
+func (r *Migration) Cancel() error {
+	if err := r.init(); err != nil {
+		return liberr.Wrap(err)
 	}
 
 	for _, vm := range r.Plan.Status.Migration.VMs {
 		if vm.HasCondition(Canceled) {
-			err = r.cleanup(vm)
-			if err != nil {
-				r.Log.Error(err,
-					"Couldn't clean up after canceled VM migration.",
-					"vm",
-					vm.String())
-				err = nil
+			dontFailOnError := func(err error) bool {
+				if err != nil {
+					r.Log.Error(liberr.Wrap(err),
+						"Couldn't clean up after canceled VM migration.",
+						"vm",
+						vm.String())
+				}
+				return false
 			}
+			_ = r.cleanup(vm, dontFailOnError)
 			if vm.RestorePowerState == On {
-				err = r.provider.PowerOn(vm.Ref)
-				if err != nil {
+				if err := r.provider.PowerOn(vm.Ref); err != nil {
 					r.Log.Error(err,
 						"Couldn't restore the power state of the source VM.",
 						"vm",
 						vm.String())
-					err = nil
 				}
 			}
 			vm.MarkCompleted()
-			for _, step := range vm.Pipeline {
-				if step.MarkedStarted() {
-					step.MarkCompleted()
-				}
-			}
+			markStartedStepsCompleted(vm)
 		}
 	}
 
+	return nil
+}
+
+func markStartedStepsCompleted(vm *plan.VMStatus) {
+	for _, step := range vm.Pipeline {
+		if step.MarkedStarted() {
+			step.MarkCompleted()
+		}
+	}
+}
+
+func (r *Migration) deletePopulatorPVCs(vm *plan.VMStatus) (err error) {
+	if r.builder.SupportsVolumePopulators() {
+		err = r.kubevirt.DeletePopulatedPVCs(vm)
+	}
 	return
 }
 
 // Delete left over migration resources associated with a VM.
-func (r *Migration) cleanup(vm *plan.VMStatus) (err error) {
+func (r *Migration) cleanup(vm *plan.VMStatus, failOnErr func(error) bool) error {
 	if !vm.HasCondition(Succeeded) {
-		err = r.kubevirt.DeleteVM(vm)
-		if err != nil {
-			return
+		if err := r.kubevirt.DeleteVM(vm); failOnErr(err) {
+			return err
+		}
+		if err := r.deletePopulatorPVCs(vm); failOnErr(err) {
+			return err
 		}
 	}
-	err = r.deleteImporterPods(vm)
-	if err != nil {
-		return
+	if err := r.deleteImporterPods(vm); failOnErr(err) {
+		return err
 	}
-	err = r.kubevirt.DeletePVCConsumerPod(vm)
-	if err != nil {
-		return
+	if err := r.kubevirt.DeletePVCConsumerPod(vm); failOnErr(err) {
+		return err
 	}
-	err = r.kubevirt.DeleteGuestConversionPod(vm)
-	if err != nil {
-		return
+	if err := r.kubevirt.DeleteGuestConversionPod(vm); failOnErr(err) {
+		return err
 	}
-	err = r.kubevirt.DeleteSecret(vm)
-	if err != nil {
-		return
+	if err := r.kubevirt.DeleteSecret(vm); failOnErr(err) {
+		return err
 	}
-	err = r.kubevirt.DeleteConfigMap(vm)
-	if err != nil {
-		return
+	if err := r.kubevirt.DeleteConfigMap(vm); failOnErr(err) {
+		return err
 	}
-	err = r.kubevirt.DeleteHookJobs(vm)
-	if err != nil {
-		return
+	if err := r.kubevirt.DeleteHookJobs(vm); failOnErr(err) {
+		return err
 	}
-	err = r.destinationClient.DeletePopulatorDataSource(vm)
-	if err != nil {
-		return
+	if err := r.destinationClient.DeletePopulatorDataSource(vm); failOnErr(err) {
+		return err
 	}
-	err = r.kubevirt.DeletePopulatorPods(vm)
-	if err != nil {
+	if err := r.kubevirt.DeletePopulatorPods(vm); failOnErr(err) {
+		return err
+	}
+	r.removeWarmSnapshots(vm)
+
+	return nil
+}
+
+func (r *Migration) removeWarmSnapshots(vm *plan.VMStatus) {
+	if vm.Warm == nil {
 		return
 	}
-	if vm.Warm != nil {
-		if errLocal := r.provider.RemoveSnapshots(vm.Ref, vm.Warm.Precopies); errLocal != nil {
-			r.Log.Error(
-				errLocal,
-				"Failed to clean up warm migration snapshots.",
-				"vm", vm)
-		}
+	if err := r.provider.RemoveSnapshots(vm.Ref, vm.Warm.Precopies); err != nil {
+		r.Log.Error(
+			err,
+			"Failed to clean up warm migration snapshots.",
+			"vm", vm)
 	}
-
-	return
 }
 
 func (r *Migration) deleteImporterPods(vm *plan.VMStatus) (err error) {
@@ -653,7 +662,7 @@ func (r *Migration) execute(vm *plan.VMStatus) (err error) {
 		vm.MarkStarted()
 		step.MarkStarted()
 		step.Phase = Running
-		err = r.cleanup(vm)
+		err = r.cleanup(vm, func(err error) bool { return err != nil })
 		if err != nil {
 			step.AddError(err.Error())
 			err = nil