[Cherry-Pick for v1.9.0] (#180)

* fix(node-io-stress): Add fix for node-io-stress (#177) Signed-off-by: Udit Gaurav <udit.gaurav@mayadata.io> * fix(abort): fixing the result updation for abort (#178) Signed-off-by: shubhamchaudhary <shubham.chaudhary@mayadata.io> * refactor(experiment): covered uncaught errors (#179) Signed-off-by: shubhamchaudhary <shubham.chaudhary@mayadata.io> Co-authored-by: UDIT GAURAV <35391335+uditgaurav@users.noreply.github.com>
litmuschaos · Oct 15, 2020 · 6128618 · 6128618
1 parent 807d033
commit 6128618
Show file tree

Hide file tree

Showing 18 changed files with 65 additions and 34 deletions.
diff --git a/chaoslib/litmus/node-io-stress/lib/node-io-stress.go b/chaoslib/litmus/node-io-stress/lib/node-io-stress.go
@@ -124,7 +124,7 @@ func CreateHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
 					Image:           experimentsDetails.LIBImage,
 					ImagePullPolicy: apiv1.PullAlways,
 					Command: []string{
-						"/stress-ng",
+						"stress-ng",
 					},
 					Args: GetContainerArguments(experimentsDetails),
 				},

diff --git a/experiments/generic/container-kill/experiment/container-kill.go b/experiments/generic/container-kill/experiment/container-kill.go
@@ -35,8 +35,10 @@ func ContainerKill(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/disk-fill/experiment/disk-fill.go b/experiments/generic/disk-fill/experiment/disk-fill.go
@@ -33,8 +33,10 @@ func DiskFill(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/kubelet-service-kill/experiment/kubelet-service-kill.go b/experiments/generic/kubelet-service-kill/experiment/kubelet-service-kill.go
@@ -33,8 +33,10 @@ func KubeletServiceKill(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/node-cpu-hog/experiment/node-cpu-hog.go b/experiments/generic/node-cpu-hog/experiment/node-cpu-hog.go
@@ -35,8 +35,10 @@ func NodeCPUHog(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/node-drain/experiment/node-drain.go b/experiments/generic/node-drain/experiment/node-drain.go
@@ -33,8 +33,10 @@ func NodeDrain(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/node-io-stress/experiment/node-io-stress.go b/experiments/generic/node-io-stress/experiment/node-io-stress.go
@@ -35,8 +35,10 @@ func NodeIOStress(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/node-memory-hog/experiment/node-memory-hog.go b/experiments/generic/node-memory-hog/experiment/node-memory-hog.go
@@ -35,8 +35,10 @@ func NodeMemoryHog(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/node-taint/experiment/node-taint.go b/experiments/generic/node-taint/experiment/node-taint.go
@@ -33,8 +33,10 @@ func NodeTaint(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/pod-cpu-hog/experiment/pod-cpu-hog.go b/experiments/generic/pod-cpu-hog/experiment/pod-cpu-hog.go
@@ -36,8 +36,10 @@ func PodCPUHog(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/pod-delete/experiment/pod-delete.go b/experiments/generic/pod-delete/experiment/pod-delete.go
@@ -35,10 +35,9 @@ func PodDelete(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
 	if err := probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
 		log.Fatalf("Unable to initialise probes details from chaosengine, err: %v", err)
-
 	}
 
 	//Updating the chaos result in the beginning of experiment

diff --git a/experiments/generic/pod-io-stress/experiment/pod-io-stress.go b/experiments/generic/pod-io-stress/experiment/pod-io-stress.go
@@ -35,8 +35,10 @@ func PodIOStress(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/pod-memory-hog/experiment/pod-memory-hog.go b/experiments/generic/pod-memory-hog/experiment/pod-memory-hog.go
@@ -36,8 +36,10 @@ func PodMemoryHog(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/pod-network-corruption/experiment/pod-network-corruption.go b/experiments/generic/pod-network-corruption/experiment/pod-network-corruption.go
@@ -36,8 +36,10 @@ func PodNetworkCorruption(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/pod-network-duplication/experiment/pod-network-duplication.go b/experiments/generic/pod-network-duplication/experiment/pod-network-duplication.go
@@ -36,8 +36,10 @@ func PodNetworkDuplication(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/pod-network-latency/experiment/pod-network-latency.go b/experiments/generic/pod-network-latency/experiment/pod-network-latency.go
@@ -36,8 +36,10 @@ func PodNetworkLatency(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginning of experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/experiments/generic/pod-network-loss/experiment/pod-network-loss.go b/experiments/generic/pod-network-loss/experiment/pod-network-loss.go
@@ -36,8 +36,10 @@ func PodNetworkLoss(clients clients.ClientSets) {
 	// Intialise Chaos Result Parameters
 	types.SetResultAttributes(&resultDetails, chaosDetails)
 
-	// Intialise the probe details
-	probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails)
+	// Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet
+	if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil {
+		log.Fatalf("Unable to initialize the probes, err: %v", err)
+	}
 
 	//Updating the chaos result in the beginningo f experiment
 	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)

diff --git a/pkg/result/chaosresult.go b/pkg/result/chaosresult.go
@@ -39,7 +39,9 @@ func ChaosResult(chaosDetails *types.ChaosDetails, clients clients.ClientSets, r
 		return err
 	}
 
-	if chaosDetails.EngineName != "" {
+	// as the chaos pod won't be available for stopped phase
+	// skipping the derivation of labels from chaos pod, if phase is stopped
+	if chaosDetails.EngineName != "" && resultDetails.Phase != "Stopped" {
 		// Getting chaos pod label and passing it in chaos result
 		chaosPod, err := clients.KubeClient.CoreV1().Pods(chaosDetails.ChaosNamespace).Get(chaosDetails.ChaosPodName, metav1.GetOptions{})
 		if err != nil {