Merge remote-tracking branch 'origin/main' into release/4.0

oracle · Apr 6, 2023 · 4df9615 · 4df9615
2 parents bed40dc + cf9bb16
commit 4df9615
Show file tree

Hide file tree

Showing 6 changed files with 126 additions and 17 deletions.
diff --git a/...ests/src/test/java/oracle/weblogic/kubernetes/ItHorizontalPodAutoscalerCustomMetrics.java b/...ests/src/test/java/oracle/weblogic/kubernetes/ItHorizontalPodAutoscalerCustomMetrics.java
@@ -8,14 +8,17 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
+import java.time.OffsetDateTime;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import io.kubernetes.client.custom.Quantity;
 import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.models.CoreV1Event;
 import io.kubernetes.client.openapi.models.V1LocalObjectReference;
 import io.kubernetes.client.openapi.models.V1ResourceRequirements;
+import io.kubernetes.client.util.Yaml;
 import oracle.weblogic.domain.ClusterResource;
 import oracle.weblogic.domain.DomainResource;
 import oracle.weblogic.domain.ServerPod;
@@ -56,11 +59,13 @@
 import static oracle.weblogic.kubernetes.actions.TestActions.deletePersistentVolumeClaim;
 import static oracle.weblogic.kubernetes.actions.TestActions.deletePod;
 import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
+import static oracle.weblogic.kubernetes.actions.TestActions.now;
 import static oracle.weblogic.kubernetes.actions.TestActions.uninstallNginx;
 import static oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes.deleteNamespace;
 import static oracle.weblogic.kubernetes.utils.ClusterUtils.createClusterAndVerify;
 import static oracle.weblogic.kubernetes.utils.ClusterUtils.createClusterResource;
 import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResource;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkClusterReplicaCountMatches;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withLongRetryPolicy;
@@ -69,6 +74,7 @@
 import static oracle.weblogic.kubernetes.utils.FileUtils.replaceStringInFile;
 import static oracle.weblogic.kubernetes.utils.ImageUtils.createBaseRepoSecret;
 import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret;
+import static oracle.weblogic.kubernetes.utils.K8sEvents.getEvents;
 import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.createIngressForDomainAndVerify;
 import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyNginx;
 import static oracle.weblogic.kubernetes.utils.MonitoringUtils.cleanupPromGrafanaClusterRoles;
@@ -80,11 +86,12 @@
 import static oracle.weblogic.kubernetes.utils.MonitoringUtils.installMonitoringExporter;
 import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator;
 import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPvAndPvc;
-import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodDeleted;
+import static oracle.weblogic.kubernetes.utils.PodUtils.isPodDeleted;
 import static oracle.weblogic.kubernetes.utils.SecretUtils.createSecretWithUsernamePassword;
 import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -296,6 +303,7 @@ void testHPAWithCustomMetrics() {
     //reboot server1 and server2 to kill open sessions
     assertDoesNotThrow(() -> deletePod(managedServerPrefix + 1, domainNamespace));
     assertDoesNotThrow(() -> deletePod(managedServerPrefix + 2, domainNamespace));
+    OffsetDateTime timestamp = now();
     // wait until reboot
     for (int i = 1; i < 3; i++) {
       checkPodReadyAndServiceExists(managedServerPrefix + i, domainUid, domainNamespace);
@@ -312,21 +320,42 @@ void testHPAWithCustomMetrics() {
         logger,
         "Checking if replica switched to 2");
 
-    try {
-      checkPodDeleted(managedServerPrefix + 3, domainUid, domainNamespace);
-    } catch (Exception ex) {
+    if (!isPodDeleted(managedServerPrefix + 3, domainUid, domainNamespace)) {
+
       //check if different server was scaled down
-      try {
-        if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 1)) {
+      assertDoesNotThrow(() -> {
+        logger.info("Checking if HPA scaled down managed server 1 or managed server 2");
+        String command = KUBERNETES_CLI + " get pods -n"  + domainNamespace;
+
+        logger.info("Executing command " + command);
+        ExecResult result = ExecCommand.exec(command);
+        logger.info(" Result output: " + result.stdout());
+        command = KUBERNETES_CLI + " describe pod " + managedServerPrefix + 3 + " -n"  + domainNamespace;
+
+        logger.info("Executing command " + command);
+        result = ExecCommand.exec(command);
+        logger.info(" Result output: " + result.stdout());
+        List<CoreV1Event> events = getEvents(domainNamespace,timestamp);
+        for (CoreV1Event event : events) {
+          logger.info("Generated events after HPA scaling " + Yaml.dump(event));
+        }
+        int numberOfManagedSvs = 3;
+        if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 1)
+            || Kubernetes.isPodTerminating(domainNamespace, domainUid, managedServerPrefix + 1)) {
           logger.info("HPA scaled down managed server 1");
-        } else if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 2)) {
+          --numberOfManagedSvs;
+        } else if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 2)
+            || Kubernetes.isPodTerminating(domainNamespace, domainUid, managedServerPrefix + 2)) {
           logger.info("HPA scaled down managed server 2");
-        } else {
-          checkPodDeleted(managedServerPrefix + 3, domainUid, domainNamespace);
+          --numberOfManagedSvs;
+        } else if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 3)
+            || Kubernetes.isPodTerminating(domainNamespace, domainUid, managedServerPrefix + 2)) {
+          logger.info("HPA scaled down managed server 3");
+          --numberOfManagedSvs;
         }
-      } catch (Exception ex1) {
-        throw ex;
-      }
+        assertTrue(checkClusterReplicaCountMatches(clusterResName, domainNamespace, 2));
+        assertEquals(2, numberOfManagedSvs);
+      });
     }
   }
 

diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java
@@ -1,4 +1,4 @@
-// Copyright (c) 2020, 2022, Oracle and/or its affiliates.
+// Copyright (c) 2020, 2023, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.weblogic.kubernetes.utils;
@@ -189,6 +189,27 @@ public static void testUntil(ConditionFactory conditionFactory, Callable<Boolean
     }
   }
 
+  /**
+   * Test assertion over time until it passes or the timeout expires.
+   * @param conditionFactory Configuration for Awaitility condition factory
+   * @param conditionEvaluator Condition evaluator
+   * @param logger Logger
+   * @param msg Message for logging
+   * @param params Parameter to message for logging
+   * @return false if timeout, true for success
+   */
+  public static boolean testUntilNoException(ConditionFactory conditionFactory, Callable<Boolean> conditionEvaluator,
+                               LoggingFacade logger, String msg, Object... params) {
+    try {
+      conditionFactory
+          .conditionEvaluationListener(createConditionEvaluationListener(logger, msg, params))
+          .until(conditionEvaluator);
+      return true;
+    } catch (ConditionTimeoutException timeout) {
+      return false;
+    }
+  }
+
   private static <T> ConditionEvaluationListener<T> createConditionEvaluationListener(
       LoggingFacade logger, String msg, Object... params) {
     return new ConditionEvaluationListener<T>() {

diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java
@@ -509,6 +509,30 @@ public static List<CoreV1Event> getEvents(String domainNamespace,
     return events;
   }
 
+  /**
+   * Get matching event objects after specific timestamp.
+   * @param namespace namespace in which the event is logged
+   * @param timestamp the timestamp after which to see events
+   * @return CoreV1Event matching event object
+   */
+  public static List<CoreV1Event> getEvents(String namespace,
+                                            OffsetDateTime timestamp) {
+
+    List<CoreV1Event> events = new ArrayList<>();
+
+    try {
+      List<CoreV1Event> allEvents = Kubernetes.listNamespacedEvents(namespace);
+      for (CoreV1Event event : allEvents) {
+        if ((isEqualOrAfter(timestamp, event))) {
+          events.add(event);
+        }
+      }
+    } catch (ApiException ex) {
+      Logger.getLogger(K8sEvents.class.getName()).log(Level.SEVERE, null, ex);
+    }
+    return events;
+  }
+
   private static boolean isEqualOrAfter(OffsetDateTime timestamp, CoreV1Event event) {
     return event.getLastTimestamp().isEqual(timestamp)
             || event.getLastTimestamp().isAfter(timestamp);

diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/PodUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/PodUtils.java
@@ -1,4 +1,4 @@
-// Copyright (c) 2021, 2022, Oracle and/or its affiliates.
+// Copyright (c) 2021, 2023, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.weblogic.kubernetes.utils;
@@ -42,6 +42,7 @@
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.podInitialized;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.podReady;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntilNoException;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withLongRetryPolicy;
 import static oracle.weblogic.kubernetes.utils.JobUtils.getIntrospectJobName;
 import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
@@ -343,6 +344,25 @@ public static void checkPodDeleted(String podName, String domainUid, String domN
         domNamespace);
   }
 
+  /**
+   * Check if the pods are deleted.
+   * @param podName pod name
+   * @param domainUid unique id of the domain
+   * @param domNamespace namespace where domain exists
+   * @return true if pod is deleted
+   */
+  public static boolean isPodDeleted(String podName, String domainUid, String domNamespace) {
+    final LoggingFacade logger = getLogger();
+    return testUntilNoException(withLongRetryPolicy,
+        assertDoesNotThrow(() -> podDoesNotExist(podName, domainUid, domNamespace),
+            String.format("podDoesNotExist failed with ApiException for %s in namespace in %s",
+                podName, domNamespace)),
+        logger,
+        "pod {0} to be deleted in namespace {1}",
+        podName,
+        domNamespace);
+  }
+
   public static String getExternalServicePodName(String adminServerPodName) {
     return getExternalServicePodName(adminServerPodName, TestConstants.DEFAULT_EXTERNAL_SERVICE_NAME_SUFFIX);
   }

diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java
@@ -498,7 +498,6 @@ private Step deleteIntrospectorJob() {
     }
 
     class DeleteDomainIntrospectorJobStep extends Step {
-
       @Override
       public NextAction apply(Packet packet) {
         logJobDeleted(getDomainUid(), getNamespace(), getJobName(), packet);
@@ -885,14 +884,18 @@ static void logJobDeleted(String domainUid, String namespace, String jobName, Pa
 
     packet.remove(ProcessingConstants.INTROSPECTOR_JOB_FAILURE_LOGGED);
     if (domainIntrospectorJob != null
-            && !JobWatcher.isComplete(domainIntrospectorJob)) {
+        && hasStatusAndCondition(domainIntrospectorJob) && !JobWatcher.isComplete(domainIntrospectorJob)) {
       logIntrospectorFailure(packet, domainIntrospectorJob);
     }
     packet.remove(ProcessingConstants.JOB_POD_NAME);
 
     LOGGER.fine(getJobDeletedMessageKey(), domainUid, namespace, jobName);
   }
 
+  private static boolean hasStatusAndCondition(V1Job job) {
+    return job.getStatus() != null && job.getStatus().getConditions() != null;
+  }
+
   static String getJobDeletedMessageKey() {
     return MessageKeys.JOB_DELETED;
   }

diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java
@@ -1317,13 +1317,25 @@ void whenIntrospectorJobNotNeeded_validateDomainAgainstPreviousTopology() throws
   }
 
   @Test
-  void whenJobLogContainsSevereError_logJobInfosOnDelete() {
+  void whenJobStatusContainsNoConditions_dontLogJobFailedAndInfosOnDelete() {
     testSupport.defineResources(createIntrospectorJob());
     IntrospectionTestUtils.defineIntrospectionPodLog(testSupport, SEVERE_MESSAGE);
     testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName()));
 
     testSupport.runSteps(JobHelper.deleteDomainIntrospectorJobStep(null));
 
+    assertThat(logRecords, not(containsInfo(getJobFailedMessageKey())));
+    assertThat(logRecords, not(containsFine(getJobFailedDetailMessageKey())));
+    assertThat(logRecords, containsFine(getJobDeletedMessageKey()));
+  }
+
+  @Test
+  void whenJobStatusHasFailedCondition_logJobInfosOnDelete() {
+    testSupport.defineResources(asFailedJob(createIntrospectorJob()));
+    testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName()));
+
+    testSupport.runSteps(JobHelper.deleteDomainIntrospectorJobStep(null));
+
     assertThat(logRecords, containsInfo(getJobFailedMessageKey()));
     assertThat(logRecords, containsFine(getJobFailedDetailMessageKey()));
     assertThat(logRecords, containsFine(getJobDeletedMessageKey()));