Fix push bug

cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandler.java

@@ -303,7 +303,7 @@ public void getAllApps(HttpRequest request, HttpResponder responder,
    * @param pageToken    A token for paginating through results.
    * @param pageSize     The size of each page for pagination.
    * @param sortOrder    The sort order for the metadata.
-   * @param sortOn  The field on which to sort the metadata.
+   * @param sortOn       The field on which to sort the metadata.
    * @param filter       A filter string for filtering the metadata.
    *                     filter query format - "name=<name-filter> AND syncStatus=<SYNCED/UNSYNCED>".
    * @throws Exception If an error occurs during the metadata retrieval process.

cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/SourceControlManagementHttpHandler.java

@@ -41,6 +41,7 @@
 import io.cdap.cdap.proto.id.NamespaceId;
 import io.cdap.cdap.proto.operation.OperationMeta;
 import io.cdap.cdap.proto.operation.OperationRun;
+import io.cdap.cdap.proto.sourcecontrol.Provider;
 import io.cdap.cdap.proto.sourcecontrol.PullMultipleAppsRequest;
 import io.cdap.cdap.proto.sourcecontrol.PushAppRequest;
 import io.cdap.cdap.proto.sourcecontrol.PushMultipleAppsRequest;
@@ -350,6 +351,14 @@ private RepositoryConfigRequest validateAndGetRepoConfig(FullHttpRequest request
       if (repoRequest == null || repoRequest.getConfig() == null) {
         throw new RepositoryConfigValidationException("Repository configuration must be specified.");
       }
+      // Only allow gitlab and bitbucket if feature flag is enabled
+      if (!Feature.SOURCE_CONTROL_MANAGEMENT_GITLAB_BITBUCKET.isEnabled(featureFlagsProvider)) {
+        if (repoRequest.getConfig().getProvider() != Provider.GITHUB) {
+          throw new BadRequestException(
+              String.format("Provider %s is not supported", repoRequest.getConfig().getProvider())
+          );
+        }
+      }
       repoRequest.getConfig().validate();
       return repoRequest;
     } catch (JsonSyntaxException e) {

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SourceControlManagementService.java

@@ -376,7 +376,7 @@ private PullAppResponse<?> pullAndValidateApplication(ApplicationReference appRe
     PullAppResponse<?> pullResponse = sourceControlOperationRunner.pull(
         new PullAppOperationRequest(appRef, repoConfig));
 
-    if (latestMeta != null
+    if (latestMeta != null && latestMeta.getFileHash() != null
         && latestMeta.getFileHash().equals(pullResponse.getApplicationFileHash())) {
       throw new NoChangesToPullException(
           String.format("Pipeline deployment was not successful because there is "

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/PullAppsOperation.java

@@ -100,8 +100,9 @@ public ListenableFuture<Set<OperationResource>> run(LongRunningOperationContext
 
         try {
           ApplicationDetail currentDetail = applicationManager.get(appRef);
-          if (currentDetail.getSourceControlMeta() != null && currentDetail.getSourceControlMeta()
-              .getFileHash().equals(response.getApplicationFileHash())) {
+          if (currentDetail.getSourceControlMeta() != null
+              && response.getApplicationFileHash()
+              .equals(currentDetail.getSourceControlMeta().getFileHash())) {
             LOG.trace("Application {} already have same commit, skipping",
                 response.getApplicationName());
             return;

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/NamespaceSourceControlMetadataStore.java

@@ -259,7 +259,7 @@ private CloseableIterator<StructuredRow> getScanIterator(
       Range range,
       SortOrder sortOrder,
       SortBy sortBy) throws IOException {
-    if (sortBy == SortBy.LAST_SYNCED_DATE) {
+    if (sortBy == SortBy.LAST_SYNCED_AT) {
       return table.scan(range, Integer.MAX_VALUE,
           StoreDefinition.NamespaceSourceControlMetadataStore.LAST_MODIFIED_FIELD, sortOrder);
     }
@@ -275,7 +275,7 @@ private List<Field<?>> getRangeFields(
             request.getNamespace()));
     fields.add(
         Fields.stringField(StoreDefinition.NamespaceSourceControlMetadataStore.TYPE_FIELD, type));
-    if (request.getSortOn() == SortBy.LAST_SYNCED_DATE) {
+    if (request.getSortOn() == SortBy.LAST_SYNCED_AT) {
       SourceControlMeta meta = get(
           new ApplicationReference(request.getNamespace(), request.getScanAfter()));
       fields.add(Fields.longField(

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/RepositorySourceControlMetadataStore.java

@@ -200,7 +200,7 @@ private CloseableIterator<StructuredRow> getScanIterator(
       Range range,
       SortOrder sortOrder,
       SortBy sortBy) throws IOException {
-    if (sortBy == SortBy.LAST_SYNCED_DATE) {
+    if (sortBy == SortBy.LAST_SYNCED_AT) {
       return table.scan(range, Integer.MAX_VALUE,
           StoreDefinition.RepositorySourceControlMetadataStore.LAST_MODIFIED_FIELD, sortOrder);
     }
@@ -245,7 +245,7 @@ private List<Field<?>> getRangeFields(ScanSourceControlMetadataRequest request,
             request.getNamespace()));
     fields.add(
         Fields.stringField(StoreDefinition.RepositorySourceControlMetadataStore.TYPE_FIELD, type));
-    if (request.getSortOn() == SortBy.LAST_SYNCED_DATE) {
+    if (request.getSortOn() == SortBy.LAST_SYNCED_AT) {
       ImmutablePair<Long, Boolean> lastModifiedAndStatusPair = get(
           new ApplicationReference(request.getNamespace(), request.getScanAfter()));
       fields.add(Fields.longField(

cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SourceControlManagementServiceTest.java

@@ -599,7 +599,7 @@ public void testScanRepoMetadata() throws Exception {
     gotRecords.clear();
     request = ScanSourceControlMetadataRequest.builder().setNamespace(Namespace.DEFAULT.getId())
         .setSortOrder(
-            SortOrder.DESC).setSortOn(SortBy.LAST_SYNCED_DATE).build();
+            SortOrder.DESC).setSortOn(SortBy.LAST_SYNCED_AT).build();
     sourceControlService.scanRepoMetadata(request, batchSize, gotRecords::add);
     expectedRecords = insertedRecords.stream()
         .sorted(Comparator.nullsFirst(

cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/NamespaceSourceControlMetadataStoreTest.java

@@ -140,7 +140,7 @@ public void testScanWithDifferentLimit() {
       // when last modified is in desc order
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
-          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedDesc;
       Assert.assertArrayEquals(expectedRecords.toArray(), gotRecords.toArray());
@@ -149,15 +149,15 @@ public void testScanWithDifferentLimit() {
       // when last modified is in asc order
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
-          .setSortOrder(SortOrder.ASC).setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOrder(SortOrder.ASC).setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedAsc;
       Assert.assertArrayEquals(expectedRecords.toArray(), gotRecords.toArray());
 
       // verify limit for testNamespaceRecordsSortedOnNameDesc
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE).setLimit(3)
-          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.PIPELINE_NAME).build();
+          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.NAME).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnNameDesc.stream().limit(3)
           .collect(Collectors.toList());
@@ -210,7 +210,7 @@ public void testScanWithPagination() {
       // verify SYNCED and name filter with pageToken and limit for testNamespaceRecordsSortedOnLastModifiedDesc
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE).setLimit(1)
-          .setScanAfter("dependent100").setSortOrder(SortOrder.DESC).setSortOn(SortBy.PIPELINE_NAME)
+          .setScanAfter("dependent100").setSortOrder(SortOrder.DESC).setSortOn(SortBy.NAME)
           .setFilter(new SourceControlMetadataFilter("t", true)).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedDesc.stream()
@@ -221,7 +221,7 @@ public void testScanWithPagination() {
       // verify UNSYNCED and name filter with pageToken and limit for testNamespaceRecordsSortedOnLastModifiedDesc
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE).setLimit(1)
-          .setScanAfter("dependent100").setSortOn(SortBy.LAST_SYNCED_DATE)
+          .setScanAfter("dependent100").setSortOn(SortBy.LAST_SYNCED_AT)
           .setSortOrder(SortOrder.DESC)
           .setFilter(new SourceControlMetadataFilter("t", false)).build();
       store.scan(request, TYPE, gotRecords::add);
@@ -254,7 +254,7 @@ record -> record.getName().equals("scm-test") || record.getName().equals("zapapp
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
           .setScanAfter("zapapp").setLimit(5).setSortOrder(SortOrder.DESC)
-          .setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedDesc.stream().filter(
           record -> record.getName().equals("newapp") || record.getName().equals("scm-test")
@@ -265,7 +265,7 @@ record -> record.getName().equals("newapp") || record.getName().equals("scm-test
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
           .setScanAfter("test").setLimit(3).setSortOrder(SortOrder.DESC)
-          .setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedDesc.stream().filter(
               record -> record.getName().equals("dependent100") || record.getName()
@@ -276,7 +276,7 @@ record -> record.getName().equals("dependent100") || record.getName()
       // verify page token for testNamespaceRecordsSortedOnNameDesc
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
-          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.PIPELINE_NAME).setScanAfter("test")
+          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.NAME).setScanAfter("test")
           .setLimit(3).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnNameDesc.stream().filter(
@@ -288,7 +288,7 @@ record -> record.getName().equals("scm-test") || record.getName().equals("depend
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
           .setScanAfter("scm-test").setLimit(4).setSortOrder(SortOrder.ASC)
-          .setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedAsc.stream().filter(
               record -> record.getName().equals("zapapp") || record.getName().equals("newapp")
@@ -300,7 +300,7 @@ record -> record.getName().equals("zapapp") || record.getName().equals("newapp")
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
           .setScanAfter("datafusionquickstart").setLimit(2).setSortOrder(SortOrder.ASC)
-          .setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedAsc.stream().filter(
               record -> record.getName().equals("newapp") || record.getName().equals("scm-test"))

cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/RepositorySourceControlMetadataStoreTest.java

@@ -129,7 +129,7 @@ public void testScanWithDifferentLimit() {
       // when last modified is in desc order
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
-          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedDesc;
       Assert.assertArrayEquals(expectedRecords.toArray(), gotRecords.toArray());
@@ -138,15 +138,15 @@ public void testScanWithDifferentLimit() {
       // when last modified is in asc order
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
-          .setSortOrder(SortOrder.ASC).setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOrder(SortOrder.ASC).setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedAsc;
       Assert.assertArrayEquals(expectedRecords.toArray(), gotRecords.toArray());
 
       // verify limit for testNamespaceRecordsSortedOnNameDesc
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE).setLimit(3)
-          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.PIPELINE_NAME).build();
+          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.NAME).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnNameDesc.stream().limit(3)
           .collect(Collectors.toList());
@@ -199,7 +199,7 @@ public void testScanWithPagination() {
       // verify SYNCED and name filter with pageToken and limit for testNamespaceRecordsSortedOnLastModifiedDesc
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE).setLimit(1)
-          .setScanAfter("dependent100").setSortOrder(SortOrder.DESC).setSortOn(SortBy.PIPELINE_NAME)
+          .setScanAfter("dependent100").setSortOrder(SortOrder.DESC).setSortOn(SortBy.NAME)
           .setFilter(new SourceControlMetadataFilter("t", true)).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedDesc.stream()
@@ -210,7 +210,7 @@ public void testScanWithPagination() {
       // verify UNSYNCED and name filter with pageToken and limit for testNamespaceRecordsSortedOnLastModifiedDesc
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE).setLimit(1)
-          .setScanAfter("dependent100").setSortOn(SortBy.LAST_SYNCED_DATE)
+          .setScanAfter("dependent100").setSortOn(SortBy.LAST_SYNCED_AT)
           .setSortOrder(SortOrder.DESC)
           .setFilter(new SourceControlMetadataFilter("t", false)).build();
       store.scan(request, TYPE, gotRecords::add);
@@ -243,7 +243,7 @@ record -> record.getName().equals("scm-test") || record.getName().equals("zapapp
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
           .setScanAfter("zapapp").setLimit(5).setSortOrder(SortOrder.DESC)
-          .setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedDesc.stream().filter(
           record -> record.getName().equals("newapp") || record.getName().equals("scm-test")
@@ -254,7 +254,7 @@ record -> record.getName().equals("newapp") || record.getName().equals("scm-test
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
           .setScanAfter("test").setLimit(3).setSortOrder(SortOrder.DESC)
-          .setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedDesc.stream().filter(
               record -> record.getName().equals("dependent100") || record.getName()
@@ -265,7 +265,7 @@ record -> record.getName().equals("dependent100") || record.getName()
       // verify page token for testNamespaceRecordsSortedOnNameDesc
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
-          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.PIPELINE_NAME).setScanAfter("test")
+          .setSortOrder(SortOrder.DESC).setSortOn(SortBy.NAME).setScanAfter("test")
           .setLimit(3).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnNameDesc.stream().filter(
@@ -277,7 +277,7 @@ record -> record.getName().equals("scm-test") || record.getName().equals("depend
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
           .setScanAfter("scm-test").setLimit(4).setSortOrder(SortOrder.ASC)
-          .setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedAsc.stream().filter(
               record -> record.getName().equals("zapapp") || record.getName().equals("newapp")
@@ -289,7 +289,7 @@ record -> record.getName().equals("zapapp") || record.getName().equals("newapp")
       gotRecords.clear();
       request = ScanSourceControlMetadataRequest.builder().setNamespace(NAMESPACE)
           .setScanAfter("datafusionquickstart").setLimit(2).setSortOrder(SortOrder.ASC)
-          .setSortOn(SortBy.LAST_SYNCED_DATE).build();
+          .setSortOn(SortBy.LAST_SYNCED_AT).build();
       store.scan(request, TYPE, gotRecords::add);
       expectedRecords = testNamespaceRecordsSortedOnLastModifiedAsc.stream().filter(
               record -> record.getName().equals("newapp") || record.getName().equals("scm-test"))

cdap-common/src/main/resources/cdap-default.xml

@@ -6009,7 +6009,7 @@
 
   <property>
     <name>feature.wrangler.kryo.serialization.enabled</name>
-    <value>false</value>
+    <value>true</value>
     <description>
       If true, wrangler will serialize rows using kryo.
     </description>
@@ -6089,6 +6089,13 @@
       pipelines will be enabled.
     </description>
   </property>
+  <property>
+    <name>feature.source.control.management.gitlab.bitbucket.enabled</name>
+    <value>true</value>
+    <description>
+      Enable gitlab and bitbucket integration
+    </description>
+  </property>
   <property>
     <name>source.control.git.command.timeout.seconds</name>
     <value>30</value>

cdap-features/src/main/java/io/cdap/cdap/features/Feature.java

@@ -43,7 +43,8 @@ public enum Feature {
   WRANGLER_EXECUTION_SQL("6.10.0"),
   WRANGLER_SCHEMA_MANAGEMENT("6.10.0"),
   NAMESPACED_SERVICE_ACCOUNTS("6.10.0"),
-  WRANGLER_KRYO_SERIALIZATION("6.10.1");
+  WRANGLER_KRYO_SERIALIZATION("6.10.1"),
+  SOURCE_CONTROL_MANAGEMENT_GITLAB_BITBUCKET("6.10.1");
 
   private final PlatformInfo.Version versionIntroduced;
   private final boolean defaultAfterIntroduction;

cdap-proto/src/main/java/io/cdap/cdap/proto/SourceControlMetadataRecord.java

@@ -16,6 +16,7 @@
 
 package io.cdap.cdap.proto;
 
+import com.google.gson.annotations.SerializedName;
 import java.util.Objects;
 import javax.annotation.Nullable;
 
@@ -32,6 +33,7 @@ public class SourceControlMetadataRecord {
   private final String specificationHash;
   @Nullable
   private final String commitId;
+  @SerializedName("lastSyncedAt")
   @Nullable
   private final Long lastModified;
   private final Boolean isSynced;

cdap-proto/src/main/java/io/cdap/cdap/proto/sourcecontrol/SortBy.java

@@ -20,5 +20,6 @@
  * Sort order options for namespace and repository pipelines.
  */
 public enum SortBy {
-  PIPELINE_NAME, LAST_SYNCED_DATE
+  NAME,
+  LAST_SYNCED_AT
 }

cdap-proto/src/main/java/io/cdap/cdap/proto/sourcecontrol/SourceControlMeta.java

@@ -63,6 +63,7 @@ public SourceControlMeta(String fileHash, @Nullable String commitId,
     this.syncStatus = syncStatus;
   }
 
+  @Nullable
   public String getFileHash() {
     return fileHash;
   }