Merge pull request #857 from google/i-would-smug-5000-photos

Allow for more than 5000 photos to be uploaded to an album on SmugMug by creating overflow albums

extensions/data-transfer/portability-data-transfer-smugmug/build.gradle

@@ -25,6 +25,9 @@ dependencies {
     compile("com.google.api-client:google-api-client:${googleApiClient}")
 
     compile("org.scribe:scribe:1.3.7")
+
+    testCompile("org.mockito:mockito-core:${mockitoVersion}")
+    testCompile project(":extensions:cloud:portability-cloud-local")
 }
 
 configurePublication(project)

extensions/data-transfer/portability-data-transfer-smugmug/src/main/java/org/datatransferproject/transfer/smugmug/photos/SmugMugInterface.java

@@ -142,12 +142,15 @@ SmugMugImageUploadResponse uploadImage(
     }
 
     // Upload photo
-    return postRequest(
-        "https://upload.smugmug.com/",
-        ImmutableMap.of(), // No content params for photo upload
-        contentBytes,
-        headersMap,
-        new TypeReference<SmugMugImageUploadResponse>() {});
+    SmugMugImageUploadResponse response =
+        postRequest(
+            "https://upload.smugmug.com/",
+            ImmutableMap.of(), // No content params for photo upload
+            contentBytes,
+            headersMap,
+            new TypeReference<SmugMugImageUploadResponse>() {});
+
+    return Preconditions.checkNotNull(response, "Image upload Response is null");
   }
 
   private SmugMugUserResponse getUserInformation() throws IOException {
@@ -181,8 +184,7 @@ private <T> SmugMugResponse<T> makeRequest(
           String.format("Error occurred in request for %s : %s", url, response.getMessage()));
     }
 
-    String result = response.getBody();
-    return mapper.readValue(result, typeReference);
+    return mapper.readValue(response.getBody(), typeReference);
   }
 
   // Makes a post request with the content parameters provided as the body, or the httpcontent as
@@ -240,7 +242,6 @@ private <T> T postRequest(
               "Error occurred in request for %s, code: %s, message: %s",
               fullUrl, response.getCode(), response.getMessage()));
     }
-
     return mapper.readValue(response.getBody(), typeReference);
   }
 

extensions/data-transfer/portability-data-transfer-smugmug/src/main/java/org/datatransferproject/transfer/smugmug/photos/SmugMugPhotoTempData.java

@@ -0,0 +1,89 @@
+package org.datatransferproject.transfer.smugmug.photos;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.MoreObjects;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.Serializable;
+import org.datatransferproject.types.common.models.DataModel;
+
+@JsonTypeName("org.dataportability:SmugMugPhotoTempData")
+public class SmugMugPhotoTempData extends DataModel implements Serializable {
+  private final String albumExportId;
+  private final String albumName;
+  private final String albumDescription;
+  private final String albumUri;
+  private int photoCount;
+  private String overflowAlbumExportId;
+
+  @JsonCreator
+  public SmugMugPhotoTempData(
+      @JsonProperty("albumExportId") String albumExportId,
+      @JsonProperty("albumName") String albumName,
+      @JsonProperty("albumDescription") String albumDescription,
+      @JsonProperty("albumUri") String albumUri) {
+    this(albumExportId, albumName, albumDescription, albumUri, 0, null);
+  }
+
+  @VisibleForTesting
+  @JsonCreator
+  public SmugMugPhotoTempData(
+      @JsonProperty("albumExportId") String albumExportId,
+      @JsonProperty("albumName") String albumName,
+      @JsonProperty("albumDescription") String albumDescription,
+      @JsonProperty("albumUri") String albumUri,
+      int photoCount,
+      String overflowAlbumExportId) {
+    this.albumExportId = albumExportId;
+    this.albumName = albumName;
+    this.albumDescription = albumDescription;
+    this.albumUri = albumUri;
+    this.photoCount = photoCount;
+    this.overflowAlbumExportId = overflowAlbumExportId;
+  }   
+
+  public String getAlbumExportId() {
+    return this.albumExportId;
+  }
+
+  public int incrementPhotoCount() {
+    return this.photoCount++;
+  }
+
+  public int getPhotoCount() {
+    return this.photoCount;
+  }
+
+  public void setOverflowAlbumExportId(String overflowAlbumExportId) {
+    this.overflowAlbumExportId = overflowAlbumExportId;
+  }
+
+  public String getOverflowAlbumExportId() {
+    return this.overflowAlbumExportId;
+  }
+
+  public String getAlbumDescription() {
+    return albumDescription;
+  }
+
+  public String getAlbumName() {
+    return albumName;
+  }
+
+  public String getAlbumUri() {
+    return albumUri;
+  }
+
+  @Override
+  public String toString() {
+    return MoreObjects.toStringHelper(this)
+        .add("albumExportId", albumExportId)
+        .add("photoCount", photoCount)
+        .add("overflowAlbumExportId", overflowAlbumExportId)
+        .add("albumName", albumName)
+        .add("albumDescription", albumDescription)
+        .add("albumUri", albumUri)
+        .toString();
+  }
+}

extensions/data-transfer/portability-data-transfer-smugmug/src/main/java/org/datatransferproject/transfer/smugmug/photos/SmugMugPhotosImporter.java

@@ -16,9 +16,15 @@
 
 package org.datatransferproject.transfer.smugmug.photos;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.UUID;
 import org.datatransferproject.api.launcher.Monitor;
 import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore;
 import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor;
@@ -46,8 +52,7 @@ public class SmugMugPhotosImporter
   private final AppCredentials appCredentials;
   private final ObjectMapper mapper;
   private final Monitor monitor;
-  private final SmugMugTransmogrificationConfig transmogrificationConfig =
-      new SmugMugTransmogrificationConfig();
+  private final SmugMugTransmogrificationConfig transmogrificationConfig;
 
   private SmugMugInterface smugMugInterface;
 
@@ -56,23 +61,27 @@ public SmugMugPhotosImporter(
       AppCredentials appCredentials,
       ObjectMapper mapper,
       Monitor monitor) {
-    this(null, jobStore, appCredentials, mapper, monitor);
+    this(null, new SmugMugTransmogrificationConfig(), jobStore, appCredentials, mapper, monitor);
   }
 
   @VisibleForTesting
   SmugMugPhotosImporter(
       SmugMugInterface smugMugInterface,
+      SmugMugTransmogrificationConfig transmogrificationConfig,
       TemporaryPerJobDataStore jobStore,
       AppCredentials appCredentials,
       ObjectMapper mapper,
       Monitor monitor) {
     this.smugMugInterface = smugMugInterface;
+    this.transmogrificationConfig = transmogrificationConfig;
     this.jobStore = jobStore;
     this.appCredentials = appCredentials;
     this.mapper = mapper;
     this.monitor = monitor;
   }
 
+
+
   @Override
   public ImportResult importItem(
       UUID jobId,
@@ -88,7 +97,9 @@ public ImportResult importItem(
       SmugMugInterface smugMugInterface = getOrCreateSmugMugInterface(authData);
       for (PhotoAlbum album : data.getAlbums()) {
         idempotentExecutor.executeAndSwallowIOExceptions(
-            album.getId(), album.getName(), () -> importSingleAlbum(album, smugMugInterface));
+            album.getId(),
+            album.getName(),
+            () -> importSingleAlbum(jobId, album, smugMugInterface));
       }
       for (PhotoModel photo : data.getPhotos()) {
         idempotentExecutor.executeAndSwallowIOExceptions(
@@ -104,10 +115,14 @@ public ImportResult importItem(
   }
 
   @VisibleForTesting
-  String importSingleAlbum(PhotoAlbum inputAlbum, SmugMugInterface smugMugInterface)
+  String importSingleAlbum(UUID jobId, PhotoAlbum inputAlbum, SmugMugInterface smugMugInterface)
       throws IOException {
-    SmugMugAlbumResponse response = smugMugInterface.createAlbum(inputAlbum.getName());
-    return response.getUri();
+    SmugMugAlbumResponse albumResponse = smugMugInterface.createAlbum(inputAlbum.getName());
+    SmugMugPhotoTempData tempData =
+        new SmugMugPhotoTempData(
+            inputAlbum.getId(), inputAlbum.getName(), inputAlbum.getDescription(), albumResponse.getUri());
+    jobStore.create(jobId, getTempDataId(inputAlbum.getId()), tempData);
+    return albumResponse.getUri();
   }
 
   @VisibleForTesting
@@ -117,20 +132,22 @@ String importSinglePhoto(
       PhotoModel inputPhoto,
       SmugMugInterface smugMugInterface)
       throws Exception {
-    String albumUri = idempotentExecutor.getCachedValue(inputPhoto.getAlbumId());
-    checkState(
-        !Strings.isNullOrEmpty(albumUri),
-        "Cached album URI for %s is null",
-        inputPhoto.getAlbumId());
-
     InputStream inputStream;
     if (inputPhoto.isInTempStore()) {
       inputStream = jobStore.getStream(jobId, inputPhoto.getFetchableUrl()).getStream();
     } else {
       inputStream = smugMugInterface.getImageAsStream(inputPhoto.getFetchableUrl());
     }
+
+    String originalAlbumId = inputPhoto.getAlbumId();
+    SmugMugPhotoTempData albumTempData =
+        getDestinationAlbumTempData(jobId, idempotentExecutor, originalAlbumId, smugMugInterface);
+
     SmugMugImageUploadResponse response =
-        smugMugInterface.uploadImage(inputPhoto, albumUri, inputStream);
+      smugMugInterface.uploadImage(inputPhoto, albumTempData.getAlbumUri(), inputStream);
+    albumTempData.incrementPhotoCount();
+    jobStore.update(jobId, getTempDataId(albumTempData.getAlbumExportId()), albumTempData);  
+
     return response.toString();
   }
 
@@ -141,4 +158,67 @@ private SmugMugInterface getOrCreateSmugMugInterface(TokenSecretAuthData authDat
         ? new SmugMugInterface(appCredentials, authData, mapper)
         : smugMugInterface;
   }
+
+  /**
+   * Get the proper album upload information for the photo. Takes into account size limits of the
+   * albums and completed uploads.
+   */
+  @VisibleForTesting
+  SmugMugPhotoTempData getDestinationAlbumTempData(
+      UUID jobId, IdempotentImportExecutor idempotentExecutor, String baseAlbumId, SmugMugInterface smugMugInterface)
+      throws Exception {
+    SmugMugPhotoTempData baseAlbumTempData =
+        jobStore.findData(jobId, getTempDataId(baseAlbumId), SmugMugPhotoTempData.class);
+    SmugMugPhotoTempData albumTempData = baseAlbumTempData;
+    int depth = 0;
+    while (albumTempData.getPhotoCount() >= transmogrificationConfig.getAlbumMaxSize()) {
+      if (albumTempData.getOverflowAlbumExportId() == null) {
+        PhotoAlbum newAlbum =
+            createOverflowAlbum(
+                baseAlbumTempData.getAlbumExportId(),
+                baseAlbumTempData.getAlbumName(),
+                baseAlbumTempData.getAlbumDescription(),
+                depth + 1);
+        // since the album is full and has no overflow, we need to create a new one
+        String newUri =
+            idempotentExecutor.executeOrThrowException(
+                newAlbum.getId(),
+                newAlbum.getName(),
+                () -> importSingleAlbum(jobId, newAlbum, smugMugInterface));
+        albumTempData.setOverflowAlbumExportId(newAlbum.getId());
+        jobStore.update(jobId, getTempDataId(albumTempData.getAlbumExportId()), albumTempData);
+        albumTempData =
+            jobStore.findData(
+                jobId, getTempDataId(albumTempData.getOverflowAlbumExportId()), SmugMugPhotoTempData.class);
+      } else {
+        albumTempData =
+            jobStore.findData(
+                jobId, getTempDataId(albumTempData.getOverflowAlbumExportId()), SmugMugPhotoTempData.class);
+      }
+      depth += 1;
+    }
+    return albumTempData;
+  }
+
+  private static String getTempDataId(String albumId) {
+    return String.format("smugmug-album-temp-data-%s", albumId);
+  }
+
+  /**
+   * Create an overflow album using the base album's id, name, and description and the overflow album's
+   * opy number.
+   * E.g. if baseAlbum needs a single overflow album, it will be created with 
+   * createOverflowAlbum("baseAlbumId", "baseAlbumName", "baseAlbumDescription", 1) and result in 
+   * an album PhotoAlbum("baseAlbumId-overflow-1", "baseAlbumName (1)", "baseAlbumDescription")
+   */
+  private static PhotoAlbum createOverflowAlbum(
+      String baseAlbumId, String baseAlbumName, String baseAlbumDescription, int copyNumber)
+      throws Exception {
+    checkState(copyNumber > 0, "copyNumber should be > 0");
+    return new PhotoAlbum(
+        String.format("%s-overflow-%d", baseAlbumId, copyNumber),
+        String.format("%s (%d)", baseAlbumName, copyNumber),
+        baseAlbumDescription);
+  }
+
 }

extensions/data-transfer/portability-data-transfer-smugmug/src/main/java/org/datatransferproject/transfer/smugmug/photos/model/SmugMugAlbum.java

@@ -40,6 +40,16 @@ public class SmugMugAlbum {
   @JsonProperty("WebUri")
   private String webUri;
 
+  public SmugMugAlbum(String date, String description, String name, String privacy, String uri, String urlName, String webUri) {
+    this.date = date;
+    this.description = description;
+    this.name = name;
+    this.privacy = privacy;
+    this.uri = uri;
+    this.urlName = urlName;
+    this.webUri = webUri;
+  }
+
   public String getDate() {
     return date;
   }

extensions/data-transfer/portability-data-transfer-smugmug/src/main/java/org/datatransferproject/transfer/smugmug/photos/model/SmugMugAlbumResponse.java

@@ -31,6 +31,16 @@ public class SmugMugAlbumResponse {
   @JsonProperty("Album")
   private SmugMugAlbum album;
 
+  public SmugMugAlbumResponse(String uri, 
+    String locator, 
+    String locatorType, 
+    SmugMugAlbum album){
+    this.uri = uri;
+    this.locator = locator;
+    this.locatorType = locatorType;
+    this.album = album;
+  }
+
   public SmugMugAlbum getAlbum() {
     return album;
   }