Skip to content

Commit

Permalink
Merge pull request #793 from wmorland/use-google-photoslibary
Browse files Browse the repository at this point in the history 
Use google photoslibary for importing video (+ upgrade Mockito)
  • Loading branch information
@wmorland
wmorland authored Dec 10, 2019
2 parents 410a6c6 + d7c8cd6 commit 4e70deb
Show file tree
Hide file tree
Showing 27 changed files with 213 additions and 166 deletions.
2 changes: 1 addition & 1 deletion extensions/cloud/portability-cloud-google/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ dependencies {
testCompile("com.google.truth:truth:${truthVersion}")
testCompile("junit:junit:${junitVersion}")
testCompile("org.junit.jupiter:junit-jupiter-api:${junitJupiterVersion}")
testCompile("org.mockito:mockito-all:${mockitoVersion}")
testCompile("org.mockito:mockito-core:${mockitoVersion}")

}

Expand Down
2 changes: 1 addition & 1 deletion ...loud-google/src/test/java/org/datatransferproject/cloud/google/GoogleCloudModuleTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.datatransferproject.api.launcher.Constants.Environment;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.junit.MockitoJUnitRunner;

@RunWith(MockitoJUnitRunner.class)
public class GoogleCloudModuleTest {
Expand Down
2 changes: 1 addition & 1 deletion ...y-cloud-google/src/test/java/org/datatransferproject/cloud/google/GoogleJobStoreTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.junit.MockitoJUnitRunner;

@RunWith(MockitoJUnitRunner.class)
public class GoogleJobStoreTest {
Expand Down
2 changes: 1 addition & 1 deletion ...fig-yaml/src/test/java/org/datatransferproject/config/yaml/YamlSettingsExtensionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.datatransferproject.config.ConfigUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.junit.MockitoJUnitRunner;

@RunWith(MockitoJUnitRunner.class)
public class YamlSettingsExtensionTest {
Expand Down
2 changes: 1 addition & 1 deletion ...est/java/org/datatransferproject/datatransfer/flickr/photos/FlickrPhotosExporterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@

import static com.google.common.truth.Truth.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Matchers.*;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

Expand Down
4 changes: 2 additions & 2 deletions ...est/java/org/datatransferproject/datatransfer/flickr/photos/FlickrPhotosImporterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@
import java.util.UUID;

import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
Expand Down
3 changes: 2 additions & 1 deletion extensions/data-transfer/portability-data-transfer-google/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ dependencies {
compile("com.google.apis:google-api-services-tasks:${googleTasksVersion}")
compile("com.googlecode.ez-vcard:ez-vcard:${ezVcardVersion}")
compile("com.google.apis:google-api-services-plus:${googlePlusVersion}")
compile("com.google.photos.library:google-photos-library-client:1.4.0")

// Needed for OpenSocial ActivityStreams, which depends on these specific version
// so not generifying them.
Expand All @@ -47,4 +48,4 @@ dependencies {
testCompile project(':extensions:cloud:portability-cloud-local')
}

configurePublication(project)
configurePublication(project)
27 changes: 13 additions & 14 deletions ...le/src/main/java/org/datatransferproject/datatransfer/google/GoogleTransferExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import org.datatransferproject.api.launcher.ExtensionContext;
import org.datatransferproject.api.launcher.Monitor;
import org.datatransferproject.datatransfer.google.calendar.GoogleCalendarExporter;
Expand All @@ -30,8 +31,6 @@
import org.datatransferproject.spi.transfer.provider.Importer;
import org.datatransferproject.types.transfer.auth.AppCredentials;

import java.io.IOException;

/*
* GoogleTransferExtension allows for importers and exporters of data types
* to be retrieved.
Expand All @@ -40,7 +39,8 @@ public class GoogleTransferExtension implements TransferExtension {
public static final String SERVICE_ID = "google";
// TODO: centralized place, or enum type for these
private static final ImmutableList<String> SUPPORTED_SERVICES =
ImmutableList.of("BLOBS", "CALENDAR", "CONTACTS", "MAIL", "PHOTOS", "SOCIAL-POSTS", "TASKS", "VIDEOS");
ImmutableList.of(
"BLOBS", "CALENDAR", "CONTACTS", "MAIL", "PHOTOS", "SOCIAL-POSTS", "TASKS", "VIDEOS");
private ImmutableMap<String, Importer> importerMap;
private ImmutableMap<String, Exporter> exporterMap;
private boolean initialized = false;
Expand Down Expand Up @@ -78,23 +78,22 @@ public void initialize(ExtensionContext context) {
AppCredentials appCredentials;
try {
appCredentials =
context
.getService(AppCredentialStore.class)
.getAppCredentials("GOOGLE_KEY", "GOOGLE_SECRET");
context
.getService(AppCredentialStore.class)
.getAppCredentials("GOOGLE_KEY", "GOOGLE_SECRET");
} catch (IOException e) {
Monitor monitor = context.getMonitor();
monitor.info(
() ->
"Unable to retrieve Google AppCredentials. Did you set GOOGLE_KEY and GOOGLE_SECRET?");
() ->
"Unable to retrieve Google AppCredentials. Did you set GOOGLE_KEY and GOOGLE_SECRET?");
return;
}

Monitor monitor = context.getMonitor();

// Create the GoogleCredentialFactory with the given {@link AppCredentials}.
GoogleCredentialFactory credentialFactory =
new GoogleCredentialFactory(httpTransport, jsonFactory, appCredentials, monitor);

new GoogleCredentialFactory(httpTransport, jsonFactory, appCredentials, monitor);

ImmutableMap.Builder<String, Importer> importerBuilder = ImmutableMap.builder();
importerBuilder.put("BLOBS", new DriveImporter(credentialFactory, jobStore, monitor));
Expand All @@ -103,8 +102,8 @@ public void initialize(ExtensionContext context) {
importerBuilder.put("MAIL", new GoogleMailImporter(credentialFactory, monitor));
importerBuilder.put("TASKS", new GoogleTasksImporter(credentialFactory));
importerBuilder.put(
"PHOTOS", new GooglePhotosImporter(credentialFactory, jobStore, jsonFactory, monitor));
importerBuilder.put("VIDEOS", new GoogleVideosImporter(credentialFactory, jsonFactory, monitor));
"PHOTOS", new GooglePhotosImporter(credentialFactory, jobStore, jsonFactory, monitor));
importerBuilder.put("VIDEOS", new GoogleVideosImporter(appCredentials, jobStore, monitor));
importerMap = importerBuilder.build();

ImmutableMap.Builder<String, Exporter> exporterBuilder = ImmutableMap.builder();
Expand All @@ -115,8 +114,8 @@ public void initialize(ExtensionContext context) {
exporterBuilder.put("SOCIAL-POSTS", new GooglePlusExporter(credentialFactory));
exporterBuilder.put("TASKS", new GoogleTasksExporter(credentialFactory, monitor));
exporterBuilder.put(
"PHOTOS", new GooglePhotosExporter(credentialFactory, jobStore, jsonFactory, monitor));
exporterBuilder.put("VIDEOS", new GoogleVideosExporter(credentialFactory,jsonFactory));
"PHOTOS", new GooglePhotosExporter(credentialFactory, jobStore, jsonFactory, monitor));
exporterBuilder.put("VIDEOS", new GoogleVideosExporter(credentialFactory, jsonFactory));

exporterMap = exporterBuilder.build();

Expand Down
138 changes: 96 additions & 42 deletions ...rc/main/java/org/datatransferproject/datatransfer/google/videos/GoogleVideosImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,24 +31,37 @@
*/
package org.datatransferproject.datatransfer.google.videos;

import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.json.JsonFactory;
import com.google.api.gax.core.FixedCredentialsProvider;
import com.google.auth.oauth2.AccessToken;
import com.google.auth.oauth2.UserCredentials;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.photos.library.v1.PhotosLibraryClient;
import com.google.photos.library.v1.PhotosLibrarySettings;
import com.google.photos.library.v1.proto.BatchCreateMediaItemsResponse;
import com.google.photos.library.v1.proto.NewMediaItem;
import com.google.photos.library.v1.proto.NewMediaItemResult;
import com.google.photos.library.v1.upload.UploadMediaItemRequest;
import com.google.photos.library.v1.upload.UploadMediaItemResponse;
import com.google.photos.library.v1.upload.UploadMediaItemResponse.Error;
import com.google.photos.library.v1.util.NewMediaItemFactory;
import com.google.rpc.Code;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import org.datatransferproject.api.launcher.Monitor;
import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory;
import org.datatransferproject.datatransfer.google.mediaModels.NewMediaItem;
import org.datatransferproject.datatransfer.google.mediaModels.NewMediaItemResult;
import org.datatransferproject.datatransfer.google.mediaModels.NewMediaItemUpload;
import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore;
import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor;
import org.datatransferproject.spi.transfer.provider.ImportResult;
import org.datatransferproject.spi.transfer.provider.Importer;
import org.datatransferproject.transfer.ImageStreamProvider;
import org.datatransferproject.types.common.models.videos.VideoObject;
import org.datatransferproject.types.common.models.videos.VideosContainerResource;
import org.datatransferproject.types.transfer.auth.AppCredentials;
import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData;

public class GoogleVideosImporter
Expand All @@ -57,29 +70,26 @@ public class GoogleVideosImporter
// TODO: internationalize copy prefix
private static final String COPY_PREFIX = "Copy of ";

private final GoogleCredentialFactory credentialFactory;
private final ImageStreamProvider videoStreamProvider;
private volatile GoogleVideosInterface videosInterface;
private Monitor monitor;
private JsonFactory jsonFactory;
private final AppCredentials appCredentials;
private final TemporaryPerJobDataStore dataStore;

public GoogleVideosImporter(
GoogleCredentialFactory credentialFactory, JsonFactory jsonFactory, Monitor monitor) {
this(credentialFactory, null, new ImageStreamProvider(), jsonFactory, monitor);
AppCredentials appCredentials, TemporaryPerJobDataStore dataStore, Monitor monitor) {
this(new ImageStreamProvider(), monitor, appCredentials, dataStore);
}

@VisibleForTesting
GoogleVideosImporter(
GoogleCredentialFactory credentialFactory,
GoogleVideosInterface videosInterface,
ImageStreamProvider videoStreamProvider,
JsonFactory jsonFactory,
Monitor monitor) {
this.credentialFactory = credentialFactory;
this.videosInterface = videosInterface;
Monitor monitor,
AppCredentials appCredentials,
TemporaryPerJobDataStore dataStore) {
this.videoStreamProvider = videoStreamProvider;
this.jsonFactory = jsonFactory;
this.monitor = monitor;
this.appCredentials = appCredentials;
this.dataStore = dataStore;
}

@Override
Expand All @@ -94,54 +104,98 @@ public ImportResult importItem(
return ImportResult.OK;
}

PhotosLibrarySettings settings =
PhotosLibrarySettings.newBuilder()
.setCredentialsProvider(
FixedCredentialsProvider.create(
UserCredentials.newBuilder()
.setClientId(appCredentials.getKey())
.setClientSecret(appCredentials.getSecret())
.setAccessToken(new AccessToken(authData.getAccessToken(), null))
.setRefreshToken(authData.getRefreshToken())
.build()))
.build();

// Uploads videos
if (data.getVideos() != null && data.getVideos().size() > 0) {
for (VideoObject video : data.getVideos()) {
executor.executeAndSwallowIOExceptions(
video.getDataId(), video.getName(), () -> importSingleVideo(authData, video));
video.getDataId(), video.getName(), () -> importSingleVideo(video, settings));
}
}
return ImportResult.OK;
}

String importSingleVideo(TokensAndUrlAuthData authData, VideoObject inputVideo) throws Exception {
// download video and create input stream
InputStream inputStream;
String importSingleVideo(VideoObject inputVideo, PhotosLibrarySettings settings)
throws Exception {
if (inputVideo.getContentUrl() == null) {
monitor.info(() -> "Content Url is empty. Make sure that you provide a valid content Url.");
return null;
}

inputStream = this.videoStreamProvider.get(inputVideo.getContentUrl().toString());

String filename;
if (Strings.isNullOrEmpty(inputVideo.getName())) {
filename = "untitled";
} else {
filename = COPY_PREFIX + inputVideo.getName();
}

String uploadToken =
getOrCreateVideosInterface(authData).uploadVideoContent(inputStream, filename);

NewMediaItem newMediaItem = new NewMediaItem(filename, uploadToken);

NewMediaItemUpload uploadItem =
new NewMediaItemUpload(null, Collections.singletonList(newMediaItem));
final File tmp;
try (InputStream inputStream =
this.videoStreamProvider.get(inputVideo.getContentUrl().toString())) {
tmp = dataStore.getTempFileFromInputStream(inputStream, filename, ".mp4");
}

final NewMediaItemResult result =
getOrCreateVideosInterface(authData).createVideo(uploadItem).getResults()[0];
return result.getMediaItem().getId();
try (PhotosLibraryClient photosLibraryClient = PhotosLibraryClient.initialize(settings)) {
UploadMediaItemRequest uploadRequest =
UploadMediaItemRequest.newBuilder()
.setFileName(filename)
.setDataFile(new RandomAccessFile(tmp, "r"))
.build();
UploadMediaItemResponse uploadResponse = photosLibraryClient.uploadMediaItem(uploadRequest);
if (uploadResponse.getError().isPresent() || !uploadResponse.getUploadToken().isPresent()) {
Error error = uploadResponse.getError().orElse(null);
throw new IOException(
"An error was encountered while uploading the video.",
error != null ? error.getCause() : null);
} else {
String uploadToken = uploadResponse.getUploadToken().get();
return createMediaItem(inputVideo, photosLibraryClient, uploadToken);
}
} finally {
//noinspection ResultOfMethodCallIgnored
tmp.delete();
}
}

private synchronized GoogleVideosInterface getOrCreateVideosInterface(
TokensAndUrlAuthData authData) {
return videosInterface == null ? makeVideosInterface(authData) : videosInterface;
}
String createMediaItem(
VideoObject inputVideo, PhotosLibraryClient photosLibraryClient, String uploadToken)
throws IOException {
NewMediaItem newMediaItem;
if (inputVideo.getDescription() != null && !inputVideo.getDescription().isEmpty()) {
newMediaItem =
NewMediaItemFactory.createNewMediaItem(uploadToken, inputVideo.getDescription());
} else {
newMediaItem = NewMediaItemFactory.createNewMediaItem(uploadToken);
}

List<NewMediaItem> newItems = Collections.singletonList(newMediaItem);

private synchronized GoogleVideosInterface makeVideosInterface(TokensAndUrlAuthData authData) {
Credential credential = credentialFactory.createCredential(authData);
videosInterface = new GoogleVideosInterface(credential, this.jsonFactory);
return videosInterface;
BatchCreateMediaItemsResponse response = photosLibraryClient.batchCreateMediaItems(newItems);
final List<NewMediaItemResult> resultsList = response.getNewMediaItemResultsList();
if (resultsList.size() != 1) {
throw new IOException("Expected resultsList to be of size 1");
} else {
final NewMediaItemResult itemResult = resultsList.get(0);
final int code = itemResult.getStatus().getCode();
if (code != Code.OK_VALUE) {
throw new IOException(
String.format(
"Video item could not be created. Code: %d Message: %s",
code, itemResult.getStatus().getMessage()));
} else {
return itemResult.getMediaItem().getId();
}
}
}
}
4 changes: 2 additions & 2 deletions ...java/org/datatransferproject/datatransfer/google/calendar/GoogleCalendarExporterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import static org.datatransferproject.datatransfer.google.common.GoogleStaticObjects.MAX_ATTENDEES;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;

import com.google.api.services.calendar.Calendar;
Expand Down Expand Up @@ -97,7 +97,7 @@ public void setup() throws IOException {
when(calendarEvents.list(CALENDAR_ID)).thenReturn(eventListRequest);
when(eventListRequest.setMaxAttendees(MAX_ATTENDEES)).thenReturn(eventListRequest);

verifyZeroInteractions(credentialFactory);
verifyNoInteractions(credentialFactory);
}

@Test
Expand Down
4 changes: 2 additions & 2 deletions ...java/org/datatransferproject/datatransfer/google/calendar/GoogleCalendarImporterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;

public class GoogleCalendarImporterTest {
Expand Down Expand Up @@ -63,7 +63,7 @@ public void setup() {
when(calendarClient.calendars()).thenReturn(calendarCalendars);
when(calendarClient.events()).thenReturn(calendarEvents);

verifyZeroInteractions(credentialFactory);
verifyNoInteractions(credentialFactory);
}

@Test
Expand Down
Loading

0 comments on commit 4e70deb

Please sign in to comment.