Merge pull request #56 from indix/multiple-bucket-support

Multiple bucket support

fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java

@@ -19,12 +19,11 @@ public class FetchExecutor {
  private static Logger logger = Logger.getLoggerFor(FetchTask.class);
 
  public TaskExecutionResult execute(Config config, final Context context) {
- final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
- if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
 
  try {
+ final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
  String artifactPathOnS3 = getArtifactsLocationTemplate(config, env);
- final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
+ final String bucket = getBucket(config, env);
  final S3ArtifactStore store = getS3ArtifactStore(env, bucket);
 
  context.printMessage(String.format("Getting artifacts from %s", store.pathString(artifactPathOnS3)));
@@ -54,6 +53,21 @@ private void setupDestinationDirectory(String destination) {
  }
  }
 
+ private String getBucket(Config config, GoEnvironment env) {
+ String repoName = config.getRepo();
+ String packageName = config.getPkg();
+ String bucketFromMaterial = env.get(String.format("GO_REPO_%s_%s_S3_BUCKET", repoName, packageName));
+ if(bucketFromMaterial != null) {
+ return bucketFromMaterial;
+ }
+
+ if(env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) {
+ throw new RuntimeException("S3 bucket to fetch from should be from material plugin or GO_ARTIFACTS_S3_BUCKET env var.");
+ }
+
+ return env.get(GO_ARTIFACTS_S3_BUCKET);
+ }
+
  private String getArtifactsLocationTemplate(Config config, GoEnvironment env) {
  String repoName = config.getRepo();
  String packageName = config.getPkg();

fetch/src/main/resources/views/task.template.html

@@ -13,6 +13,3 @@
  <input type="text" ng-model="Destination" ng-required="true">
  <span class="form_error" ng-show="GOINPUTNAME[Destination].$error.server">{{ GOINPUTNAME[Destination].$error.server }}</span>
 </div>
-<div class="form_item_block">
- <p class="required">Make sure GO_ARTIFACTS_S3_BUCKET environment variables is present with appropriate values on any of pipeline / Go Environments / on all agent machines. </p>
-</div>

fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java

@@ -36,7 +36,7 @@ public void setUp() throws Exception {
  .with(GO_ARTIFACTS_S3_BUCKET, bucket)
  .with(GO_SERVER_DASHBOARD_URL, "http://go.server:8153")
  .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_LABEL", "20.1")
- .with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket)
+
  .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_PIPELINE_NAME", "TestPublish")
  .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_STAGE_NAME", "defaultStage")
  .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_JOB_NAME", "defaultJob");
@@ -85,9 +85,6 @@ public void shouldBeFailureIfUnableToFetchArtifacts() {
  @Test
  public void shouldBeSuccessResultOnSuccessfulFetch() {
  Map<String, String> mockVariables = mockEnvironmentVariables.build();
- AmazonS3Client mockClient = mockClient();
- S3ArtifactStore store = new S3ArtifactStore(mockClient, bucket);
- doReturn(store).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), eq(bucket));
  S3ArtifactStore mockStore = mockStore();
 
  doReturn(mockStore).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), any(String.class));
@@ -99,6 +96,23 @@ public void shouldBeSuccessResultOnSuccessfulFetch() {
 
  }
 
+ @Test
+ public void shouldGetBucketInfoFromMaterialEnvVars() {
+ Map<String, String> mockVariables = mockEnvironmentVariables
+ .with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket)
+ .with(GO_ARTIFACTS_S3_BUCKET, "")
+ .build();
+ S3ArtifactStore mockStore = mockStore();
+
+ doReturn(mockStore).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), eq(bucket));
+ TaskExecutionResult result = fetchExecutor.execute(config, mockContext(mockVariables));
+
+ assertTrue(result.isSuccessful());
+ assertThat(result.message(), is("Fetched all artifacts"));
+ verify(mockStore, times(1)).getPrefix("TestPublish/defaultStage/defaultJob/20.1", "here/artifacts");
+
+ }
+
  @Test
  public void shouldBeAbleToHandleTaskConfigEntriesWithDashesInTheName() {
  Map<String, String> mockVariables = mockEnvironmentVariables

publish/src/main/java/com/indix/gocd/s3publish/Config.java

@@ -9,18 +9,20 @@
 import java.util.List;
 import java.util.Map;
 
+import static com.indix.gocd.utils.Constants.ARTIFACTS_BUCKET;
 import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX;
 import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS;
 
 public class Config {
 
  public String sourceDestinationsJson;
  public String destinationPrefix;
-
+ public String artifactsBucket;
 
  public Config(Map config) {
  sourceDestinationsJson = getValue(config, SOURCEDESTINATIONS);
  destinationPrefix = getValue(config, DESTINATION_PREFIX);
+ artifactsBucket = getValue(config, ARTIFACTS_BUCKET);
  }
 
  public List<SourceDestination> sourceDestinations() throws JsonSyntaxException {

publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java

@@ -33,10 +33,11 @@ public class PublishExecutor {
  public TaskExecutionResult execute(Config config, final Context context) {
  try {
  final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
- if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
  if (env.isAbsent(GO_SERVER_DASHBOARD_URL)) return envNotFound(GO_SERVER_DASHBOARD_URL);
 
- final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
+ String bucket = getBucket(env, config);
+ if(bucket == null) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
+
  final S3ArtifactStore store = getS3ArtifactStore(env, bucket);
  store.setStorageClass(env.getOrElse(AWS_STORAGE_CLASS, STORAGE_CLASS_STANDARD));
 
@@ -69,6 +70,15 @@ public TaskExecutionResult execute(Config config, final Context context) {
  }
  }
 
+ private String getBucket(GoEnvironment env, Config config) {
+ if(StringUtils.isNotBlank(config.artifactsBucket)) {
+ return config.artifactsBucket;
+ } else if(env.has(GO_ARTIFACTS_S3_BUCKET)) {
+ return env.get(GO_ARTIFACTS_S3_BUCKET);
+ }
+ return null;
+ }
+
  protected S3ArtifactStore getS3ArtifactStore(GoEnvironment env, String bucket) {
  return new S3ArtifactStore(env, bucket);
  }

publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java

@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static com.indix.gocd.utils.Constants.ARTIFACTS_BUCKET;
 import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX;
 import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS;
 import static com.indix.gocd.utils.utils.Lists.foreach;
@@ -110,6 +111,7 @@ public void execute(SourceDestination input) {
 
  private GoPluginApiResponse handleGetConfigRequest() {
  HashMap config = new HashMap();
+
  HashMap sourceDestinations = new HashMap();
  sourceDestinations.put("default-value", "");
  sourceDestinations.put("required", true);
@@ -120,6 +122,11 @@ private GoPluginApiResponse handleGetConfigRequest() {
  destinationPrefix.put("required", false);
  config.put(DESTINATION_PREFIX, destinationPrefix);
 
+ HashMap artifactsBucket = new HashMap();
+ artifactsBucket.put("default-value", "");
+ artifactsBucket.put("required", false);
+ config.put(ARTIFACTS_BUCKET, artifactsBucket);
+
  return createResponse(DefaultGoPluginApiResponse.SUCCESS_RESPONSE_CODE, config);
  }
 

publish/src/main/resources/views/task.template.html

@@ -37,7 +37,18 @@
  </p>
 </div>
 <div class="form_item_block">
- <p class="required">Make sure GO_ARTIFACTS_S3_BUCKET and GO_SERVER_DASHBOARD_URL environment variables are present with appropriate values on any of pipeline / Go Environments / on all agent machines. </p>
+ <label for="artifactsBucket">Artifacts Bucket</label>
+ <input id="artifactsBucket" type="text" ng-model="artifactsBucket" />
+ <span class="form_error" ng-show="GOINPUTNAME[artifactsBucket].$error.server">{{ GOINPUTNAME[artifactsBucket].$error.server }}</span>
+</div>
+<div class="form_item_block">
+ <p>
+ <span ng-show="artifactsBucket">The artifacts will be pushed to the bucket {{artifactsBucket}}</span>
+ <span ng-show="!artifactsBucket">GO_ARTIFACTS_S3_BUCKET env var must be set if the above is not configured.</span>
+ </p>
+</div>
+<div class="form_item_block">
+ <p class="required">Make sure GO_SERVER_DASHBOARD_URL environment variable is present with appropriate value on any of pipeline / Go Environments / on all agent machines. </p>
 </div>
 <script type="text/javascript">
  jQuery(document).ready(function() {

publish/src/test/java/com/indix/gocd/s3publish/PublishExecutorTest.java

@@ -60,6 +60,7 @@ public void shouldFailIfNoFilesToUploadBasedOnSource() {
  Config config = new Config(Maps.builder()
  .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
  .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+ .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
  .build());
 
  TaskExecutionResult result = executeMockPublish(
@@ -83,6 +84,7 @@ public void shouldNotThrowIfAWSUseIAMRoleIsTrueAndAWS_SECRET_ACCESS_KEYNotPresen
  Config config = new Config(Maps.builder()
  .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
  .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+ .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
  .build());
 
  TaskExecutionResult result = executeMockPublish(
@@ -100,18 +102,46 @@ public void shouldNotThrowIfAWSUseIAMRoleIsTrueAndAWS_SECRET_ACCESS_KEYNotPresen
  public void shouldThrowIfGO_ARTIFACTS_S3_BUCKETNotPresent() {
  Map<String, String> mockVariables = mockEnvironmentVariables.with(GO_ARTIFACTS_S3_BUCKET, "").build();
 
+ Config config = new Config(Maps.builder()
+ .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
+ .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+ .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
+ .build());
+
  TaskExecutionResult result = publishExecutor.execute(config, mockContext(mockVariables));
  assertFalse(result.isSuccessful());
  assertThat(result.message(), is("GO_ARTIFACTS_S3_BUCKET environment variable is not set"));
  }
 
+ @Test
+ public void shouldGetBucketFromConfig() {
+ AmazonS3Client mockClient = mockClient();
+
+ Config config = new Config(Maps.builder()
+ .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
+ .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+ .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", testS3Bucket).build())
+ .build());
+
+ TaskExecutionResult result = executeMockPublish(
+ mockClient,
+ config,
+ new String[]{"README.md"},
+ mockEnvironmentVariables.with(GO_ARTIFACTS_S3_BUCKET, "")
+ );
+
+ assertTrue(result.isSuccessful());
+ assertThat(result.message(), is("Published all artifacts to S3 successfully"));
+ }
+
  @Test
  public void shouldGetDisplayMessageAfterUpload() {
  AmazonS3Client mockClient = mockClient();
 
  Config config = new Config(Maps.builder()
  .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
  .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+ .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
  .build());
 
  TaskExecutionResult result = executeMockPublish(
@@ -131,6 +161,7 @@ public void shouldUploadALocalFileToS3WithDefaultPrefix() {
  Config config = new Config(Maps.builder()
  .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
  .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+ .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
  .build());
 
  TaskExecutionResult result = executeMockPublish(
@@ -173,6 +204,7 @@ public void shouldUploadALocalFileToS3WithDestinationPrefix() {
  Config config = new Config(Maps.builder()
  .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
  .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "destinationPrefix").build())
+ .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
  .build());
 
  TaskExecutionResult result = executeMockPublish(
@@ -204,6 +236,7 @@ public void shouldUploadALocalFileToS3WithDestinationPrefixUsingEnvVariable() {
  Config config = new Config(Maps.builder()
  .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
  .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "test/${GO_PIPELINE_COUNTER}/").build())
+ .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
  .build());
 
  TaskExecutionResult result = executeMockPublish(
@@ -235,6 +268,7 @@ public void shouldUploadALocalFileToS3WithSlashDestinationPrefix() {
  Config config = new Config(Maps.builder()
  .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
  .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "/").build())
+ .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
  .build());
 
  TaskExecutionResult result = executeMockPublish(
@@ -258,6 +292,7 @@ public void shouldUploadALocalFileToS3WithSlashDestinationPrefix() {
  assertThat(jarPutRequest.getKey(), is("s3publish-0.1.31.jar"));
  assertNull(jarPutRequest.getMetadata());
  }
+
  private TaskExecutionResult executeMockPublish(final AmazonS3Client mockClient, Config config, String[] files) {
  return executeMockPublish(mockClient, config, files, mockEnvironmentVariables);
  }

utils/src/main/java/com/indix/gocd/utils/Constants.java

@@ -10,6 +10,7 @@ public class Constants {
 
  public static final String SOURCEDESTINATIONS = "sourceDestinations";
  public static final String DESTINATION_PREFIX = "destinationPrefix";
+ public static final String ARTIFACTS_BUCKET = "artifactsBucket";
 
  public static final String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY";
  public static final String AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID";