1616
1717require "open-uri" unless defined? ( OpenURI )
1818require "ruby-progressbar"
19+ require "aws-sdk-s3"
1920
2021module Omnibus
2122 module DownloadHelpers
@@ -28,7 +29,7 @@ module InstanceMethods
2829
2930 #
3031 # Downloads from a given URL to a given path.
31- # Supports direct S3 downloads using AWS SDK when downloading from S3 URLs (if available) ,
32+ # Supports direct S3 downloads using AWS SDK when downloading from S3 URLs,
3233 # and falls back to Ruby's OpenURI implementation for standard HTTP/HTTPS URLs.
3334 #
3435 # @param [String] from_url
@@ -46,48 +47,38 @@ module InstanceMethods
4647 # @raise [Errno::ENETUNREACH]
4748 # @raise [Timeout::Error]
4849 # @raise [OpenURI::HTTPError]
50+ # @raise [Aws::S3::Errors::ServiceError]
4951 #
5052 # @return [void]
5153 #
5254 def download_file! ( from_url , to_path , download_options = { } )
5355 options = download_options . dup
5456
55- # Try S3 download first if this is an S3 URL and we have S3 credentials
57+ # Try S3 download first if this looks like an S3 URL and we have S3 configuration
5658 if is_s3_url? ( from_url ) && has_s3_credentials?
5759 begin
5860 log . info ( log_key ) { "Attempting S3 direct download for: #{ from_url } " }
5961
60- # Use AWS SDK for S3 direct download if aws-sdk-s3 is available
61- begin
62- require "aws-sdk-s3"
63-
64- # Parse S3 URL to extract bucket and key
65- uri = URI ( from_url )
66- bucket , key = extract_s3_bucket_and_key ( from_url , uri )
67-
68- log . debug ( log_key ) { "S3 bucket: #{ bucket } , key: #{ key } " }
69-
70- # Create S3 client with credentials from Config
71- s3_client = create_s3_client
72-
73- # Download directly to the destination path
74- s3_client . get_object (
75- bucket : bucket ,
76- key : key ,
77- response_target : to_path
78- )
79-
80- log . debug ( log_key ) { "Successfully downloaded S3 object to #{ to_path } " }
81- return # Exit early if S3 download succeeds
82- rescue LoadError => e
83- log . warn ( log_key ) { "AWS SDK for S3 not available: #{ e . message } . Falling back to HTTP download." }
84- # Fall through to regular HTTP download if aws-sdk-s3 isn't available
85- rescue => e
86- log . warn ( log_key ) { "S3 download failed: #{ e . message } . Falling back to HTTP download." }
87- # Fall through to regular HTTP download
88- end
62+ # Parse S3 URL to extract bucket and key
63+ uri = URI ( from_url )
64+ bucket , key = extract_s3_bucket_and_key ( from_url , uri )
65+
66+ log . debug ( log_key ) { "S3 bucket: #{ bucket } , key: #{ key } " }
67+
68+ # Create S3 client with credentials from Config
69+ s3_client = create_s3_client
70+
71+ # Download directly to the destination path
72+ s3_client . get_object (
73+ bucket : bucket ,
74+ key : key ,
75+ response_target : to_path
76+ )
77+
78+ log . debug ( log_key ) { "Successfully downloaded S3 object to #{ to_path } " }
79+ return # Exit early if S3 download succeeds
8980 rescue => e
90- log . warn ( log_key ) { "S3 download setup failed: #{ e . message } . Falling back to HTTP download." }
81+ log . warn ( log_key ) { "S3 download failed: #{ e . message } . Falling back to HTTP download." }
9182 # Fall through to regular HTTP download
9283 end
9384 end
@@ -98,39 +89,27 @@ def download_file!(from_url, to_path, download_options = {})
9889 enable_progress_bar = options . delete ( :enable_progress_bar )
9990 enable_progress_bar = true if enable_progress_bar . nil?
10091
101- # Safely access download_headers or use empty hash
102- headers = { }
103- begin
104- headers = download_headers if respond_to? ( :download_headers )
105- rescue => e
106- log . warn ( log_key ) { "Error getting download headers: #{ e . message } . Using empty headers." }
107- end
108- options . merge! ( headers )
109-
92+ # Safely merge download headers if they exist
93+ options . merge! ( download_headers || { } )
11094 options [ :read_timeout ] = Config . fetcher_read_timeout
11195
11296 fetcher_retries ||= Config . fetcher_retries
11397
11498 reported_total = 0
11599 if enable_progress_bar
116- begin
117- progress_bar = ProgressBar . create (
118- output : $stdout,
119- format : "%e %B %p%% (%r KB/sec)" ,
120- rate_scale : -> ( rate ) { rate / 1024 }
121- )
100+ progress_bar = ProgressBar . create (
101+ output : $stdout,
102+ format : "%e %B %p%% (%r KB/sec)" ,
103+ rate_scale : -> ( rate ) { rate / 1024 }
104+ )
122105
123- options [ :content_length_proc ] = -> ( total ) do
124- reported_total = total
125- progress_bar . total = total
126- end
127- options [ :progress_proc ] = -> ( step ) do
128- downloaded_amount = reported_total ? [ step , reported_total ] . min : step
129- progress_bar . progress = downloaded_amount
130- end
131- rescue => e
132- log . warn ( log_key ) { "Could not create progress bar: #{ e . message } " }
133- # Proceed without progress bar
106+ options [ :content_length_proc ] = -> ( total ) do
107+ reported_total = total
108+ progress_bar . total = total
109+ end
110+ options [ :progress_proc ] = -> ( step ) do
111+ downloaded_amount = reported_total ? [ step , reported_total ] . min : step
112+ progress_bar . progress = downloaded_amount
134113 end
135114 end
136115
@@ -160,6 +139,17 @@ def download_file!(from_url, to_path, download_options = {})
160139 end
161140 end
162141
142+ #
143+ # Default empty implementation of download_headers
144+ # This can be overridden by classes that include this module
145+ #
146+ # @return [Hash]
147+ # empty hash of headers by default
148+ #
149+ def download_headers
150+ { }
151+ end
152+
163153 #
164154 # Checks if a URL appears to be an S3 URL
165155 #
@@ -184,10 +174,9 @@ def is_s3_url?(url)
184174 # true if S3 credentials are available, false otherwise
185175 #
186176 def has_s3_credentials?
187- Config . use_s3_caching &&
188- ( Config . s3_iam_role_arn ||
189- Config . s3_profile ||
190- ( Config . s3_access_key && Config . s3_secret_key ) )
177+ Config . s3_iam_role_arn ||
178+ Config . s3_profile ||
179+ ( Config . s3_access_key && Config . s3_secret_key )
191180 end
192181
193182 #
@@ -248,6 +237,9 @@ def create_s3_client
248237
249238 Aws ::S3 ::Client . new ( params )
250239 end
240+
241+ # Rest of the file remains unchanged
242+ # ...
251243 end
252244 end
253245end
0 commit comments