Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Large file support on 32bit platforms #1636

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Large file support on 32bit platforms #1636

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
33 changes: 17 additions & 16 deletions Release/include/cpprest/details/fileio.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,16 +54,16 @@ struct _file_info

// Positional data

size_t m_rdpos;
size_t m_wrpos;
utility::size64_t m_rdpos;
utility::size64_t m_wrpos;
bool m_atend;

// Input buffer

size_t m_buffer_size; // The intended size of the buffer to read into.
char* m_buffer;

size_t m_bufoff; // File position that the start of the buffer represents.
utility::size64_t m_bufoff; // File position that the start of the buffer represents.
msl::safeint3::SafeInt<size_t> m_bufsize; // Buffer allocated size, as actually allocated.
size_t m_buffill; // Amount of file data actually in the buffer

Expand Down Expand Up @@ -176,7 +176,8 @@ extern "C"
/// </summary>
/// <param name="info">The file info record of the file</param>
/// <param name="callback">A pointer to the callback interface to invoke when the write request is
/// completed.</param> <returns><c>true</c> if the request was initiated</returns>
/// completed.</param>
/// <returns><c>true</c> if the request was initiated</returns>
_ASYNCRTIMP bool __cdecl _sync_fsb(_In_ concurrency::streams::details::_file_info* info,
_In_ concurrency::streams::details::_filestream_callback* callback);

Expand All @@ -193,28 +194,28 @@ extern "C"
/// </summary>
/// <param name="info">The file info record of the file</param>
/// <param name="pos">The new position (offset from the start) in the file stream</param>
/// <returns><c>true</c> if the request was initiated</returns>
_ASYNCRTIMP size_t __cdecl _seekrdpos_fsb(_In_ concurrency::streams::details::_file_info* info,
size_t pos,
/// <returns>New file position or (utility::size64_t)-1 on error</returns>
_ASYNCRTIMP utility::size64_t __cdecl _seekrdpos_fsb(_In_ concurrency::streams::details::_file_info* info,
utility::size64_t pos,
size_t char_size);

/// <summary>
/// Adjust the internal buffers and pointers when the application seeks to a new read location in the stream.
/// </summary>
/// <param name="info">The file info record of the file</param>
/// <param name="pos">The new position (offset from the start) in the file stream</param>
/// <returns><c>true</c> if the request was initiated</returns>
_ASYNCRTIMP size_t __cdecl _seekrdtoend_fsb(_In_ concurrency::streams::details::_file_info* info,
int64_t offset,
size_t char_size);
/// <param name="offset">offset relative to the end of the stream</param>
/// <returns>New file position or (utility::size64_t)-1 on error</returns>
_ASYNCRTIMP utility::size64_t __cdecl _seekrdtoend_fsb(_In_ concurrency::streams::details::_file_info* info,
int64_t offset,
size_t char_size);

/// <summary>
/// Adjust the internal buffers and pointers when the application seeks to a new write location in the stream.
/// </summary>
/// <param name="info">The file info record of the file</param>
/// <param name="pos">The new position (offset from the start) in the file stream</param>
/// <returns><c>true</c> if the request was initiated</returns>
_ASYNCRTIMP size_t __cdecl _seekwrpos_fsb(_In_ concurrency::streams::details::_file_info* info,
size_t pos,
size_t char_size);
/// <returns>New file position or (utility::size64_t)-1 on error</returns>
_ASYNCRTIMP utility::size64_t __cdecl _seekwrpos_fsb(_In_ concurrency::streams::details::_file_info* info,
utility::size64_t pos,
size_t char_size);
}
32 changes: 16 additions & 16 deletions Release/include/cpprest/filestream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ class basic_file_buffer : public details::streambuf_state_manager<_CharType>

if (_in_avail_unprot() > 0)
{
auto bufoff = m_info->m_rdpos - m_info->m_bufoff;
auto bufoff = static_cast<size_t>(m_info->m_rdpos - m_info->m_bufoff);
_CharType ch = m_info->m_buffer[bufoff * sizeof(_CharType)];
m_info->m_rdpos += 1;
return pplx::task_from_result<int_type>(ch);
Expand Down Expand Up @@ -423,7 +423,7 @@ class basic_file_buffer : public details::streambuf_state_manager<_CharType>

if (_in_avail_unprot() == 0) return traits::requires_async();

auto bufoff = m_info->m_rdpos - m_info->m_bufoff;
auto bufoff = static_cast<size_t>(m_info->m_rdpos - m_info->m_bufoff);
_CharType ch = m_info->m_buffer[bufoff * sizeof(_CharType)];
m_info->m_rdpos += 1;
return (int_type)ch;
Expand All @@ -439,7 +439,7 @@ class basic_file_buffer : public details::streambuf_state_manager<_CharType>

if (_in_avail_unprot() > 0)
{
auto bufoff = m_info->m_rdpos - m_info->m_bufoff;
auto bufoff = static_cast<size_t>(m_info->m_rdpos - m_info->m_bufoff);
_CharType ch = m_info->m_buffer[bufoff * sizeof(_CharType)];
return pplx::task_from_result<int_type>(ch);
}
Expand Down Expand Up @@ -485,7 +485,7 @@ class basic_file_buffer : public details::streambuf_state_manager<_CharType>

if (_in_avail_unprot() == 0) return traits::requires_async();

auto bufoff = m_info->m_rdpos - m_info->m_bufoff;
auto bufoff = static_cast<size_t>(m_info->m_rdpos - m_info->m_bufoff);
_CharType ch = m_info->m_buffer[bufoff * sizeof(_CharType)];
return (int_type)ch;
}
Expand Down Expand Up @@ -538,7 +538,7 @@ class basic_file_buffer : public details::streambuf_state_manager<_CharType>

if (_in_avail_unprot() >= count)
{
auto bufoff = m_info->m_rdpos - m_info->m_bufoff;
auto bufoff = static_cast<size_t>(m_info->m_rdpos - m_info->m_bufoff);
std::memcpy(
(void*)ptr, this->m_info->m_buffer + bufoff * sizeof(_CharType), count * sizeof(_CharType));

Expand Down Expand Up @@ -582,7 +582,7 @@ class basic_file_buffer : public details::streambuf_state_manager<_CharType>
size_t available = _in_avail_unprot();
size_t copy = (count < available) ? count : available;

auto bufoff = m_info->m_rdpos - m_info->m_bufoff;
auto bufoff = static_cast<size_t>(m_info->m_rdpos - m_info->m_bufoff);
std::memcpy((void*)ptr, this->m_info->m_buffer + bufoff * sizeof(_CharType), copy * sizeof(_CharType));

m_info->m_rdpos += copy;
Expand Down Expand Up @@ -626,11 +626,11 @@ class basic_file_buffer : public details::streambuf_state_manager<_CharType>
if (mode == std::ios_base::in)
{
m_readOps.wait();
return (pos_type)_seekrdpos_fsb(m_info, size_t(pos), sizeof(_CharType));
return (pos_type)_seekrdpos_fsb(m_info, pos, sizeof(_CharType));
}
else if ((m_info->m_mode & std::ios::ios_base::app) == 0)
{
return (pos_type)_seekwrpos_fsb(m_info, size_t(pos), sizeof(_CharType));
return (pos_type)_seekwrpos_fsb(m_info, pos, sizeof(_CharType));
}
return (pos_type)Concurrency::streams::char_traits<_CharType>::eof();
}
Expand All @@ -649,15 +649,15 @@ class basic_file_buffer : public details::streambuf_state_manager<_CharType>
if (mode == std::ios_base::in)
{
m_readOps.wait();
size_t current_pos = static_cast<size_t>(-1);
auto current_pos = static_cast<utility::size64_t>(-1);
switch (way)
{
case std::ios_base::beg: return (pos_type)_seekrdpos_fsb(m_info, size_t(offset), sizeof(_CharType));
case std::ios_base::beg: return (pos_type)_seekrdpos_fsb(m_info, offset, sizeof(_CharType));
case std::ios_base::cur:
return (pos_type)_seekrdpos_fsb(m_info, size_t(m_info->m_rdpos + offset), sizeof(_CharType));
return (pos_type)_seekrdpos_fsb(m_info, m_info->m_rdpos + offset, sizeof(_CharType));
case std::ios_base::end:
current_pos = _seekrdtoend_fsb(m_info, int64_t(offset), sizeof(_CharType));
if (current_pos == static_cast<size_t>(-1))
current_pos = _seekrdtoend_fsb(m_info, offset, sizeof(_CharType));
if (current_pos == static_cast<utility::size64_t>(-1))
{
return -1;
}
Expand All @@ -671,10 +671,10 @@ class basic_file_buffer : public details::streambuf_state_manager<_CharType>
{
switch (way)
{
case std::ios_base::beg: return (pos_type)_seekwrpos_fsb(m_info, size_t(offset), sizeof(_CharType));
case std::ios_base::beg: return (pos_type)_seekwrpos_fsb(m_info, offset, sizeof(_CharType));
case std::ios_base::cur:
return (pos_type)_seekwrpos_fsb(m_info, size_t(m_info->m_wrpos + offset), sizeof(_CharType));
case std::ios_base::end: return (pos_type)_seekwrpos_fsb(m_info, size_t(-1), sizeof(_CharType));
return (pos_type)_seekwrpos_fsb(m_info, m_info->m_wrpos + offset, sizeof(_CharType));
case std::ios_base::end: return (pos_type)_seekwrpos_fsb(m_info, utility::size64_t(-1), sizeof(_CharType));
default:
// Fail on invalid input (_S_ios_seekdir_end)
assert(false);
Expand Down
48 changes: 24 additions & 24 deletions Release/src/streams/fileio_posix.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ bool _finish_create(int fh, _filestream_callback* callback, std::ios_base::openm

if (mode & std::ios_base::app || mode & std::ios_base::ate)
{
info->m_wrpos = static_cast<size_t>(-1); // Start at the end of the file.
info->m_wrpos = static_cast<utility::size64_t>(-1); // Start at the end of the file.
}

callback->on_opened(info);
Expand Down Expand Up @@ -248,23 +248,23 @@ size_t _write_file_async(Concurrency::streams::details::_file_info_impl* fInfo,
Concurrency::streams::details::_filestream_callback* callback,
const void* ptr,
size_t count,
size_t position)
utility::size64_t position)
{
++fInfo->m_outstanding_writes;

pplx::create_task([=]() -> void {
off_t abs_position;
bool must_restore_pos;
off_t orig_pos;
if (position == static_cast<size_t>(-1))
if (position == static_cast<utility::size64_t>(-1))
{
orig_pos = lseek(fInfo->m_handle, 0, SEEK_CUR);
abs_position = lseek(fInfo->m_handle, 0, SEEK_END);
must_restore_pos = true;
}
else
{
abs_position = position;
abs_position = static_cast<off_t>(position);
orig_pos = 0;
must_restore_pos = false;
}
Expand Down Expand Up @@ -316,10 +316,10 @@ size_t _read_file_async(Concurrency::streams::details::_file_info_impl* fInfo,
Concurrency::streams::details::_filestream_callback* callback,
void* ptr,
size_t count,
size_t offset)
utility::size64_t offset)
{
pplx::create_task([=]() -> void {
auto bytes_read = pread(fInfo->m_handle, ptr, count, offset);
auto bytes_read = pread(fInfo->m_handle, ptr, count, static_cast<off_t>(offset));
if (bytes_read < 0)
{
callback->on_error(std::make_exception_ptr(utility::details::create_system_error(errno)));
Expand Down Expand Up @@ -391,8 +391,8 @@ size_t _fill_buffer_fsb(_file_info_impl* fInfo, _filestream_callback* callback,
// First, we need to understand how far into the buffer we have already read
// and how much remains.

size_t bufpos = fInfo->m_rdpos - fInfo->m_bufoff;
size_t bufrem = fInfo->m_buffill - bufpos;
auto bufpos = fInfo->m_rdpos - fInfo->m_bufoff;
auto bufrem = static_cast<size_t>(fInfo->m_buffill - bufpos);

if (bufrem < count)
{
Expand Down Expand Up @@ -457,7 +457,7 @@ size_t _getn_fsb(Concurrency::streams::details::_file_info* info,
{
auto cb = create_callback(fInfo, callback, [=](size_t read) {
auto copy = (std::min)(read, byteCount);
auto bufoff = fInfo->m_rdpos - fInfo->m_bufoff;
auto bufoff = static_cast<size_t>(fInfo->m_rdpos - fInfo->m_bufoff);
memcpy(ptr, fInfo->m_buffer + bufoff * charSize, copy);
fInfo->m_atend = copy < byteCount;
callback->on_completed(copy);
Expand All @@ -468,7 +468,7 @@ size_t _getn_fsb(Concurrency::streams::details::_file_info* info,
if (static_cast<int>(read) > 0)
{
auto copy = (std::min)(read, byteCount);
auto bufoff = fInfo->m_rdpos - fInfo->m_bufoff;
auto bufoff = static_cast<size_t>(fInfo->m_rdpos - fInfo->m_bufoff);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is concerning to me since it seems like it could cause truncation on a size passed to memcpy, and may need a validation check.

memcpy(ptr, fInfo->m_buffer + bufoff * charSize, copy);
fInfo->m_atend = copy < byteCount;
return copy;
Expand Down Expand Up @@ -509,7 +509,7 @@ size_t _putn_fsb(Concurrency::streams::details::_file_info* info,

// To preserve the async write order, we have to move the write head before read.
auto lastPos = fInfo->m_wrpos;
if (fInfo->m_wrpos != static_cast<size_t>(-1))
if (fInfo->m_wrpos != static_cast<utility::size64_t>(-1))
{
fInfo->m_wrpos += count;
lastPos *= charSize;
Expand Down Expand Up @@ -550,15 +550,15 @@ bool _sync_fsb(Concurrency::streams::details::_file_info* info,
/// <param name="info">The file info record of the file</param>
/// <param name="pos">The new position (offset from the start) in the file stream</param>
/// <returns>New file position or -1 if error</returns>
size_t _seekrdtoend_fsb(Concurrency::streams::details::_file_info* info, int64_t offset, size_t char_size)
utility::size64_t _seekrdtoend_fsb(Concurrency::streams::details::_file_info* info, int64_t offset, size_t char_size)
{
if (info == nullptr) return static_cast<size_t>(-1);
if (info == nullptr) return static_cast<utility::size64_t>(-1);

_file_info_impl* fInfo = static_cast<_file_info_impl*>(info);

pplx::extensibility::scoped_recursive_lock_t lock(info->m_lock);

if (fInfo->m_handle == -1) return static_cast<size_t>(-1);
if (fInfo->m_handle == -1) return static_cast<utility::size64_t>(-1);

if (fInfo->m_buffer != nullptr)
{
Expand All @@ -569,21 +569,21 @@ size_t _seekrdtoend_fsb(Concurrency::streams::details::_file_info* info, int64_t

auto newpos = lseek(fInfo->m_handle, static_cast<off_t>(offset * char_size), SEEK_END);

if (newpos == -1) return static_cast<size_t>(-1);
if (newpos == -1) return static_cast<utility::size64_t>(-1);

fInfo->m_rdpos = static_cast<size_t>(newpos) / char_size;
fInfo->m_rdpos = newpos / char_size;
return fInfo->m_rdpos;
}

utility::size64_t _get_size(_In_ concurrency::streams::details::_file_info* info, size_t char_size)
{
if (info == nullptr) return static_cast<size_t>(-1);
if (info == nullptr) return static_cast<utility::size64_t>(-1);

_file_info_impl* fInfo = static_cast<_file_info_impl*>(info);

pplx::extensibility::scoped_recursive_lock_t lock(info->m_lock);

if (fInfo->m_handle == -1) return static_cast<size_t>(-1);
if (fInfo->m_handle == -1) return static_cast<utility::size64_t>(-1);

if (fInfo->m_buffer != nullptr)
{
Expand Down Expand Up @@ -611,15 +611,15 @@ utility::size64_t _get_size(_In_ concurrency::streams::details::_file_info* info
/// <param name="info">The file info record of the file</param>
/// <param name="pos">The new position (offset from the start) in the file stream</param>
/// <returns>New file position or -1 if error</returns>
size_t _seekrdpos_fsb(Concurrency::streams::details::_file_info* info, size_t pos, size_t)
utility::size64_t _seekrdpos_fsb(Concurrency::streams::details::_file_info* info, utility::size64_t pos, size_t)
{
if (info == nullptr) return static_cast<size_t>(-1);
if (info == nullptr) return static_cast<utility::size64_t>(-1);

_file_info_impl* fInfo = static_cast<_file_info_impl*>(info);

pplx::extensibility::scoped_recursive_lock_t lock(info->m_lock);

if (fInfo->m_handle == -1) return static_cast<size_t>(-1);
if (fInfo->m_handle == -1) return static_cast<utility::size64_t>(-1);

if (pos < fInfo->m_bufoff || pos > (fInfo->m_bufoff + fInfo->m_buffill))
{
Expand All @@ -638,15 +638,15 @@ size_t _seekrdpos_fsb(Concurrency::streams::details::_file_info* info, size_t po
/// <param name="info">The file info record of the file</param>
/// <param name="pos">The new position (offset from the start) in the file stream</param>
/// <returns>New file position or -1 if error</returns>
size_t _seekwrpos_fsb(Concurrency::streams::details::_file_info* info, size_t pos, size_t)
utility::size64_t _seekwrpos_fsb(Concurrency::streams::details::_file_info* info, utility::size64_t pos, size_t)
{
if (info == nullptr) return static_cast<size_t>(-1);
if (info == nullptr) return static_cast<utility::size64_t>(-1);

_file_info_impl* fInfo = static_cast<_file_info_impl*>(info);

pplx::extensibility::scoped_recursive_lock_t lock(info->m_lock);

if (fInfo->m_handle == -1) return static_cast<size_t>(-1);
if (fInfo->m_handle == -1) return static_cast<utility::size64_t>(-1);

fInfo->m_wrpos = pos;
return fInfo->m_wrpos;
Expand Down