diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a18c124..cf5e26a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,13 @@ - **CHANGED API** `Longtail_JobAPI_JobFunc` renamed `is_cancelled` to `detected_error`, now contains first error returned from a job task in the same job group (if any) or ECANCELLED if job group was cancelled If `detected_error` is non-zero, try to exit (and cleanup) your task directly and return `0`. - **CHANGED_API** JobAPI `WaitForAllJobs` now returns first error encountered in a job group for a task as well as any error in the job api itself, removing the need to book keep the error for tasks separately -- **ADDED** memtracer now tracks allocations in stb_ds -- **ADDED** memtracer now tracks allocations in zstd +- **CHANGED API** `Longtail_StorageAPI.OpenAppend` added to `Longtail_StorageAPI` to open files without truncating existing data +- **CHANGED API** `Longtail_CreateConcurrentChunkWriteAPI` changed to take `source_version_index` and `version_diff` +- **CHANGED API** `Longtail_ConcurrentChunkWriteAPI` refactored to use asset index and open/close files instead of keeping all open during entire lifetime + - `Longtail_ConcurrentChunkWriteAPI.CreateDir` now takes asset index instead of version local path + - `Longtail_ConcurrentChunkWriteAPI.Open` now takes asset index instead of version local path and dropping `chunk_write_count` parameter + - `Longtail_ConcurrentChunkWriteAPI.Write` now takes asset index instead of version local path and dropping `chunk_write_count` parameter +- **CHANGED API** `Longtail_SetMonitor` callback functions refactored to accomodate changes in `Longtail_ConcurrentChunkWriteAPI` - **NEW API** `Longtail_SetReAllocAndFree` - **NEW API** `Longtail_ReAlloc` - **NEW API** `Longtail_MemTracer_ReAlloc` @@ -12,12 +17,31 @@ - **NEW API** `Longtail_GetFilesRecursively2` that executes using parallel jobs improving execution speed for large file trees significantly - **REMOVED API** `Longtail_SetAllocAndFree` is replaced by `Longtail_SetReAllocAndFree` - **REMOVED API** `Longtail_MemTracer_Alloc` is replaced by `Longtail_MemTracer_ReAlloc` -- **CHANGED API** `Longtail_ConcurrentChunkWriteAPI::Write` has new parameter `out_chunks_remaining` which is set to the remaining number of chunks to write to asset after call completes +- **REMOVED API** Remove platform api for Read/Write mutex + - `Longtail_GetRWLockSize` + - `Longtail_CreateRWLock` + - `Longtail_DeleteRWLock` + - `Longtail_LockRWLockRead` + - `Longtail_LockRWLockWrite` + - `Longtail_UnlockRWLockRead` + - `Longtail_UnlockRWLockWrite` +- **ADDED** memtracer now tracks allocations in stb_ds +- **ADDED** memtracer now tracks allocations in zstd - **FIXED** Fixed memory leaks in command tool - **FIXED** `Longtail_ChangeVersion2()` can now handle workloads with a block count larger than 65535 - **FIXED** Bikeshed JobAPI implementation does efficient wait when task queue is full - **FIXED** Bikeshed JobAPI::CreateJobs implementation now properly drains both task channels when task queue is full - **FIXED** Make sure we retain order of assets with equal length when sorting them +- **FIXED** Fixed excessive "Disk Used" increase during `Longtail_ChangeVersion2` execution causing Out Of Disk space errors. + The changes also improves performance for more common cases with smaller archive sizes (60 Gb raw data/many files) but causes a small regression compared to 0.4.1 for archives with many very large files. It is still performing much more reasonable than 0.4.0 for these cases. + | Version | Files | Raw Size | Compressed Size | Unpack Time | Peak Memory | + |-|-|-|-|-|-| + |0.4.0|1019|735 GB|214 GB|2h44m26s|7.9 GB| + |0.4.1|1019|735 GB|214 GB|0h12m14s|1.9 GB| + |0.4.2|1019|735 GB|214 GB|0h13m25s|2.2 GB| + |0.4.0|239 340|60 GB|17 GB|0h01m24s|4.2 GB| + |0.4.1|239 340|60 GB|17 GB|0h02m48s|0.9 GB| + |0.4.2|239 340|60 GB|17 GB|0h01m12s|0.9 GB| - **CHANGED** Refactored all internal usage of JobAPI `ReadyJobs` with new error handling - **UPDATED** Update of ZStd: 1.5.5 https://github.com/facebook/zstd/releases/tag/v1.5.5 - **UPDATED** Update of Blake3: 1.5.0 https://github.com/BLAKE3-team/BLAKE3/releases/tag/1.5.0 diff --git a/cmd/main.c b/cmd/main.c index dfc31427..33bfb83c 100644 --- a/cmd/main.c +++ b/cmd/main.c @@ -144,7 +144,7 @@ struct AssetInfo { TLongtail_Atomic64 m_AccessCount; uint32_t m_TotalChunkCount; - TLongtail_Atomic64 m_WriteCount; + TLongtail_Atomic64 m_ReadWriteCount; TLongtail_Atomic64 m_PendingChunkCount; TLongtail_Atomic32 m_ActivityIndicator; }; @@ -152,7 +152,7 @@ struct AssetInfo uint32_t MonitorAssetInfosCount = 0; struct AssetInfo* MonitorAssetInfos = 0; -void MonitorAssetRemove(const struct Longtail_VersionIndex* version_index, uint32_t asset_index) +void MonitorAssetRemove(const struct Longtail_VersionIndex* version_index, uint32_t asset_index, int err) { if (MonitorAssetInfos) { @@ -161,34 +161,26 @@ void MonitorAssetRemove(const struct Longtail_VersionIndex* version_index, uint3 } } -void MonitorAssetOpen(const struct Longtail_VersionIndex* version_index, uint32_t asset_index) +void MonitorAssetOpen(const struct Longtail_VersionIndex* version_index, uint32_t asset_index, int err) { if (MonitorAssetInfos) { Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_AccessCount, 1); - Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_WriteCount, 1); + Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_ReadWriteCount, 1); + Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_PendingChunkCount, 1); } } -void MonitorAssetWrite(const struct Longtail_StoreIndex* store_index, const struct Longtail_VersionIndex* version_index, uint32_t asset_index, uint64_t write_offset, uint32_t size, uint32_t chunk_index, uint32_t chunk_index_in_block, uint32_t chunk_count_in_block, uint32_t block_index, uint32_t block_data_offset) +void MonitorAssetWrite(const struct Longtail_StoreIndex* store_index, const struct Longtail_VersionIndex* version_index, uint32_t asset_index, uint64_t write_offset, uint32_t size, uint32_t chunk_index, uint32_t chunk_index_in_block, uint32_t chunk_count_in_block, uint32_t block_index, uint32_t block_data_offset, int err) { if (MonitorAssetInfos) { Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_AccessCount, 1); - Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_WriteCount, 1); + Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_ReadWriteCount, 1); Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_PendingChunkCount, -((int64_t)chunk_count_in_block)); } } -void MonitorAssetComplete(const struct Longtail_VersionIndex* version_index, uint32_t asset_index, int err) -{ - if (MonitorAssetInfos) - { - Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_AccessCount, 1); -// Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_WriteCount, 1); - } -} - struct MonitorChunkInfo { TLongtail_Atomic64 m_AccessCount; @@ -197,7 +189,7 @@ struct MonitorChunkInfo uint32_t MonitorChunkInfosCount = 0; struct MonitorChunkInfo* MonitorChunkInfos = 0; -void MonitorChunkRead(const struct Longtail_StoreIndex* store_index, const struct Longtail_VersionIndex* version_index, uint32_t block_index, uint32_t chunk_index, uint32_t chunk_index_in_block) +void MonitorChunkRead(const struct Longtail_StoreIndex* store_index, const struct Longtail_VersionIndex* version_index, uint32_t block_index, uint32_t chunk_index, uint32_t chunk_index_in_block, int err) { if (MonitorBlockInfos) { @@ -237,22 +229,22 @@ void MonitorBlockSaved(const struct Longtail_StoreIndex* store_index, uint32_t b } } -void MonitorAssetRead(const struct Longtail_StoreIndex* store_index, const struct Longtail_VersionIndex* version_index, uint32_t asset_index, uint64_t read_offset, uint32_t size, TLongtail_Hash chunk_hash, uint32_t block_index, uint32_t block_data_offset) +void MonitorAssetRead(const struct Longtail_StoreIndex* store_index, const struct Longtail_VersionIndex* version_index, uint32_t asset_index, uint64_t read_offset, uint32_t size, TLongtail_Hash chunk_hash, uint32_t block_index, uint32_t block_data_offset, int err) { if (MonitorAssetInfos) { Longtail_AtomicAdd64(&MonitorBlockInfos[block_index].m_AccessCount, 1); - Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_AccessCount, 1); - Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_WriteCount, 1); + Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_ReadWriteCount, 1); } } -void MonitorAssetClose(const struct Longtail_VersionIndex* version_index, uint32_t asset_index, int err) +void MonitorAssetClose(const struct Longtail_VersionIndex* version_index, uint32_t asset_index) { if (MonitorAssetInfos) { Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_AccessCount, 1); - Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_PendingChunkCount, -MonitorAssetInfos[asset_index].m_PendingChunkCount); + Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_ReadWriteCount, 1); + Longtail_AtomicAdd64(&MonitorAssetInfos[asset_index].m_PendingChunkCount, -1); } } @@ -415,7 +407,7 @@ static int UpdateProgressWindow() Longtail_AtomicAdd32(&MonitorAssetInfos[a].m_ActivityIndicator, -1); continue; } - if (MonitorAssetInfos[a].m_WriteCount == 0) + if (MonitorAssetInfos[a].m_ReadWriteCount == 0) { SetAsset(a, Grey); } @@ -550,7 +542,7 @@ static int UpdateProgressWindow() } else { - if (MonitorAssetInfos[a].m_WriteCount == 0) + if (MonitorAssetInfos[a].m_ReadWriteCount == 0) { tmp_buffer[u] = ' '; incomplete_asset_count++; @@ -622,13 +614,6 @@ void InitMonitor(struct Longtail_StoreIndex* store_index, struct Longtail_Versio MonitorAssetInfos[a].m_PendingChunkCount = version_index->m_AssetChunkCounts[a]; } } - else - { - for (uint32_t a = 0; a < MonitorAssetInfosCount; a++) - { - MonitorAssetInfos[a].m_PendingChunkCount = 1; - } - } MonitorChunkInfosCount = (*version_index->m_ChunkCount); size_t MonitorChunkInfosSize = sizeof(struct MonitorChunkInfo) * MonitorChunkInfosCount; @@ -645,7 +630,7 @@ void InitMonitor(struct Longtail_StoreIndex* store_index, struct Longtail_Versio MonitorAssetOpen, MonitorAssetWrite, MonitorChunkRead, - MonitorAssetComplete, +// MonitorAssetComplete, MonitorBlockCompose, MonitorBlockSave, MonitorBlockSaved, @@ -1541,7 +1526,7 @@ int DownSync( struct Longtail_ProgressAPI* progress = MakeProgressAPI("Updating version", enable_detailed_progress ? 0 : 5); if (progress) { - struct Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write = Longtail_CreateConcurrentChunkWriteAPI(storage_api, target_path); + struct Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write = Longtail_CreateConcurrentChunkWriteAPI(storage_api, source_version_index, version_diff, target_path); if (concurrent_chunk_write) { err = Longtail_ChangeVersion2( @@ -2663,7 +2648,7 @@ int Unpack( struct Longtail_ProgressAPI* progress = MakeProgressAPI("Updating version", enable_detailed_progress ? 0 : 5); if (progress) { - struct Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write = Longtail_CreateConcurrentChunkWriteAPI(storage_api, target_path); + struct Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write = Longtail_CreateConcurrentChunkWriteAPI(storage_api, &archive_index->m_VersionIndex, version_diff, target_path); if (concurrent_chunk_write) { err = Longtail_ChangeVersion2( diff --git a/lib/blockstorestorage/longtail_blockstorestorage.c b/lib/blockstorestorage/longtail_blockstorestorage.c index b9881a6c..dfe6ddef 100644 --- a/lib/blockstorestorage/longtail_blockstorestorage.c +++ b/lib/blockstorestorage/longtail_blockstorestorage.c @@ -776,6 +776,25 @@ static int BlockStoreStorageAPI_OpenWriteFile( return ENOTSUP; } +static int BlockStoreStorageAPI_OpenAppendFile( + struct Longtail_StorageAPI* storage_api, + const char* path, + Longtail_StorageAPI_HOpenFile* out_open_file) +{ + MAKE_LOG_CONTEXT_FIELDS(ctx) + LONGTAIL_LOGFIELD(storage_api, "%p"), + LONGTAIL_LOGFIELD(path, "%s"), + LONGTAIL_LOGFIELD(out_open_file, "%p") + MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_OFF) + + LONGTAIL_VALIDATE_INPUT(ctx, storage_api != 0, return 0) + LONGTAIL_VALIDATE_INPUT(ctx, path != 0, return 0) + LONGTAIL_VALIDATE_INPUT(ctx, out_open_file != 0, return 0) + + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Unsupported, failed with %d", ENOTSUP) + return ENOTSUP; +} + static int BlockStoreStorageAPI_Write( struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HOpenFile f, @@ -1407,7 +1426,8 @@ static int BlockStoreStorageAPI_Init( BlockStoreStorageAPI_UnlockFile, BlockStoreStorageAPI_GetParentPath, BlockStoreStorageAPI_MapFile, - BlockStoreStorageAPI_UnmapFile); + BlockStoreStorageAPI_UnmapFile, + BlockStoreStorageAPI_OpenAppendFile); struct BlockStoreStorageAPI* block_store_fs = (struct BlockStoreStorageAPI*)api; diff --git a/lib/concurrentchunkwrite/longtail_concurrentchunkwrite.c b/lib/concurrentchunkwrite/longtail_concurrentchunkwrite.c index b1a20396..a884b9c0 100644 --- a/lib/concurrentchunkwrite/longtail_concurrentchunkwrite.c +++ b/lib/concurrentchunkwrite/longtail_concurrentchunkwrite.c @@ -5,419 +5,277 @@ #include #include -#if defined(_MSC_VER) - -#define FORCE_INLINE __forceinline - -#include - -#define ROTL32(x,y) _rotl(x,y) -#define ROTL64(x,y) _rotl64(x,y) - -#define BIG_CONSTANT(x) (x) - -// Other compilers - -#else // defined(_MSC_VER) - -#define FORCE_INLINE inline __attribute__((always_inline)) - -inline uint32_t rotl32(uint32_t x, int8_t r) -{ - return (x << r) | (x >> (32 - r)); -} - -inline uint64_t rotl64(uint64_t x, int8_t r) -{ - return (x << r) | (x >> (64 - r)); -} - -#define ROTL32(x,y) rotl32(x,y) -#define ROTL64(x,y) rotl64(x,y) - -#define BIG_CONSTANT(x) (x##LLU) - -#endif // !defined(_MSC_VER) - -FORCE_INLINE uint64_t getblock64(const uint64_t* p, size_t i) -{ - return p[i]; -} - -FORCE_INLINE uint64_t fmix64(uint64_t k) -{ - k ^= k >> 33; - k *= BIG_CONSTANT(0xff51afd7ed558ccd); - k ^= k >> 33; - k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); - k ^= k >> 33; - - return k; -} - -uint64_t hashmurmur3_64(const void* key, size_t len) { - const uint8_t* data = (const uint8_t*)key; - const size_t nblocks = len / 16; - - uint64_t h1 = BIG_CONSTANT(0xf12c4da7f12c4da7); - uint64_t h2 = BIG_CONSTANT(0x4da7f12c4da7f12c); - - const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); - const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); - - //---------- - // body - - const uint64_t* blocks = (const uint64_t*)(data); - - for (size_t i = 0; i < nblocks; i++) - { - uint64_t k1 = getblock64(blocks, i * 2 + 0); - uint64_t k2 = getblock64(blocks, i * 2 + 1); - - k1 *= c1; k1 = ROTL64(k1, 31); k1 *= c2; h1 ^= k1; - - h1 = ROTL64(h1, 27); h1 += h2; h1 = h1 * 5 + 0x52dce729; - - k2 *= c2; k2 = ROTL64(k2, 33); k2 *= c1; h2 ^= k2; - - h2 = ROTL64(h2, 31); h2 += h1; h2 = h2 * 5 + 0x38495ab5; - } - - //---------- - // tail - - const uint8_t* tail = (const uint8_t*)(data + nblocks * 16); - - uint64_t k1 = 0; - uint64_t k2 = 0; - - switch (len & 15) - { - case 15: k2 ^= ((uint64_t)tail[14]) << 48; - case 14: k2 ^= ((uint64_t)tail[13]) << 40; - case 13: k2 ^= ((uint64_t)tail[12]) << 32; - case 12: k2 ^= ((uint64_t)tail[11]) << 24; - case 11: k2 ^= ((uint64_t)tail[10]) << 16; - case 10: k2 ^= ((uint64_t)tail[9]) << 8; - case 9: k2 ^= ((uint64_t)tail[8]) << 0; - k2 *= c2; k2 = ROTL64(k2, 33); k2 *= c1; h2 ^= k2; - - case 8: k1 ^= ((uint64_t)tail[7]) << 56; - case 7: k1 ^= ((uint64_t)tail[6]) << 48; - case 6: k1 ^= ((uint64_t)tail[5]) << 40; - case 5: k1 ^= ((uint64_t)tail[4]) << 32; - case 4: k1 ^= ((uint64_t)tail[3]) << 24; - case 3: k1 ^= ((uint64_t)tail[2]) << 16; - case 2: k1 ^= ((uint64_t)tail[1]) << 8; - case 1: k1 ^= ((uint64_t)tail[0]) << 0; - k1 *= c1; k1 = ROTL64(k1, 31); k1 *= c2; h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; h2 ^= len; - - h1 += h2; - h2 += h1; - - h1 = fmix64(h1); - h2 = fmix64(h2); - - h1 += h2; - h2 += h1; - - return h2; -} - -static uint64_t ConcurrentChunkWriteAPI_GetPathHash(const char* path) -{ - uint32_t pathlen = (uint32_t)strlen(path); - return hashmurmur3_64((const void*)path, pathlen); -} - struct OpenFileEntry { - uint32_t m_TotalWriteCount; - TLongtail_Atomic64 m_PendingWriteCount; Longtail_StorageAPI_HOpenFile m_FileHandle; -}; - -struct PathLookup -{ - uint64_t key; // path_hash - ptrdiff_t value; // index into ConcurrentChunkWriteAPI::m_OpenFileEntries -}; - -struct HandleLookup -{ - Longtail_StorageAPI_HOpenFile key; // open_file_handle - ptrdiff_t value; // index into ConcurrentChunkWriteAPI::m_OpenFileEntries + char* m_FullPath; + TLongtail_Atomic32 m_ActiveOpenCount; + HLongtail_SpinLock m_SpinLock; + uint32_t m_OpenCount; + TLongtail_Atomic64 m_BytesLeftToWrite; }; struct ConcurrentChunkWriteAPI { struct Longtail_ConcurrentChunkWriteAPI m_ConcurrentChunkWriteAPI; struct Longtail_StorageAPI* m_StorageAPI; - HLongtail_RWLock m_RWLock; - struct PathLookup* m_PathHashToOpenFile; - struct HandleLookup* m_FileHandleToOpenFile; - struct OpenFileEntry* m_OpenFileEntries; + const struct Longtail_VersionIndex* m_VersionIndex; + struct OpenFileEntry** m_AssetEntries; char* m_BasePath; }; static int ConcurrentChunkWriteAPI_Open( - struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, - const char* path, - uint32_t chunk_write_count, - Longtail_ConcurrentChunkWriteAPI_HOpenFile* out_open_file) + struct Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api, + uint32_t asset_index) { #if defined(LONGTAIL_ASSERTS) MAKE_LOG_CONTEXT_FIELDS(ctx) - LONGTAIL_LOGFIELD(concurrent_file_write_api, "%p"), - LONGTAIL_LOGFIELD(path, "%s"), - LONGTAIL_LOGFIELD(chunk_write_count, "%u"), - LONGTAIL_LOGFIELD(out_open_file, "%p") + LONGTAIL_LOGFIELD(concurrent_chunk_write_api, "%p"), + LONGTAIL_LOGFIELD(asset_index, "%u") MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) #else struct Longtail_LogContextFmt_Private* ctx = 0; #endif // defined(LONGTAIL_ASSERTS) - LONGTAIL_VALIDATE_INPUT(ctx, concurrent_file_write_api != 0, return EINVAL); - LONGTAIL_VALIDATE_INPUT(ctx, path != 0, return EINVAL); - LONGTAIL_VALIDATE_INPUT(ctx, out_open_file != 0, return EINVAL); + LONGTAIL_VALIDATE_INPUT(ctx, concurrent_chunk_write_api != 0, return EINVAL); - struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_file_write_api; - - uint64_t path_hash = ConcurrentChunkWriteAPI_GetPathHash(path); + struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_chunk_write_api; + LONGTAIL_VALIDATE_INPUT(ctx, asset_index < *api->m_VersionIndex->m_AssetCount, return EINVAL); + struct OpenFileEntry* open_file_entry = api->m_AssetEntries[asset_index]; + if (open_file_entry->m_SpinLock == 0) { - Longtail_LockRWLockRead(api->m_RWLock); - ptrdiff_t tmp; - intptr_t i = api->m_PathHashToOpenFile ? hmgeti_ts(api->m_PathHashToOpenFile, path_hash, tmp) : -1; - if (i != -1) + LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_FileHandle == 0, return EINVAL); + int err = EnsureParentPathExists(api->m_StorageAPI, open_file_entry->m_FullPath); + if (err != 0) + { + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "EnsureParentPathExists() failed with %d", err) + return err; + } + err = api->m_StorageAPI->OpenWriteFile(api->m_StorageAPI, open_file_entry->m_FullPath, 0, &open_file_entry->m_FileHandle); + if (err) { - intptr_t open_file_index = api->m_PathHashToOpenFile[i].value; - const struct OpenFileEntry* open_file_entry = &api->m_OpenFileEntries[open_file_index]; - LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_TotalWriteCount == chunk_write_count, Longtail_UnlockRWLockRead(api->m_RWLock); return EINVAL); - LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_PendingWriteCount > 0, Longtail_UnlockRWLockRead(api->m_RWLock); return EINVAL); - LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_FileHandle != 0, Longtail_UnlockRWLockRead(api->m_RWLock); return EINVAL); - *out_open_file = (Longtail_ConcurrentChunkWriteAPI_HOpenFile)open_file_entry->m_FileHandle; - Longtail_UnlockRWLockRead(api->m_RWLock); - return 0; + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "OpenWriteFile() failed with %d", err) + return err; } - Longtail_UnlockRWLockRead(api->m_RWLock); + open_file_entry->m_OpenCount++; + Longtail_AtomicAdd32(&open_file_entry->m_ActiveOpenCount, 1); + return 0; } - char* full_asset_path = api->m_StorageAPI->ConcatPath(api->m_StorageAPI, api->m_BasePath, path); - if (full_asset_path == 0) + Longtail_LockSpinLock(open_file_entry->m_SpinLock); + if (open_file_entry->m_FileHandle != 0) { - LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "ConcatPath() failed with %d", ENOMEM) - return ENOMEM; + Longtail_AtomicAdd32(&open_file_entry->m_ActiveOpenCount, 1); + Longtail_UnlockSpinLock(open_file_entry->m_SpinLock); + return 0; + } + if (open_file_entry->m_OpenCount > 0) + { + int err = api->m_StorageAPI->OpenAppendFile(api->m_StorageAPI, open_file_entry->m_FullPath, &open_file_entry->m_FileHandle); + if (err != 0) + { + Longtail_UnlockSpinLock(open_file_entry->m_SpinLock); + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "OpenAppendFile() failed with %d", err) + return err; + } + + open_file_entry->m_OpenCount++; + Longtail_AtomicAdd32(&open_file_entry->m_ActiveOpenCount, 1); + Longtail_UnlockSpinLock(open_file_entry->m_SpinLock); + return 0; } + Longtail_UnlockSpinLock(open_file_entry->m_SpinLock); - int err = EnsureParentPathExists(api->m_StorageAPI, full_asset_path); + int err = EnsureParentPathExists(api->m_StorageAPI, open_file_entry->m_FullPath); if (err != 0) { - Longtail_Free(full_asset_path); LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "EnsureParentPathExists() failed with %d", err) return err; } + Longtail_LockSpinLock(open_file_entry->m_SpinLock); + if (open_file_entry->m_FileHandle != 0) { - Longtail_LockRWLockWrite(api->m_RWLock); - ptrdiff_t tmp; - intptr_t i = api->m_PathHashToOpenFile ? hmgeti_ts(api->m_PathHashToOpenFile, path_hash, tmp) : -1; - if (i != -1) - { - intptr_t open_file_index = api->m_PathHashToOpenFile[i].value; - const struct OpenFileEntry* open_file_entry = &api->m_OpenFileEntries[open_file_index]; - LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_TotalWriteCount == chunk_write_count, Longtail_UnlockRWLockWrite(api->m_RWLock); Longtail_Free(full_asset_path); return EINVAL); - LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_PendingWriteCount > 0, Longtail_UnlockRWLockWrite(api->m_RWLock); Longtail_Free(full_asset_path); return EINVAL); - LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_FileHandle != 0, Longtail_UnlockRWLockWrite(api->m_RWLock); return EINVAL); - *out_open_file = (Longtail_ConcurrentChunkWriteAPI_HOpenFile)open_file_entry->m_FileHandle; - Longtail_UnlockRWLockWrite(api->m_RWLock); - Longtail_Free(full_asset_path); - return 0; - } + Longtail_AtomicAdd32(&open_file_entry->m_ActiveOpenCount, 1); + Longtail_UnlockSpinLock(open_file_entry->m_SpinLock); + return 0; + } - Longtail_StorageAPI_HOpenFile r; - err = api->m_StorageAPI->OpenWriteFile(api->m_StorageAPI, full_asset_path, 0, &r); - if (err != 0) - { - Longtail_UnlockRWLockWrite(api->m_RWLock); - Longtail_Free(full_asset_path); - LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "OpenWriteFile() failed with %d", err) - return err; - } - if (chunk_write_count == 0) - { - Longtail_UnlockRWLockWrite(api->m_RWLock); - // Empty file, close immediately - api->m_StorageAPI->CloseFile(api->m_StorageAPI, r); - Longtail_Free(full_asset_path); - *out_open_file = 0; - return 0; - } - struct OpenFileEntry entry; - entry.m_FileHandle = r; - entry.m_TotalWriteCount = chunk_write_count; - entry.m_PendingWriteCount = (int64_t)chunk_write_count; - ptrdiff_t entry_index = arrlen(api->m_OpenFileEntries); - arrput(api->m_OpenFileEntries, entry); - hmput(api->m_PathHashToOpenFile, path_hash, entry_index); - hmput(api->m_FileHandleToOpenFile, r, entry_index); - *out_open_file = (Longtail_ConcurrentChunkWriteAPI_HOpenFile)r; - Longtail_UnlockRWLockWrite(api->m_RWLock); + if (open_file_entry->m_OpenCount == 0) + { + err = api->m_StorageAPI->OpenWriteFile(api->m_StorageAPI, open_file_entry->m_FullPath, 0, &open_file_entry->m_FileHandle); } - Longtail_Free(full_asset_path); + else + { + err = api->m_StorageAPI->OpenAppendFile(api->m_StorageAPI, open_file_entry->m_FullPath, &open_file_entry->m_FileHandle); + } + if (err) + { + Longtail_UnlockSpinLock(open_file_entry->m_SpinLock); + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "OpenWriteFile/OpenAppendFile() failed with %d", err) + return err; + } + open_file_entry->m_OpenCount++; + Longtail_AtomicAdd32(&open_file_entry->m_ActiveOpenCount, 1); + Longtail_UnlockSpinLock(open_file_entry->m_SpinLock); + return 0; } static int ConcurrentChunkWriteAPI_Write( - struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, - Longtail_ConcurrentChunkWriteAPI_HOpenFile in_open_file, + struct Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api, + uint32_t asset_index, uint64_t offset, uint32_t size, - uint32_t chunk_count, - const void* input, - uint32_t* out_chunks_remaining) + const void* input) { #if defined(LONGTAIL_ASSERTS) MAKE_LOG_CONTEXT_FIELDS(ctx) - LONGTAIL_LOGFIELD(concurrent_file_write_api, "%p"), - LONGTAIL_LOGFIELD(in_open_file, "%p"), + LONGTAIL_LOGFIELD(concurrent_chunk_write_api, "%p"), + LONGTAIL_LOGFIELD(asset_index, "%u"), LONGTAIL_LOGFIELD(offset, "%" PRIu64), LONGTAIL_LOGFIELD(size, "%u"), - LONGTAIL_LOGFIELD(chunk_count, "%u"), LONGTAIL_LOGFIELD(input, "%p"), - LONGTAIL_LOGFIELD(out_chunks_remaining, "%p") MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) #else struct Longtail_LogContextFmt_Private* ctx = 0; #endif // defined(LONGTAIL_ASSERTS) - LONGTAIL_VALIDATE_INPUT(ctx, concurrent_file_write_api != 0, return EINVAL); - LONGTAIL_VALIDATE_INPUT(ctx, (uintptr_t)in_open_file != 0, return EINVAL); + LONGTAIL_VALIDATE_INPUT(ctx, concurrent_chunk_write_api != 0, return EINVAL); LONGTAIL_VALIDATE_INPUT(ctx, size != 0, return EINVAL); LONGTAIL_VALIDATE_INPUT(ctx, input != 0, return EINVAL); - struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_file_write_api; - Longtail_StorageAPI_HOpenFile file_handle = (Longtail_StorageAPI_HOpenFile)in_open_file; + struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_chunk_write_api; + LONGTAIL_VALIDATE_INPUT(ctx, asset_index < *api->m_VersionIndex->m_AssetCount, return EINVAL); + + struct OpenFileEntry* open_file_entry = api->m_AssetEntries[asset_index]; + LONGTAIL_FATAL_ASSERT(ctx, open_file_entry != 0, return EINVAL) + LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_ActiveOpenCount > 0, return EINVAL) + LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_FileHandle != 0, return EINVAL) + LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_FullPath != 0, return EINVAL) - int err = api->m_StorageAPI->Write(api->m_StorageAPI, file_handle, offset, size, input); + Longtail_AtomicAdd64(&open_file_entry->m_BytesLeftToWrite, -(int64_t)size); + int err = api->m_StorageAPI->Write(api->m_StorageAPI, open_file_entry->m_FileHandle, offset, size, input); if (err) { - { - Longtail_LockRWLockWrite(api->m_RWLock); - ptrdiff_t open_file_index = 0; - struct OpenFileEntry* open_file_entry = &api->m_OpenFileEntries[open_file_index]; - open_file_entry->m_FileHandle = 0; - hmdel(api->m_FileHandleToOpenFile, file_handle); - Longtail_UnlockRWLockWrite(api->m_RWLock); - } - api->m_StorageAPI->CloseFile(api->m_StorageAPI, file_handle); return err; } + return 0; +} + +static void ConcurrentChunkWriteAPI_Close( + struct Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api, + uint32_t asset_index) +{ +#if defined(LONGTAIL_ASSERTS) + MAKE_LOG_CONTEXT_FIELDS(ctx) + LONGTAIL_LOGFIELD(concurrent_chunk_write_api, "%p"), + MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) +#else + struct Longtail_LogContextFmt_Private* ctx = 0; +#endif // defined(LONGTAIL_ASSERTS) - ptrdiff_t open_file_index = 0; - int close_on_write = 0; + LONGTAIL_VALIDATE_INPUT(ctx, concurrent_chunk_write_api != 0, return); + + struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_chunk_write_api; + LONGTAIL_VALIDATE_INPUT(ctx, asset_index < *api->m_VersionIndex->m_AssetCount, return ); + struct OpenFileEntry* open_file_entry = api->m_AssetEntries[asset_index]; + + int32_t OpenCount = Longtail_AtomicAdd32(&open_file_entry->m_ActiveOpenCount, -1); + if (open_file_entry->m_SpinLock == 0) { - Longtail_LockRWLockRead(api->m_RWLock); - ptrdiff_t tmp; - intptr_t i = api->m_PathHashToOpenFile ? hmgeti_ts(api->m_FileHandleToOpenFile, file_handle, tmp) : -1; - if (i == -1) - { - Longtail_UnlockRWLockRead(api->m_RWLock); - LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "ConcurrentChunkWriteAPI_Write() file not open, error %d", EINVAL) - return EINVAL; - } - open_file_index = api->m_FileHandleToOpenFile[i].value; - struct OpenFileEntry* open_file_entry = &api->m_OpenFileEntries[open_file_index]; - LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_PendingWriteCount >= (int64_t)chunk_count, Longtail_UnlockRWLockRead(api->m_RWLock); return EINVAL); - LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_FileHandle == file_handle, Longtail_UnlockRWLockRead(api->m_RWLock); return EINVAL); - int64_t pending_count = Longtail_AtomicAdd64(&open_file_entry->m_PendingWriteCount, -(int64_t)chunk_count); - Longtail_UnlockRWLockRead(api->m_RWLock); - close_on_write = (pending_count == 0); - *out_chunks_remaining = (uint32_t)pending_count; + LONGTAIL_FATAL_ASSERT(ctx, OpenCount == 0, return) + LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_FileHandle != 0, return) + api->m_StorageAPI->CloseFile(api->m_StorageAPI, open_file_entry->m_FileHandle); + open_file_entry->m_FileHandle = 0; + return; } - - if (close_on_write) + if (OpenCount == 0) { + if (open_file_entry->m_BytesLeftToWrite == 0) { - Longtail_LockRWLockWrite(api->m_RWLock); - struct OpenFileEntry* open_file_entry = &api->m_OpenFileEntries[open_file_index]; - LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_PendingWriteCount == 0, Longtail_UnlockRWLockWrite(api->m_RWLock); return EINVAL); - LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_FileHandle == file_handle, Longtail_UnlockRWLockWrite(api->m_RWLock); return EINVAL); - open_file_entry->m_FileHandle = 0; - hmdel(api->m_FileHandleToOpenFile, file_handle); - Longtail_UnlockRWLockWrite(api->m_RWLock); + Longtail_LockSpinLock(open_file_entry->m_SpinLock); + Longtail_StorageAPI_HOpenFile file_handle = open_file_entry->m_FileHandle; + if (file_handle) + { + api->m_StorageAPI->CloseFile(api->m_StorageAPI, file_handle); + open_file_entry->m_FileHandle = 0; + } + Longtail_UnlockSpinLock(open_file_entry->m_SpinLock); } - api->m_StorageAPI->CloseFile(api->m_StorageAPI, file_handle); } - return err; } static int ConcurrentChunkWriteAPI_Flush( - struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api) + struct Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api) { #if defined(LONGTAIL_ASSERTS) MAKE_LOG_CONTEXT_FIELDS(ctx) - LONGTAIL_LOGFIELD(concurrent_file_write_api, "%p"), + LONGTAIL_LOGFIELD(concurrent_chunk_write_api, "%p"), MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) #else struct Longtail_LogContextFmt_Private* ctx = 0; #endif // defined(LONGTAIL_ASSERTS) - LONGTAIL_VALIDATE_INPUT(ctx, concurrent_file_write_api != 0, return EINVAL); - struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_file_write_api; - Longtail_LockRWLockWrite(api->m_RWLock); - ptrdiff_t entry_count = arrlen(api->m_OpenFileEntries); - for (ptrdiff_t i = 0; i < entry_count; ++i) + LONGTAIL_VALIDATE_INPUT(ctx, concurrent_chunk_write_api != 0, return EINVAL); + struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_chunk_write_api; + + for (ptrdiff_t i = 0; i < *api->m_VersionIndex->m_AssetCount; ++i) { - Longtail_StorageAPI_HOpenFile file_handle = api->m_OpenFileEntries[i].m_FileHandle; - if (file_handle) + struct OpenFileEntry* open_file_entry = api->m_AssetEntries[i]; + if (open_file_entry == 0) + { + continue; + } + if (open_file_entry->m_SpinLock == 0) + { + continue; + } + if (open_file_entry->m_ActiveOpenCount > 0) + { + continue; + } + Longtail_LockSpinLock(open_file_entry->m_SpinLock); + if (open_file_entry->m_ActiveOpenCount == 0) { - api->m_StorageAPI->CloseFile(api->m_StorageAPI, file_handle); + Longtail_StorageAPI_HOpenFile file_handle = open_file_entry->m_FileHandle; + if (file_handle) + { + api->m_StorageAPI->CloseFile(api->m_StorageAPI, file_handle); + open_file_entry->m_FileHandle = 0; + } } + Longtail_UnlockSpinLock(open_file_entry->m_SpinLock); } - hmfree(api->m_PathHashToOpenFile); - hmfree(api->m_FileHandleToOpenFile); - arrfree(api->m_OpenFileEntries); - Longtail_UnlockRWLockWrite(api->m_RWLock); return 0; } -static int ConcurrentChunkWriteAPI_CreateDir(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, const char* path) +static int ConcurrentChunkWriteAPI_CreateDir(struct Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api, uint32_t asset_index) { #if defined(LONGTAIL_ASSERTS) MAKE_LOG_CONTEXT_FIELDS(ctx) - LONGTAIL_LOGFIELD(concurrent_file_write_api, "%p"), - LONGTAIL_LOGFIELD(path, "%s") + LONGTAIL_LOGFIELD(concurrent_chunk_write_api, "%p"), + LONGTAIL_LOGFIELD(asset_index, "%u") MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) #else struct Longtail_LogContextFmt_Private* ctx = 0; #endif // defined(LONGTAIL_ASSERTS) - LONGTAIL_VALIDATE_INPUT(ctx, concurrent_file_write_api != 0, return EINVAL); - LONGTAIL_VALIDATE_INPUT(ctx, path != 0, return EINVAL); + LONGTAIL_VALIDATE_INPUT(ctx, concurrent_chunk_write_api != 0, return EINVAL); - struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_file_write_api; + struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_chunk_write_api; + LONGTAIL_VALIDATE_INPUT(ctx, asset_index < *api->m_VersionIndex->m_AssetCount, return EINVAL); - char* full_asset_path = api->m_StorageAPI->ConcatPath(api->m_StorageAPI, api->m_BasePath, path); + const char* asset_path = &api->m_VersionIndex->m_NameData[api->m_VersionIndex->m_NameOffsets[asset_index]]; + char* full_asset_path = api->m_StorageAPI->ConcatPath(api->m_StorageAPI, api->m_BasePath, asset_path); if (full_asset_path == 0) { - LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "ConcatPath() failed with %d", ENOMEM) - return ENOMEM; + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "storageAPI->ConcatPath() failed with %d", ENOMEM) + return ENOMEM; } full_asset_path[strlen(full_asset_path) - 1] = '\0'; - if (api->m_StorageAPI->IsDir(api->m_StorageAPI, full_asset_path)) - { - Longtail_Free(full_asset_path); - return 0; - } int err = EnsureParentPathExists(api->m_StorageAPI, full_asset_path); if (err) { @@ -435,34 +293,145 @@ static int ConcurrentChunkWriteAPI_CreateDir(struct Longtail_ConcurrentChunkWrit return err; } -static void ConcurrentChunkWriteAPI_Dispose(struct Longtail_API* concurrent_file_write_api) +static void DisposeFileEntries(struct ConcurrentChunkWriteAPI* api) +{ + MAKE_LOG_CONTEXT_FIELDS(ctx) + LONGTAIL_LOGFIELD(api, "%p") + MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) + + uint32_t asset_count = *api->m_VersionIndex->m_AssetCount; + for (uint32_t a = 0; a < asset_count; ++a) + { + struct OpenFileEntry* open_file_entry = api->m_AssetEntries[a]; + if (open_file_entry == 0) + { + continue; + } + LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_ActiveOpenCount == 0, return) + LONGTAIL_FATAL_ASSERT(ctx, open_file_entry->m_FileHandle == 0, return) + if (open_file_entry->m_SpinLock) + { + Longtail_DeleteSpinLock(open_file_entry->m_SpinLock); + } + Longtail_Free((void*)open_file_entry); + } +} + +static void ConcurrentChunkWriteAPI_Dispose(struct Longtail_API* concurrent_chunk_write_api) { MAKE_LOG_CONTEXT_FIELDS(ctx) - LONGTAIL_LOGFIELD(concurrent_file_write_api, "%p") - MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) + LONGTAIL_LOGFIELD(concurrent_chunk_write_api, "%p") + MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) - LONGTAIL_FATAL_ASSERT(ctx, concurrent_file_write_api != 0, return); - struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_file_write_api; + LONGTAIL_FATAL_ASSERT(ctx, concurrent_chunk_write_api != 0, return); + struct ConcurrentChunkWriteAPI* api = (struct ConcurrentChunkWriteAPI*)concurrent_chunk_write_api; +// api->m_MaxOpenFileEntryCount = 0; int err = ConcurrentChunkWriteAPI_Flush(&api->m_ConcurrentChunkWriteAPI); if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "ConcurrentChunkWriteAPI_Flush() failed with %d", err) } + + DisposeFileEntries(api); + Longtail_Free(api->m_BasePath); - Longtail_DeleteRWLock(api->m_RWLock); - Longtail_Free(concurrent_file_write_api); + Longtail_Free(concurrent_chunk_write_api); +} + +static int AllocateFileEntry( + struct Longtail_StorageAPI* storageAPI, + struct Longtail_VersionIndex* version_index, + const char* base_path, + uint32_t asset_index, + struct OpenFileEntry** out_file_entry) +{ + MAKE_LOG_CONTEXT_FIELDS(ctx) + LONGTAIL_LOGFIELD(storageAPI, "%p"), + LONGTAIL_LOGFIELD(version_index, "%p"), + LONGTAIL_LOGFIELD(base_path, "%s"), + LONGTAIL_LOGFIELD(asset_index, "%u") + MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) + + LONGTAIL_VALIDATE_INPUT(ctx, storageAPI != 0, return 0); + LONGTAIL_VALIDATE_INPUT(ctx, version_index != 0, return 0); + LONGTAIL_VALIDATE_INPUT(ctx, base_path != 0, return 0); + LONGTAIL_VALIDATE_INPUT(ctx, asset_index < *version_index->m_AssetCount, return 0); + + const char* asset_path = &version_index->m_NameData[version_index->m_NameOffsets[asset_index]]; + int is_dir_path = asset_path[0] && asset_path[strlen(asset_path) - 1] == '/'; + if (is_dir_path) + { + *out_file_entry = 0; + return 0; + } + char* full_asset_path = storageAPI->ConcatPath(storageAPI, base_path, asset_path); + if (full_asset_path == 0) + { + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "storageAPI->ConcatPath() failed with %d", ENOMEM) + return ENOMEM; + } + if (is_dir_path) + { + full_asset_path[strlen(full_asset_path) - 1] = '\0'; + } + + // If we had access to the store index we could figure out if an asset spans multiple blocks + // more exactly, but that would probably add more execution time at this initializaton - not sure if + // it is worth it. + int has_spin_lock = version_index->m_AssetChunkCounts[asset_index] > 1; + + size_t open_file_entry_size = sizeof(struct OpenFileEntry) + (has_spin_lock ? Longtail_GetSpinLockSize() : 0) + strlen(full_asset_path) + 1; + void* new_open_file_entry_mem = Longtail_Alloc("AllocateFileEntry", open_file_entry_size); + if (new_open_file_entry_mem == 0) + { + Longtail_Free(full_asset_path); + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "Longtail_Alloc() failed with %d", ENOMEM) + return ENOMEM; + } + char* p = (char*)new_open_file_entry_mem; + struct OpenFileEntry* new_open_file_entry = (struct OpenFileEntry*)p; + p += sizeof(struct OpenFileEntry); + if (has_spin_lock) + { + int err = Longtail_CreateSpinLock(p, &new_open_file_entry->m_SpinLock); + if (err) + { + Longtail_Free((void*)new_open_file_entry); + Longtail_Free(full_asset_path); + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "Longtail_CreateSpinLock() failed with %d", err) + return err; + } + p += Longtail_GetSpinLockSize(); + } + else + { + new_open_file_entry->m_SpinLock = 0; + } + new_open_file_entry->m_FileHandle = 0; + new_open_file_entry->m_ActiveOpenCount = 0; + new_open_file_entry->m_FullPath = p; + new_open_file_entry->m_OpenCount = 0; + new_open_file_entry->m_BytesLeftToWrite = version_index->m_AssetSizes[asset_index]; + strcpy(new_open_file_entry->m_FullPath, full_asset_path); + + Longtail_Free(full_asset_path); + full_asset_path = 0; + *out_file_entry = new_open_file_entry; + return 0; } static int ConcurrentChunkWriteAPI_Init( void* mem, struct Longtail_StorageAPI* storageAPI, + struct Longtail_VersionIndex* version_index, + struct Longtail_VersionDiff* version_diff, const char* base_path, - struct Longtail_ConcurrentChunkWriteAPI** out_storage_api) + struct Longtail_ConcurrentChunkWriteAPI** out_concurrent_chunk_write_api) { MAKE_LOG_CONTEXT_FIELDS(ctx) LONGTAIL_LOGFIELD(mem, "%p"), - LONGTAIL_LOGFIELD(out_storage_api, "%p"), + LONGTAIL_LOGFIELD(out_concurrent_chunk_write_api, "%p"), LONGTAIL_LOGFIELD(base_path, "%p") MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) @@ -474,27 +443,52 @@ static int ConcurrentChunkWriteAPI_Init( ConcurrentChunkWriteAPI_Dispose, ConcurrentChunkWriteAPI_CreateDir, ConcurrentChunkWriteAPI_Open, + ConcurrentChunkWriteAPI_Close, ConcurrentChunkWriteAPI_Write, ConcurrentChunkWriteAPI_Flush); - struct ConcurrentChunkWriteAPI* storage_api = (struct ConcurrentChunkWriteAPI*)api; - storage_api->m_StorageAPI = storageAPI; - Longtail_CreateRWLock(&storage_api[1], &storage_api->m_RWLock); - storage_api->m_PathHashToOpenFile = 0; - storage_api->m_FileHandleToOpenFile = 0; - storage_api->m_OpenFileEntries = 0; - storage_api->m_BasePath = Longtail_Strdup(base_path); - - *out_storage_api = api; + struct ConcurrentChunkWriteAPI* concurrent_chunk_write_api = (struct ConcurrentChunkWriteAPI*)api; + concurrent_chunk_write_api->m_StorageAPI = storageAPI; + concurrent_chunk_write_api->m_VersionIndex = version_index; + uint32_t asset_count = *version_index->m_AssetCount; + concurrent_chunk_write_api->m_AssetEntries = (struct OpenFileEntry**)&concurrent_chunk_write_api[1]; + memset(concurrent_chunk_write_api->m_AssetEntries, 0, sizeof(struct OpenFileEntry*) * asset_count); + for (uint32_t m = 0; m < *version_diff->m_ModifiedContentCount; m++) + { + uint32_t asset_index = version_diff->m_TargetContentModifiedAssetIndexes[m]; + int err = AllocateFileEntry(storageAPI, version_index, base_path, asset_index, &concurrent_chunk_write_api->m_AssetEntries[asset_index]); + if (err != 0) + { + DisposeFileEntries(concurrent_chunk_write_api); + return err; + } + } + for (uint32_t a = 0; a < *version_diff->m_TargetAddedCount; a++) + { + uint32_t asset_index = version_diff->m_TargetAddedAssetIndexes[a]; + int err = AllocateFileEntry(storageAPI, version_index, base_path, asset_index, &concurrent_chunk_write_api->m_AssetEntries[asset_index]); + if (err != 0) + { + DisposeFileEntries(concurrent_chunk_write_api); + return err; + } + } + concurrent_chunk_write_api->m_BasePath = Longtail_Strdup(base_path); +// concurrent_chunk_write_api->m_MaxOpenFileEntryCount = 0; + *out_concurrent_chunk_write_api = api; return 0; } -struct Longtail_ConcurrentChunkWriteAPI* Longtail_CreateConcurrentChunkWriteAPI(struct Longtail_StorageAPI* storageAPI, const char* base_path) +struct Longtail_ConcurrentChunkWriteAPI* Longtail_CreateConcurrentChunkWriteAPI( + struct Longtail_StorageAPI* storageAPI, + struct Longtail_VersionIndex* version_index, + struct Longtail_VersionDiff* version_diff, + const char* base_path) { MAKE_LOG_CONTEXT(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) LONGTAIL_VALIDATE_INPUT(ctx, storageAPI != 0, return 0); - size_t mem_size = sizeof(struct ConcurrentChunkWriteAPI) + Longtail_GetRWLockSize(); + size_t mem_size = sizeof(struct ConcurrentChunkWriteAPI) + sizeof(struct OpenFileEntry*) * (*version_index->m_AssetCount); void* mem = Longtail_Alloc("ConcurrentChunkWriteAPI", mem_size); if (!mem) { @@ -502,10 +496,11 @@ struct Longtail_ConcurrentChunkWriteAPI* Longtail_CreateConcurrentChunkWriteAPI( return 0; } struct Longtail_ConcurrentChunkWriteAPI* out_api; - int err = ConcurrentChunkWriteAPI_Init(mem, storageAPI, base_path, &out_api); + int err = ConcurrentChunkWriteAPI_Init(mem, storageAPI, version_index, version_diff, base_path, &out_api); if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "ConcurrentChunkWriteAPI_Init() failed with %d", err) + Longtail_Free(mem); return 0; } return out_api; diff --git a/lib/concurrentchunkwrite/longtail_concurrentchunkwrite.h b/lib/concurrentchunkwrite/longtail_concurrentchunkwrite.h index 84f308cb..7a77929a 100644 --- a/lib/concurrentchunkwrite/longtail_concurrentchunkwrite.h +++ b/lib/concurrentchunkwrite/longtail_concurrentchunkwrite.h @@ -6,7 +6,11 @@ extern "C" { #endif -LONGTAIL_EXPORT extern struct Longtail_ConcurrentChunkWriteAPI* Longtail_CreateConcurrentChunkWriteAPI(struct Longtail_StorageAPI* storageAPI, const char* base_path); +LONGTAIL_EXPORT extern struct Longtail_ConcurrentChunkWriteAPI* Longtail_CreateConcurrentChunkWriteAPI( + struct Longtail_StorageAPI* storageAPI, + struct Longtail_VersionIndex* version_index, + struct Longtail_VersionDiff* version_diff, + const char* base_path); #ifdef __cplusplus } diff --git a/lib/filestorage/longtail_filestorage.c b/lib/filestorage/longtail_filestorage.c index bcd441c0..b747d5e1 100644 --- a/lib/filestorage/longtail_filestorage.c +++ b/lib/filestorage/longtail_filestorage.c @@ -154,6 +154,35 @@ static int FSStorageAPI_OpenWriteFile( return 0; } +static int FSStorageAPI_OpenAppendFile( + struct Longtail_StorageAPI* storage_api, + const char* path, + Longtail_StorageAPI_HOpenFile* out_open_file) +{ +#if defined(LONGTAIL_ASSERTS) + MAKE_LOG_CONTEXT_FIELDS(ctx) + LONGTAIL_LOGFIELD(storage_api, "%p"), + LONGTAIL_LOGFIELD(path, "%s"), + LONGTAIL_LOGFIELD(out_open_file, "%p") + MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) +#else + struct Longtail_LogContextFmt_Private* ctx = 0; +#endif // defined(LONGTAIL_ASSERTS) + + LONGTAIL_VALIDATE_INPUT(ctx, storage_api != 0, return EINVAL); + LONGTAIL_VALIDATE_INPUT(ctx, path != 0, return EINVAL); + LONGTAIL_VALIDATE_INPUT(ctx, out_open_file != 0, return EINVAL); + HLongtail_OpenFile r; + int err = Longtail_OpenAppendFile(path, &r); + if (err) + { + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "Longtail_OpenWriteFile() failed with %d", err) + return err; + } + *out_open_file = (Longtail_StorageAPI_HOpenFile)r; + return 0; +} + static int FSStorageAPI_Write( struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HOpenFile f, @@ -667,7 +696,8 @@ static int FSStorageAPI_Init( FSStorageAPI_UnlockFile, FSStorageAPI_GetParentPath, FSStorageAPI_MapFile, - FSStorageAPI_UnmapFile); + FSStorageAPI_UnmapFile, + FSStorageAPI_OpenAppendFile); *out_storage_api = api; return 0; } diff --git a/lib/hpcdcchunker/longtail_hpcdcchunker.c b/lib/hpcdcchunker/longtail_hpcdcchunker.c index 0f695f46..4d8e9865 100644 --- a/lib/hpcdcchunker/longtail_hpcdcchunker.c +++ b/lib/hpcdcchunker/longtail_hpcdcchunker.c @@ -128,9 +128,7 @@ static uint32_t HPCDCDiscriminatorFromAvg(double avg) return (uint32_t)(avg / (-1.42888852e-7*avg + 1.33237515)); } -// TODO: Create a cache for chunkers? - -int Longtail_HPCDCCreateChunker( +static int Longtail_HPCDCCreateChunker( struct Longtail_HPCDCChunkerParams* params, struct Longtail_HPCDCChunker* optional_cached_chunker, struct Longtail_HPCDCChunker** out_chunker) diff --git a/lib/longtail_platform.c b/lib/longtail_platform.c index e2eb39e8..f5c25e20 100644 --- a/lib/longtail_platform.c +++ b/lib/longtail_platform.c @@ -379,48 +379,6 @@ void Longtail_UnlockSpinLock(HLongtail_SpinLock spin_lock) ReleaseSRWLockExclusive(&spin_lock->m_Lock); } -struct Longtail_RWLock -{ - SRWLOCK m_RWLock; -}; - -size_t Longtail_GetRWLockSize() -{ - return sizeof(struct Longtail_RWLock); -} - -int Longtail_CreateRWLock(void* mem, HLongtail_RWLock* out_rwlock) -{ - HLongtail_RWLock rwlock = (HLongtail_RWLock)mem; - InitializeSRWLock(&rwlock->m_RWLock); - *out_rwlock = rwlock; - return 0; -} - -void Longtail_DeleteRWLock(HLongtail_RWLock rwlock) -{ -} - -void Longtail_LockRWLockRead(HLongtail_RWLock rwlock) -{ - AcquireSRWLockShared(&rwlock->m_RWLock); -} - -void Longtail_LockRWLockWrite(HLongtail_RWLock rwlock) -{ - AcquireSRWLockExclusive(&rwlock->m_RWLock); -} - -void Longtail_UnlockRWLockRead(HLongtail_RWLock rwlock) -{ - ReleaseSRWLockShared(&rwlock->m_RWLock); -} - -void Longtail_UnlockRWLockWrite(HLongtail_RWLock rwlock) -{ - ReleaseSRWLockExclusive(&rwlock->m_RWLock); -} - static wchar_t* MakeWCharString(const char* s, wchar_t* buffer, size_t buffer_size) { struct Longtail_LogContextFmt_Private* ctx = 0; @@ -905,12 +863,12 @@ int Longtail_OpenReadFile(const char* path, HLongtail_OpenFile* out_read_file) return 0; } -DWORD NativeOpenWriteFileWithRetry(wchar_t* long_path, DWORD create_disposition, HANDLE* out_handle) +DWORD NativeOpenWriteFileWithRetry(wchar_t* long_path, DWORD desired_access, DWORD create_disposition, HANDLE* out_handle) { int retry_count = 10; while (1) { - HANDLE handle = CreateFileW(long_path, GENERIC_READ | GENERIC_WRITE, 0, 0, create_disposition, 0, 0); + HANDLE handle = CreateFileW(long_path, desired_access, 0, 0, create_disposition, 0, 0); if (handle == INVALID_HANDLE_VALUE) { DWORD error = GetLastError(); @@ -937,7 +895,7 @@ int Longtail_OpenWriteFile(const char* path, uint64_t initial_size, HLongtail_Op wchar_t long_path_buffer[512]; wchar_t* long_path = MakeLongPlatformPath(path, long_path_buffer, sizeof(long_path_buffer)); HANDLE handle; - DWORD error = NativeOpenWriteFileWithRetry(long_path, initial_size == 0 ? CREATE_ALWAYS : OPEN_ALWAYS, &handle); + DWORD error = NativeOpenWriteFileWithRetry(long_path, GENERIC_WRITE, initial_size == 0 ? CREATE_ALWAYS : OPEN_ALWAYS, &handle); if (long_path != long_path_buffer) { Longtail_Free(long_path); @@ -969,6 +927,25 @@ int Longtail_OpenWriteFile(const char* path, uint64_t initial_size, HLongtail_Op return 0; } +int Longtail_OpenAppendFile(const char* path, HLongtail_OpenFile* out_write_file) +{ + wchar_t long_path_buffer[512]; + wchar_t* long_path = MakeLongPlatformPath(path, long_path_buffer, sizeof(long_path_buffer)); + HANDLE handle; + DWORD error = NativeOpenWriteFileWithRetry(long_path, GENERIC_WRITE, OPEN_ALWAYS, &handle); + if (long_path != long_path_buffer) + { + Longtail_Free(long_path); + } + if (error != ERROR_SUCCESS) + { + return Win32ErrorToErrno(error); + } + + *out_write_file = (HLongtail_OpenFile)handle; + return 0; +} + int Longtail_SetFileSize(HLongtail_OpenFile handle, uint64_t length) { HANDLE h = (HANDLE)(handle); @@ -1862,101 +1839,6 @@ void Longtail_UnlockSpinLock(HLongtail_SpinLock spin_lock) #endif -struct Longtail_RWLock -{ - pthread_rwlock_t m_RWLock; -}; - -size_t Longtail_GetRWLockSize() -{ - return sizeof(struct Longtail_RWLock); -} - -int Longtail_CreateRWLock(void* mem, HLongtail_RWLock* out_rwlock) -{ - HLongtail_RWLock rwlock = (HLongtail_RWLock)mem; - int err = pthread_rwlock_init(&rwlock->m_RWLock, 0); - if (err) - { - return err; - } - *out_rwlock = rwlock; - return 0; -} - -void Longtail_DeleteRWLock(HLongtail_RWLock rwlock) -{ - pthread_rwlock_destroy(&rwlock->m_RWLock); -} - -void Longtail_LockRWLockRead(HLongtail_RWLock rwlock) -{ -#if defined(LONGTAIL_ASSERTS) - MAKE_LOG_CONTEXT_FIELDS(ctx) - LONGTAIL_LOGFIELD(rwlock, "%p"), - MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_OFF) -#else - struct Longtail_LogContextFmt_Private* ctx = 0; -#endif // defined(LONGTAIL_ASSERTS) - - int err = pthread_rwlock_rdlock(&rwlock->m_RWLock); - if (err) - { - LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to acquire read lock, reason: %d", err) - } -} - -void Longtail_LockRWLockWrite(HLongtail_RWLock rwlock) -{ -#if defined(LONGTAIL_ASSERTS) - MAKE_LOG_CONTEXT_FIELDS(ctx) - LONGTAIL_LOGFIELD(rwlock, "%p"), - MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_OFF) -#else - struct Longtail_LogContextFmt_Private* ctx = 0; -#endif // defined(LONGTAIL_ASSERTS) - - int err = pthread_rwlock_wrlock(&rwlock->m_RWLock); - if (err) - { - LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to acquire write lock, reason: %d", err) - } -} - -void Longtail_UnlockRWLockRead(HLongtail_RWLock rwlock) -{ -#if defined(LONGTAIL_ASSERTS) - MAKE_LOG_CONTEXT_FIELDS(ctx) - LONGTAIL_LOGFIELD(rwlock, "%p"), - MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_OFF) -#else - struct Longtail_LogContextFmt_Private* ctx = 0; -#endif // defined(LONGTAIL_ASSERTS) - - int err = pthread_rwlock_unlock(&rwlock->m_RWLock); - if (err) - { - LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to unlock read lock, reason: %d", err) - } -} - -void Longtail_UnlockRWLockWrite(HLongtail_RWLock rwlock) -{ -#if defined(LONGTAIL_ASSERTS) - MAKE_LOG_CONTEXT_FIELDS(ctx) - LONGTAIL_LOGFIELD(rwlock, "%p"), - MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_OFF) -#else - struct Longtail_LogContextFmt_Private* ctx = 0; -#endif // defined(LONGTAIL_ASSERTS) - - int err = pthread_rwlock_unlock(&rwlock->m_RWLock); - if (err) - { - LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to unlock write lock, reason: %d", err) - } -} - int Longtail_CreateDirectory(const char* path) { int err = mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); @@ -2259,7 +2141,7 @@ int Longtail_OpenWriteFile(const char* path, uint64_t initial_size, HLongtail_Op int e = errno; return e; } - if (initial_size > 0) + if (initial_size > 0) { int err = ftruncate64(fileno(f), (off64_t)initial_size); if (err != 0) @@ -2273,6 +2155,18 @@ int Longtail_OpenWriteFile(const char* path, uint64_t initial_size, HLongtail_Op return 0; } +int Longtail_OpenAppendFile(const char* path, HLongtail_OpenFile* out_write_file) +{ + FILE* f = fopen(path, "ab"); + if (!f) + { + int e = errno; + return e; + } + *out_write_file = (HLongtail_OpenFile)f; + return 0; +} + int Longtail_SetFileSize(HLongtail_OpenFile handle, uint64_t length) { FILE* f = (FILE*)handle; diff --git a/lib/longtail_platform.h b/lib/longtail_platform.h index e9848bb4..dae91f7a 100644 --- a/lib/longtail_platform.h +++ b/lib/longtail_platform.h @@ -44,15 +44,6 @@ void Longtail_DeleteSpinLock(HLongtail_SpinLock spin_lock); void Longtail_LockSpinLock(HLongtail_SpinLock spin_lock); void Longtail_UnlockSpinLock(HLongtail_SpinLock spin_lock); -typedef struct Longtail_RWLock* HLongtail_RWLock; -size_t Longtail_GetRWLockSize(); -int Longtail_CreateRWLock(void* mem, HLongtail_RWLock* out_rwlock); -void Longtail_DeleteRWLock(HLongtail_RWLock rwlock); -void Longtail_LockRWLockRead(HLongtail_RWLock rwlock); -void Longtail_LockRWLockWrite(HLongtail_RWLock rwlock); -void Longtail_UnlockRWLockRead(HLongtail_RWLock rwlock); -void Longtail_UnlockRWLockWrite(HLongtail_RWLock rwlock); - typedef struct Longtail_FSIterator_private* HLongtail_FSIterator; size_t Longtail_GetFSIteratorSize(); @@ -75,6 +66,7 @@ typedef struct Longtail_OpenFile_private* HLongtail_OpenFile; int Longtail_OpenReadFile(const char* path, HLongtail_OpenFile* out_read_file); int Longtail_OpenWriteFile(const char* path, uint64_t initial_size, HLongtail_OpenFile* out_write_file); +int Longtail_OpenAppendFile(const char* path, HLongtail_OpenFile* out_write_file); int Longtail_SetFileSize(HLongtail_OpenFile handle, uint64_t length); int Longtail_SetFilePermissions(const char* path, uint16_t permissions); int Longtail_GetFilePermissions(const char* path, uint16_t* out_permissions); diff --git a/lib/memstorage/longtail_memstorage.c b/lib/memstorage/longtail_memstorage.c index 07b7a4fe..ccbec20d 100644 --- a/lib/memstorage/longtail_memstorage.c +++ b/lib/memstorage/longtail_memstorage.c @@ -330,6 +330,76 @@ static int InMemStorageAPI_OpenWriteFile(struct Longtail_StorageAPI* storage_api return 0; } +static int InMemStorageAPI_OpenAppendFile(struct Longtail_StorageAPI* storage_api, const char* path, Longtail_StorageAPI_HOpenFile* out_open_file) +{ +#if defined(LONGTAIL_ASSERTS) + MAKE_LOG_CONTEXT_FIELDS(ctx) + LONGTAIL_LOGFIELD(storage_api, "%p"), + LONGTAIL_LOGFIELD(path, "%s"), + LONGTAIL_LOGFIELD(out_open_file, "%p") + MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) +#else + struct Longtail_LogContextFmt_Private* ctx = 0; +#endif // defined(LONGTAIL_ASSERTS) + + LONGTAIL_VALIDATE_INPUT(ctx, storage_api != 0, return EINVAL); + LONGTAIL_VALIDATE_INPUT(ctx, path != 0, return EINVAL); + LONGTAIL_VALIDATE_INPUT(ctx, out_open_file != 0, return EINVAL); + struct InMemStorageAPI* instance = (struct InMemStorageAPI*)storage_api; + uint32_t path_hash = InMemStorageAPI_GetPathHash(path); + uint32_t parent_path_hash = InMemStorageAPI_GetParentPathHash(path); + Longtail_LockSpinLock(instance->m_SpinLock); + if (parent_path_hash != 0 && hmgeti(instance->m_PathHashToContent, parent_path_hash) == -1) + { + Longtail_UnlockSpinLock(instance->m_SpinLock); + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "File not found, failed with %d", ENOENT) + return ENOENT; + } + struct PathEntry* path_entry = 0; + intptr_t it = hmgeti(instance->m_PathHashToContent, path_hash); + if (it != -1) + { + path_entry = &instance->m_PathEntries[instance->m_PathHashToContent[it].value]; + if ((path_entry->m_Permissions & (Longtail_StorageAPI_OtherWriteAccess | Longtail_StorageAPI_GroupWriteAccess | Longtail_StorageAPI_UserWriteAccess)) == 0) + { + Longtail_UnlockSpinLock(instance->m_SpinLock); + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "No permission to write, failed with %d", EACCES) + return EACCES; + } + if (path_entry->m_IsOpenWrite) + { + Longtail_UnlockSpinLock(instance->m_SpinLock); + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "File already open for write, failed with %d", EPERM) + return EPERM; + } + + if (path_entry->m_IsOpenRead) + { + Longtail_UnlockSpinLock(instance->m_SpinLock); + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_INFO, "File already open for read, failed with %d", EPERM) + return EPERM; + } + path_entry->m_IsOpenWrite = 1; + } + else + { + ptrdiff_t entry_index = arrlen(instance->m_PathEntries); + arrsetlen(instance->m_PathEntries, (size_t)(entry_index + 1)); + path_entry = &instance->m_PathEntries[entry_index]; + path_entry->m_ParentHash = parent_path_hash; + path_entry->m_FileName = Longtail_Strdup(InMemStorageAPI_GetFileNamePart(path)); + path_entry->m_Content = 0; + path_entry->m_Permissions = 0644; + path_entry->m_IsOpenRead = 0; + path_entry->m_IsOpenWrite = 1; + arrsetcap(path_entry->m_Content, 16); + hmput(instance->m_PathHashToContent, path_hash, (uint32_t)entry_index); + } + Longtail_UnlockSpinLock(instance->m_SpinLock); + *out_open_file = (Longtail_StorageAPI_HOpenFile)(uintptr_t)path_hash; + return 0; +} + static int InMemStorageAPI_Write(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HOpenFile f, uint64_t offset, uint64_t length, const void* input) { #if defined(LONGTAIL_ASSERTS) @@ -1204,7 +1274,8 @@ static int InMemStorageAPI_Init( InMemStorageAPI_UnlockFile, InMemStorageAPI_GetParentPath, InMemStorageAPI_MapFile, - InMemStorageAPI_UnmapFile); + InMemStorageAPI_UnmapFile, + InMemStorageAPI_OpenAppendFile); struct InMemStorageAPI* storage_api = (struct InMemStorageAPI*)api; diff --git a/src/longtail.c b/src/longtail.c index 4b900fb0..a1349374 100644 --- a/src/longtail.c +++ b/src/longtail.c @@ -14,6 +14,7 @@ #include #define LONGTAIL_VERSION(major, minor, patch) ((((uint32_t)major) << 24) | ((uint32_t)minor << 16) | ((uint32_t)patch)) +#define LONGTAIL_VERSION_INDEX_VERSION_0_0_1 LONGTAIL_VERSION(0,0,1) #define LONGTAIL_VERSION_INDEX_VERSION_0_0_2 LONGTAIL_VERSION(0,0,2) #define LONGTAIL_STORE_INDEX_VERSION_1_0_0 LONGTAIL_VERSION(1,0,0) #define LONGTAIL_ARCHIVE_VERSION_0_0_1 LONGTAIL_VERSION(0,0,1) @@ -264,7 +265,8 @@ struct Longtail_StorageAPI* Longtail_MakeStorageAPI( Longtail_Storage_UnlockFileFunc unlock_file_func, Longtail_Storage_GetParentPathFunc get_parent_path_func, Longtail_Storage_MapFileFunc map_file_func, - Longtail_Storage_UnmapFileFunc unmap_file_func) + Longtail_Storage_UnmapFileFunc unmap_file_func, + Longtail_Storage_OpenAppendFileFunc open_append_file_func) { MAKE_LOG_CONTEXT_FIELDS(ctx) LONGTAIL_LOGFIELD(mem, "%p"), @@ -293,7 +295,8 @@ struct Longtail_StorageAPI* Longtail_MakeStorageAPI( LONGTAIL_LOGFIELD(unlock_file_func, "%p"), LONGTAIL_LOGFIELD(get_parent_path_func, "%p"), LONGTAIL_LOGFIELD(map_file_func, "%p"), - LONGTAIL_LOGFIELD(unmap_file_func, "%p") + LONGTAIL_LOGFIELD(unmap_file_func, "%p"), + LONGTAIL_LOGFIELD(open_append_file_func, "%p") MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) LONGTAIL_VALIDATE_INPUT(ctx, mem != 0, return 0) @@ -324,6 +327,7 @@ struct Longtail_StorageAPI* Longtail_MakeStorageAPI( api->GetParentPath = get_parent_path_func; api->MapFile = map_file_func; api->UnMapFile = unmap_file_func; + api->OpenAppendFile = open_append_file_func; return api; } @@ -352,6 +356,7 @@ int Longtail_Storage_UnlockFile(struct Longtail_StorageAPI* storage_api, Longtai char* Longtail_Storage_GetParentPath(struct Longtail_StorageAPI* storage_api, const char* path) { return storage_api->GetParentPath(storage_api, path); } int Longtail_Storage_MapFile(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HOpenFile f, uint64_t offset, uint64_t length, Longtail_StorageAPI_HFileMap* out_file_map, const void** out_data_ptr) { return storage_api->MapFile(storage_api, f, offset, length, out_file_map, out_data_ptr); } void Longtail_Storage_UnmapFile(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HFileMap m) { storage_api->UnMapFile(storage_api, m); } +void Longtail_Storage_OpenAppendfile(struct Longtail_StorageAPI* storage_api, const char* path, Longtail_StorageAPI_HOpenFile* out_open_file) { storage_api->OpenAppendFile(storage_api, path, out_open_file); } ////////////// ConcurrentChunkWriteAPI @@ -365,6 +370,7 @@ LONGTAIL_EXPORT struct Longtail_ConcurrentChunkWriteAPI* Longtail_MakeConcurrent Longtail_DisposeFunc dispose_func, Longtail_ConcurrentChunkWrite_CreateDirFunc create_dir_func, Longtail_ConcurrentChunkWrite_OpenFunc open_func, + Longtail_ConcurrentChunkWrite_CloseFunc close_func, Longtail_ConcurrentChunkWrite_WriteFunc write_func, Longtail_ConcurrentChunkWrite_FlushFunc flush_func) { @@ -373,23 +379,26 @@ LONGTAIL_EXPORT struct Longtail_ConcurrentChunkWriteAPI* Longtail_MakeConcurrent LONGTAIL_LOGFIELD(dispose_func, "%p"), LONGTAIL_LOGFIELD(create_dir_func, "%p"), LONGTAIL_LOGFIELD(open_func, "%p"), + LONGTAIL_LOGFIELD(close_func, "%p"), LONGTAIL_LOGFIELD(write_func, "%p"), LONGTAIL_LOGFIELD(flush_func, "%p") - MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) + MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_DEBUG) LONGTAIL_VALIDATE_INPUT(ctx, mem != 0, return 0) struct Longtail_ConcurrentChunkWriteAPI* api = (struct Longtail_ConcurrentChunkWriteAPI*)mem; api->m_API.Dispose = dispose_func; api->CreateDir = create_dir_func; api->Open = open_func; + api->Close = close_func; api->Write = write_func; api->Flush = flush_func; return api; } -int Longtail_ConcurrentChunkWrite_CreateDir(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, const char* path) { return concurrent_file_write_api->CreateDir(concurrent_file_write_api, path); } -int Longtail_ConcurrentChunkWrite_Open(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, const char* path, uint32_t chunk_write_count, Longtail_ConcurrentChunkWriteAPI_HOpenFile* out_open_file) { return concurrent_file_write_api->Open(concurrent_file_write_api, path, chunk_write_count, out_open_file); } -int Longtail_ConcurrentChunkWrite_Write(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, Longtail_ConcurrentChunkWriteAPI_HOpenFile in_open_file, uint64_t offset, uint32_t size, uint32_t chunk_count, const void* input, uint32_t* out_chunks_remaining) { return concurrent_file_write_api->Write(concurrent_file_write_api, in_open_file, offset, size, chunk_count, input, out_chunks_remaining); } +int Longtail_ConcurrentChunkWrite_CreateDir(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index) { return concurrent_file_write_api->CreateDir(concurrent_file_write_api, asset_index); } +int Longtail_ConcurrentChunkWrite_Open(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index) { return concurrent_file_write_api->Open(concurrent_file_write_api, asset_index); } +void Longtail_ConcurrentChunkWrite_Close(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index) { concurrent_file_write_api->Close(concurrent_file_write_api, asset_index); } +int Longtail_ConcurrentChunkWrite_Write(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index, uint64_t offset, uint32_t size, const void* input) { return concurrent_file_write_api->Write(concurrent_file_write_api, asset_index, offset, size, input); } int Longtail_ConcurrentChunkWrite_Flush(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api) { return concurrent_file_write_api->Flush(concurrent_file_write_api); } ////////////// ProgressAPI @@ -738,16 +747,16 @@ static struct Longtail_Monitor Monitor_private = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, #define LONGTAIL_MONTITOR_BLOCK_LOADED(store_index, block_index, err) if (Monitor_private.BlockLoaded) {Monitor_private.BlockLoaded(store_index, block_index, err);} #define LONGTAIL_MONTITOR_BLOCK_COMPLETE(store_index, block_index, err) if (Monitor_private.BlockLoadComplete) {Monitor_private.BlockLoadComplete(store_index, block_index, err);} -#define LONGTAIL_MONITOR_ASSET_REMOVE(version_index, asset_index) if (Monitor_private.AssetRemove) {Monitor_private.AssetRemove(version_index, asset_index);} -#define LONGTAIL_MONTITOR_ASSET_OPEN(version_index, asset_index) if (Monitor_private.AssetOpen) {Monitor_private.AssetOpen(version_index, asset_index);} -#define LONGTAIL_MONTITOR_ASSET_WRITE(store_index, version_index, asset_index, write_offset, size, chunk_index, chunk_index_in_block, chunk_count_in_block, block_index, block_data_offset) if (Monitor_private.AssetWrite) {Monitor_private.AssetWrite(store_index, version_index, asset_index, write_offset, size, chunk_index, chunk_index_in_block, chunk_count_in_block, block_index, block_data_offset);} -#define LONGTAIL_MONTITOR_ASSET_COMPLETE(version_index, asset_index, err) if (Monitor_private.AssetComplete) {Monitor_private.AssetComplete(version_index, asset_index, err);}; +#define LONGTAIL_MONITOR_ASSET_REMOVE(version_index, asset_index, err) if (Monitor_private.AssetRemove) {Monitor_private.AssetRemove(version_index, asset_index, err);} +#define LONGTAIL_MONTITOR_ASSET_OPEN(version_index, asset_index, err) if (Monitor_private.AssetOpen) {Monitor_private.AssetOpen(version_index, asset_index, err);} +#define LONGTAIL_MONTITOR_ASSET_WRITE(store_index, version_index, asset_index, write_offset, size, chunk_index, chunk_index_in_block, chunk_count_in_block, block_index, block_data_offset, err) if (Monitor_private.AssetWrite) {Monitor_private.AssetWrite(store_index, version_index, asset_index, write_offset, size, chunk_index, chunk_index_in_block, chunk_count_in_block, block_index, block_data_offset, err);} +//#define LONGTAIL_MONTITOR_ASSET_COMPLETE(version_index, asset_index, err) if (Monitor_private.AssetComplete) {Monitor_private.AssetComplete(version_index, asset_index, err);}; -#define LONGTAIL_MONTITOR_CHUNK_READ(store_index, version_index, block_index, chunk_index, chunk_index_in_block) if (Monitor_private.ChunkRead) {Monitor_private.ChunkRead(store_index, version_index, block_index, chunk_index, chunk_index_in_block);} +#define LONGTAIL_MONTITOR_CHUNK_READ(store_index, version_index, block_index, chunk_index, chunk_index_in_block, err) if (Monitor_private.ChunkRead) {Monitor_private.ChunkRead(store_index, version_index, block_index, chunk_index, chunk_index_in_block, err);} #define LONGTAIL_MONTITOR_BLOCK_COMPOSE(store_index, block_index) if (Monitor_private.BlockCompose) {Monitor_private.BlockCompose(store_index, block_index);} -#define LONGTAIL_MONTITOR_ASSET_READ(store_index, version_index, asset_index, read_offset, size, chunk_hash, block_index, block_data_offset) if (Monitor_private.AssetRead) {Monitor_private.AssetRead(store_index, version_index, asset_index, read_offset, size, chunk_hash, block_index, block_data_offset);} -#define LONGTAIL_MONTITOR_ASSET_CLOSE(version_index, asset_index, err) if (Monitor_private.AssetClose) {Monitor_private.AssetClose(version_index, asset_index, err);} +#define LONGTAIL_MONTITOR_ASSET_READ(store_index, version_index, asset_index, read_offset, size, chunk_hash, block_index, block_data_offset, err) if (Monitor_private.AssetRead) {Monitor_private.AssetRead(store_index, version_index, asset_index, read_offset, size, chunk_hash, block_index, block_data_offset, err);} +#define LONGTAIL_MONTITOR_ASSET_CLOSE(version_index, asset_index) if (Monitor_private.AssetClose) {Monitor_private.AssetClose(version_index, asset_index);} #define LONGTAIL_MONITOR_BLOCK_SAVE(store_index, block_index, block_size) if (Monitor_private.BlockSave) {Monitor_private.BlockSave(store_index, block_index, block_size);} #define LONGTAIL_MONITOR_BLOCK_SAVED(store_index, block_index, err) if (Monitor_private.BlockSaved) {Monitor_private.BlockSaved(store_index, block_index, err);} @@ -1210,6 +1219,7 @@ struct Longtail_LookupTable* LongtailPrivate_LookupTable_Create(void* mem, uint3 MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_OFF) struct Longtail_LookupTable* lut = (struct Longtail_LookupTable*)mem; + memset(lut, 0xff, LongtailPrivate_LookupTable_GetSize(capacity)); uint32_t table_size = GetLookupTableSize(capacity); lut->m_BucketCount = table_size; lut->m_Capacity = capacity; @@ -1219,8 +1229,8 @@ struct Longtail_LookupTable* LongtailPrivate_LookupTable_Create(void* mem, uint3 lut->m_Values = (uint32_t*)&lut->m_Keys[capacity]; lut->m_NextIndex = &lut->m_Values[capacity]; - memset(lut->m_Buckets, 0xff, sizeof(uint32_t) * table_size); - memset(lut->m_NextIndex, 0xff, sizeof(uint32_t) * capacity); +// memset(lut->m_Buckets, 0xff, sizeof(uint32_t) * table_size); +// memset(lut->m_NextIndex, 0xff, sizeof(uint32_t) * capacity); if (optional_source_entries == 0) { @@ -1257,6 +1267,18 @@ static int IsDirPath(const char* path) return path[0] ? path[strlen(path) - 1] == '/' : 0; } +static int GetPathHashWithLength(struct Longtail_HashAPI* hash_api, const char* path, uint32_t pathlen, TLongtail_Hash* out_hash) +{ + uint64_t hash; + int err = hash_api->HashBuffer(hash_api, pathlen, (void*)path, &hash); + if (err) + { + return err; + } + *out_hash = (TLongtail_Hash)hash; + return 0; +} + int LongtailPrivate_GetPathHash(struct Longtail_HashAPI* hash_api, const char* path, TLongtail_Hash* out_hash) { #if defined(LONGTAIL_ASSERTS) @@ -1273,14 +1295,12 @@ int LongtailPrivate_GetPathHash(struct Longtail_HashAPI* hash_api, const char* p LONGTAIL_FATAL_ASSERT(ctx, path != 0, return EINVAL) LONGTAIL_FATAL_ASSERT(ctx, out_hash != 0, return EINVAL) uint32_t pathlen = (uint32_t)strlen(path); - uint64_t hash; - int err = hash_api->HashBuffer(hash_api, pathlen, (void*)path, &hash); + int err = GetPathHashWithLength(hash_api, path, pathlen, out_hash); if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "hash_api->HashBuffer() failed with %d", err) return err; } - *out_hash = (TLongtail_Hash)hash; return 0; } @@ -1962,12 +1982,12 @@ static int DynamicChunking(void* context, uint32_t job_id, int detected_error) return err; } } - - if (IsDirPath(hash_job->m_Path)) + if (hash_job->m_SizeRange == 0) { *hash_job->m_AssetChunkCount = 0; return 0; } + uint32_t chunk_count = 0; struct Longtail_StorageAPI* storage_api = hash_job->m_StorageAPI; @@ -4616,14 +4636,14 @@ static int WriteContentBlockJob(void* context, uint32_t job_id, int detected_err { source_storage_api->CloseFile(source_storage_api, file_handle); file_handle = 0; - LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, last_asset_index, 0); + LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, last_asset_index); } const char* asset_path = &job->m_VersionIndex->m_NameData[next_path_name_offset]; LONGTAIL_FATAL_ASSERT(ctx, !IsDirPath(asset_path), return EINVAL) - LONGTAIL_MONTITOR_ASSET_OPEN(job->m_VersionIndex, asset_index); char* full_path = source_storage_api->ConcatPath(source_storage_api, job->m_AssetsFolder, asset_path); int err = source_storage_api->OpenReadFile(source_storage_api, full_path, &file_handle); + LONGTAIL_MONTITOR_ASSET_OPEN(job->m_VersionIndex, asset_index, err); Longtail_Free(full_path); full_path = 0; if (err) @@ -4639,7 +4659,7 @@ static int WriteContentBlockJob(void* context, uint32_t job_id, int detected_err { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "source_storage_api->GetSize() failed with %d", err); Longtail_Free(put_block_mem); - LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, last_asset_index, err); + LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, last_asset_index); source_storage_api->CloseFile(source_storage_api, file_handle); return err; } @@ -4654,17 +4674,17 @@ static int WriteContentBlockJob(void* context, uint32_t job_id, int detected_err asset_file_size, (asset_offset + chunk_size)) Longtail_Free(put_block_mem); source_storage_api->CloseFile(source_storage_api, file_handle); - LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, asset_index, EBADF); + LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, asset_index); return EBADF; } - LONGTAIL_MONTITOR_ASSET_READ(job->m_StoreIndex, job->m_VersionIndex, asset_index, asset_offset, chunk_size, chunk_hash, job->m_BlockIndex, write_offset); int err = source_storage_api->Read(source_storage_api, file_handle, asset_offset, chunk_size, write_buffer + write_offset); + LONGTAIL_MONTITOR_ASSET_READ(job->m_StoreIndex, job->m_VersionIndex, asset_index, asset_offset, chunk_size, chunk_hash, job->m_BlockIndex, write_offset, err); if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "source_storage_api->Read() failed with %d", err); Longtail_Free(put_block_mem); source_storage_api->CloseFile(source_storage_api, file_handle); - LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, last_asset_index, err); + LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, last_asset_index); return err; } write_offset += chunk_size; @@ -4674,7 +4694,7 @@ static int WriteContentBlockJob(void* context, uint32_t job_id, int detected_err { source_storage_api->CloseFile(source_storage_api, file_handle); file_handle = 0; - LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, last_asset_index, 0); + LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, last_asset_index); } Longtail_InitBlockIndex(block_index_ptr, chunk_count); @@ -7309,13 +7329,11 @@ static SORTFUNC(SortPathShortToLong) LONGTAIL_FATAL_ASSERT(ctx, a_ptr != 0, return 0) LONGTAIL_FATAL_ASSERT(ctx, b_ptr != 0, return 0) - const struct Longtail_VersionIndex* version_index = (const struct Longtail_VersionIndex*)context; + const uint32_t* asset_path_lengths = (const uint32_t*)context; uint32_t a = *(const uint32_t*)a_ptr; uint32_t b = *(const uint32_t*)b_ptr; - const char* a_path = &version_index->m_NameData[version_index->m_NameOffsets[a]]; - const char* b_path = &version_index->m_NameData[version_index->m_NameOffsets[b]]; - size_t a_len = strlen(a_path); - size_t b_len = strlen(b_path); + uint32_t a_len = asset_path_lengths[a]; + uint32_t b_len = asset_path_lengths[b]; if (a_len < b_len) { return -1; @@ -7350,15 +7368,28 @@ static SORTFUNC(SortPathLongToShort) LONGTAIL_FATAL_ASSERT(ctx, context != 0, return 0) LONGTAIL_FATAL_ASSERT(ctx, a_ptr != 0, return 0) LONGTAIL_FATAL_ASSERT(ctx, b_ptr != 0, return 0) - - const struct Longtail_VersionIndex* version_index = (const struct Longtail_VersionIndex*)context; + const uint32_t* asset_path_lengths = (const uint32_t*)context; uint32_t a = *(const uint32_t*)a_ptr; uint32_t b = *(const uint32_t*)b_ptr; - const char* a_path = &version_index->m_NameData[version_index->m_NameOffsets[a]]; - const char* b_path = &version_index->m_NameData[version_index->m_NameOffsets[b]]; - size_t a_len = strlen(a_path); - size_t b_len = strlen(b_path); - return (a_len < b_len) ? 1 : (a_len > b_len) ? -1 : 0; + uint32_t a_len = asset_path_lengths[a]; + uint32_t b_len = asset_path_lengths[b]; + if (a_len < b_len) + { + return 1; + } + if (a_len > b_len) + { + return -1; + } + if (a < b) + { + return 1; + } + if (a > b) + { + return -1; + } + return 0; } static size_t GetVersionDiffDataSize(uint32_t removed_count, uint32_t added_count, uint32_t modified_content_count, uint32_t modified_permission_count) @@ -7465,6 +7496,8 @@ int Longtail_CreateVersionDiff( sizeof(uint32_t) * source_asset_count + sizeof(uint32_t) * target_asset_count + sizeof(uint32_t) * source_asset_count + + sizeof(uint32_t) * target_asset_count + + sizeof(uint32_t) * source_asset_count + sizeof(uint32_t) * target_asset_count; void* work_mem = Longtail_Alloc("CreateVersionDiff", work_mem_size); uint8_t* p = (uint8_t*)work_mem; @@ -7486,32 +7519,63 @@ int Longtail_CreateVersionDiff( uint32_t* modified_source_permissions_indexes = &modified_target_content_indexes[target_asset_count]; uint32_t* modified_target_permissions_indexes = &modified_source_permissions_indexes[source_asset_count]; - for (uint32_t i = 0; i < source_asset_count; ++i) + uint32_t* source_assets_path_lengths = &modified_target_permissions_indexes[target_asset_count]; + uint32_t* target_assets_path_lengths = &source_assets_path_lengths[source_asset_count]; + + if (*source_version->m_Version < LONGTAIL_VERSION_INDEX_VERSION_0_0_2) { // We are re-hashing since we might have an older version hash that is incompatible - const char* path = &source_version->m_NameData[source_version->m_NameOffsets[i]]; - int err = LongtailPrivate_GetPathHash(hash_api, path, &source_path_hashes[i]); - if (err) + for (uint32_t i = 0; i < source_asset_count; ++i) { - LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "LongtailPrivate_GetPathHash() failed with %d", err) - Longtail_Free(work_mem); - return err; + const char* path = &source_version->m_NameData[source_version->m_NameOffsets[i]]; + source_assets_path_lengths[i] = (uint32_t)strlen(path); + int err = GetPathHashWithLength(hash_api, path, source_assets_path_lengths[i], &source_path_hashes[i]); + if (err) + { + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "LongtailPrivate_GetPathHash() failed with %d", err) + Longtail_Free(work_mem); + return err; + } + LongtailPrivate_LookupTable_Put(source_path_hash_to_index, source_path_hashes[i], i); + } + } + else + { + memcpy(source_path_hashes, source_version->m_PathHashes, sizeof(TLongtail_Hash) * source_asset_count); + for (uint32_t i = 0; i < source_asset_count; ++i) + { + const char* path = &source_version->m_NameData[source_version->m_NameOffsets[i]]; + source_assets_path_lengths[i] = (uint32_t)strlen(path); + LongtailPrivate_LookupTable_Put(source_path_hash_to_index, source_version->m_PathHashes[i], i); } - LongtailPrivate_LookupTable_Put(source_path_hash_to_index, source_path_hashes[i], i); } - for (uint32_t i = 0; i < target_asset_count; ++i) + if (*target_version->m_Version < LONGTAIL_VERSION_INDEX_VERSION_0_0_2) { // We are re-hashing since we might have an older version hash that is incompatible - const char* path = &target_version->m_NameData[target_version->m_NameOffsets[i]]; - int err = LongtailPrivate_GetPathHash(hash_api, path, &target_path_hashes[i]); - if (err) + for (uint32_t i = 0; i < target_asset_count; ++i) { - LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "LongtailPrivate_GetPathHash() failed with %d", err) - Longtail_Free(work_mem); - return err; + const char* path = &target_version->m_NameData[target_version->m_NameOffsets[i]]; + target_assets_path_lengths[i] = (uint32_t)strlen(path); + int err = GetPathHashWithLength(hash_api, path, target_assets_path_lengths[i], &target_path_hashes[i]); + if (err) + { + LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "LongtailPrivate_GetPathHash() failed with %d", err) + Longtail_Free(work_mem); + return err; + } + LongtailPrivate_LookupTable_Put(target_path_hash_to_index, target_path_hashes[i], i); + } + } + else + { + memcpy(target_path_hashes, target_version->m_PathHashes, sizeof(TLongtail_Hash) * target_asset_count); + for (uint32_t i = 0; i < target_asset_count; ++i) + { + const char* path = &target_version->m_NameData[target_version->m_NameOffsets[i]]; + target_assets_path_lengths[i] = (uint32_t)strlen(path); + LongtailPrivate_LookupTable_Put(target_path_hash_to_index, target_version->m_PathHashes[i], i); } - LongtailPrivate_LookupTable_Put(target_path_hash_to_index, target_path_hashes[i], i); } qsort(source_path_hashes, source_asset_count, sizeof(TLongtail_Hash), CompareHashes); @@ -7655,8 +7719,8 @@ int Longtail_CreateVersionDiff( memmove(version_diff->m_SourcePermissionsModifiedAssetIndexes, modified_source_permissions_indexes, sizeof(uint32_t) * modified_permissions_count); memmove(version_diff->m_TargetPermissionsModifiedAssetIndexes, modified_target_permissions_indexes, sizeof(uint32_t) * modified_permissions_count); - QSORT(version_diff->m_SourceRemovedAssetIndexes, source_removed_count, sizeof(uint32_t), SortPathLongToShort, (void*)source_version); - QSORT(version_diff->m_TargetAddedAssetIndexes, target_added_count, sizeof(uint32_t), SortPathShortToLong, (void*)target_version); + QSORT(version_diff->m_SourceRemovedAssetIndexes, source_removed_count, sizeof(uint32_t), SortPathLongToShort, (void*)source_assets_path_lengths); + QSORT(version_diff->m_TargetAddedAssetIndexes, target_added_count, sizeof(uint32_t), SortPathShortToLong, (void*)target_assets_path_lengths); Longtail_Free(work_mem); *out_version_diff = version_diff; @@ -7740,12 +7804,12 @@ static int CleanUpRemoveAssets( Longtail_Free(remove_indexes); return err; } - LONGTAIL_MONITOR_ASSET_REMOVE(source_version, asset_index) uint16_t permissions = 0; err = version_storage_api->GetPermissions(version_storage_api, full_asset_path, &permissions); if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "version_storage_api->GetPermissions() failed with %d", err) + LONGTAIL_MONITOR_ASSET_REMOVE(source_version, asset_index, err) Longtail_Free(full_asset_path); Longtail_Free(remove_indexes); return err; @@ -7756,6 +7820,7 @@ static int CleanUpRemoveAssets( if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "version_storage_api->SetPermissions() failed with %d", err) + LONGTAIL_MONITOR_ASSET_REMOVE(source_version, asset_index, err) Longtail_Free(full_asset_path); Longtail_Free(remove_indexes); return err; @@ -7769,12 +7834,14 @@ static int CleanUpRemoveAssets( LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Can't to remove dir `%s`, failed with %d", full_asset_path, err) Longtail_Free(full_asset_path); Longtail_Free(remove_indexes); + LONGTAIL_MONITOR_ASSET_REMOVE(source_version, asset_index, err) return err; } Longtail_Free(full_asset_path); full_asset_path = 0; continue; } + LONGTAIL_MONITOR_ASSET_REMOVE(source_version, asset_index, 0) } else { @@ -7784,12 +7851,12 @@ static int CleanUpRemoveAssets( Longtail_Free(full_asset_path); continue; } - LONGTAIL_MONITOR_ASSET_REMOVE(source_version, asset_index) uint16_t permissions = 0; err = version_storage_api->GetPermissions(version_storage_api, full_asset_path, &permissions); if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "version_storage_api->GetPermissions() failed with %d", err) + LONGTAIL_MONITOR_ASSET_REMOVE(source_version, asset_index, err) Longtail_Free(full_asset_path); Longtail_Free(remove_indexes); return err; @@ -7800,6 +7867,7 @@ static int CleanUpRemoveAssets( if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "version_storage_api->SetPermissions() failed with %d", err) + LONGTAIL_MONITOR_ASSET_REMOVE(source_version, asset_index, err) Longtail_Free(full_asset_path); Longtail_Free(remove_indexes); return err; @@ -7811,6 +7879,7 @@ static int CleanUpRemoveAssets( if (!retry_count) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Can't to file dir `%s`, failed with %d", full_asset_path, err) + LONGTAIL_MONITOR_ASSET_REMOVE(source_version, asset_index, err) Longtail_Free(full_asset_path); Longtail_Free(remove_indexes); return err; @@ -7819,6 +7888,7 @@ static int CleanUpRemoveAssets( full_asset_path = 0; continue; } + LONGTAIL_MONITOR_ASSET_REMOVE(source_version, asset_index, 0) } Longtail_Free(full_asset_path); full_asset_path = 0; @@ -8189,26 +8259,28 @@ static int WriteNonBlockAssetsJob(void* context, uint32_t job_id, int detected_e const TBlockChunkWriteArray block_chunk_write_info = &block_write_infos[block_write_chunk_info_index]; const uint32_t asset_index = block_chunk_write_info->AssetIndex; const char* asset_path = &job->m_VersionIndex->m_NameData[job->m_VersionIndex->m_NameOffsets[asset_index]]; - LONGTAIL_MONTITOR_ASSET_OPEN(job->m_VersionIndex, asset_index); if (IsDirPath(asset_path)) { - int err = job->m_ConcurrentChunkWriteApi->CreateDir(job->m_ConcurrentChunkWriteApi, asset_path); + int err = job->m_ConcurrentChunkWriteApi->CreateDir(job->m_ConcurrentChunkWriteApi, asset_index); if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "job->m_ConcurrentChunkWriteApi->CreateDir() failed with %d", err) + LONGTAIL_MONTITOR_ASSET_OPEN(job->m_VersionIndex, asset_index, err); return err; } } else { - Longtail_ConcurrentChunkWriteAPI_HOpenFile asset_file_handle = 0; - int err = job->m_ConcurrentChunkWriteApi->Open(job->m_ConcurrentChunkWriteApi, asset_path, 0, &asset_file_handle); + int err = job->m_ConcurrentChunkWriteApi->Open(job->m_ConcurrentChunkWriteApi, asset_index); if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "m_ConcurrentChunkWriteApi->Open() failed with %d", err) return err; } + LONGTAIL_MONTITOR_ASSET_OPEN(job->m_VersionIndex, asset_index, err); + job->m_ConcurrentChunkWriteApi->Close(job->m_ConcurrentChunkWriteApi, asset_index); } + LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_VersionIndex, asset_index); } return 0; } @@ -8296,8 +8368,8 @@ static int WriteContentBlock2Job(void* context, uint32_t job_id, int detected_er const uint8_t* block_data = (const uint8_t*)stored_block->m_BlockData; + int asset_is_open = 0; uint32_t last_asset_index = 0; - Longtail_ConcurrentChunkWriteAPI_HOpenFile asset_file_handle = 0; TBlockChunkWriteArray write_infos = job->m_Context->m_BlockWriteInfos->m_BlockWritesArrays[block_index]; ptrdiff_t block_write_chunk_info_count = arrlen(write_infos); @@ -8311,29 +8383,30 @@ static int WriteContentBlock2Job(void* context, uint32_t job_id, int detected_er const uint32_t asset_index = block_chunk_write_info->AssetIndex; const char* asset_path = &job->m_Context->m_VersionIndex->m_NameData[job->m_Context->m_VersionIndex->m_NameOffsets[asset_index]]; - if (asset_file_handle != 0) + if (asset_is_open) { if (last_asset_index != asset_index) { - asset_file_handle = 0; + job->m_Context->m_ConcurrentChunkWriteApi->Close(job->m_Context->m_ConcurrentChunkWriteApi, last_asset_index); + asset_is_open = 0; } } - if (asset_file_handle == 0) + if (asset_is_open == 0) { uint32_t asset_chunk_count = job->m_Context->m_VersionIndex->m_AssetChunkCounts[asset_index]; - LONGTAIL_MONTITOR_ASSET_OPEN(job->m_Context->m_VersionIndex, asset_index); - int err = job->m_Context->m_ConcurrentChunkWriteApi->Open(job->m_Context->m_ConcurrentChunkWriteApi, asset_path, asset_chunk_count, &asset_file_handle); + int err = job->m_Context->m_ConcurrentChunkWriteApi->Open(job->m_Context->m_ConcurrentChunkWriteApi, asset_index); if (err) { LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Open() failed for `%s` with %d", asset_path, err) + LONGTAIL_MONTITOR_ASSET_OPEN(job->m_Context->m_VersionIndex, asset_index, err); Longtail_Free(work_mem); SAFE_DISPOSE_STORED_BLOCK(job->m_StoredBlock); - LONGTAIL_MONTITOR_ASSET_COMPLETE(job->m_Context->m_VersionIndex, asset_index, err); LONGTAIL_MONTITOR_BLOCK_COMPLETE(job->m_Context->m_StoreIndex, job->m_BlockIndex, err); return err; } last_asset_index = asset_index; + asset_is_open = 1; } uint32_t chunk_index = block_chunk_write_info->ChunkIndex; @@ -8341,10 +8414,12 @@ static int WriteContentBlock2Job(void* context, uint32_t job_id, int detected_er uint32_t* chunk_index_in_block_ptr = LongtailPrivate_LookupTable_Get(chunk_hash_to_chunk_index, chunk_hash); if (chunk_index_in_block_ptr == 0) { + job->m_Context->m_ConcurrentChunkWriteApi->Close(job->m_Context->m_ConcurrentChunkWriteApi, asset_index); + asset_is_open = 0; + LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_Context->m_VersionIndex, asset_index); LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to find chunk in block for `%s` with %d", asset_path, ENOENT) Longtail_Free(work_mem); SAFE_DISPOSE_STORED_BLOCK(job->m_StoredBlock); - LONGTAIL_MONTITOR_ASSET_COMPLETE(job->m_Context->m_VersionIndex, asset_index, ENOENT); LONGTAIL_MONTITOR_BLOCK_COMPLETE(job->m_Context->m_StoreIndex, job->m_BlockIndex, ENOENT); return ENOENT; } @@ -8352,7 +8427,7 @@ static int WriteContentBlock2Job(void* context, uint32_t job_id, int detected_er uint32_t chunk_index_in_block = *chunk_index_in_block_ptr; uint32_t chunk_size = stored_block->m_BlockIndex->m_ChunkSizes[chunk_index_in_block]; uint32_t chunk_offset_in_block = chunk_offsets_in_block[chunk_index_in_block]; - LONGTAIL_MONTITOR_CHUNK_READ(job->m_Context->m_StoreIndex, job->m_Context->m_VersionIndex, block_index, chunk_index, chunk_index_in_block); + LONGTAIL_MONTITOR_CHUNK_READ(job->m_Context->m_StoreIndex, job->m_Context->m_VersionIndex, block_index, chunk_index, chunk_index_in_block, 0); uint32_t chunk_run_count = 1; uint32_t chunk_run_size = chunk_size; @@ -8372,10 +8447,12 @@ static int WriteContentBlock2Job(void* context, uint32_t job_id, int detected_er uint32_t* chunk_index_in_block_ptr_next = LongtailPrivate_LookupTable_Get(chunk_hash_to_chunk_index, chunk_hash_next); if (chunk_index_in_block_ptr_next == 0) { + job->m_Context->m_ConcurrentChunkWriteApi->Close(job->m_Context->m_ConcurrentChunkWriteApi, asset_index); + asset_is_open = 0; + LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_Context->m_VersionIndex, asset_index); LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to find chunk in block for `%s` with %d", asset_path, ENOENT) Longtail_Free(work_mem); SAFE_DISPOSE_STORED_BLOCK(job->m_StoredBlock); - LONGTAIL_MONTITOR_ASSET_COMPLETE(job->m_Context->m_VersionIndex, asset_index, ENOENT); LONGTAIL_MONTITOR_BLOCK_COMPLETE(job->m_Context->m_StoreIndex, job->m_BlockIndex, ENOENT); return ENOENT; } @@ -8385,33 +8462,38 @@ static int WriteContentBlock2Job(void* context, uint32_t job_id, int detected_er break; } uint32_t chunk_size_next = stored_block->m_BlockIndex->m_ChunkSizes[chunk_index_in_block_next]; - LONGTAIL_MONTITOR_CHUNK_READ(job->m_Context->m_StoreIndex, job->m_Context->m_VersionIndex, block_index, chunk_index_next, chunk_index_in_block_next); + LONGTAIL_MONTITOR_CHUNK_READ(job->m_Context->m_StoreIndex, job->m_Context->m_VersionIndex, block_index, chunk_index_next, chunk_index_in_block_next, 0); chunk_run_size += chunk_size_next; chunk_run_count++; } - LONGTAIL_MONTITOR_ASSET_WRITE(job->m_Context->m_StoreIndex, job->m_Context->m_VersionIndex, asset_index, block_chunk_write_info->Offset, chunk_run_size, chunk_index, chunk_index_in_block, chunk_run_count, block_index, chunk_offset_in_block); - uint32_t pending_chunk_count = 0; - int err = job->m_Context->m_ConcurrentChunkWriteApi->Write(job->m_Context->m_ConcurrentChunkWriteApi, asset_file_handle, block_chunk_write_info->Offset, chunk_run_size, chunk_run_count, &block_data[chunk_offset_in_block], &pending_chunk_count); + int err = job->m_Context->m_ConcurrentChunkWriteApi->Write(job->m_Context->m_ConcurrentChunkWriteApi, asset_index, block_chunk_write_info->Offset, chunk_run_size, &block_data[chunk_offset_in_block]); + LONGTAIL_MONTITOR_ASSET_WRITE(job->m_Context->m_StoreIndex, job->m_Context->m_VersionIndex, asset_index, block_chunk_write_info->Offset, chunk_run_size, chunk_index, chunk_index_in_block, chunk_run_count, block_index, chunk_offset_in_block, err); if (err) { + job->m_Context->m_ConcurrentChunkWriteApi->Close(job->m_Context->m_ConcurrentChunkWriteApi, asset_index); + asset_is_open = 0; + LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_Context->m_VersionIndex, asset_index); LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Write() failed for `%s` with %d", asset_path, err) Longtail_Free(work_mem); SAFE_DISPOSE_STORED_BLOCK(job->m_StoredBlock); - LONGTAIL_MONTITOR_ASSET_COMPLETE(job->m_Context->m_VersionIndex, asset_index, err); LONGTAIL_MONTITOR_BLOCK_COMPLETE(job->m_Context->m_StoreIndex, job->m_BlockIndex, err); return err; } - if (pending_chunk_count == 0) - { - LONGTAIL_MONTITOR_ASSET_COMPLETE(job->m_Context->m_VersionIndex, asset_index, 0); - } block_write_chunk_info_index += chunk_run_count; } + if (asset_is_open != 0) + { + job->m_Context->m_ConcurrentChunkWriteApi->Close(job->m_Context->m_ConcurrentChunkWriteApi, last_asset_index); + asset_is_open = 0; + LONGTAIL_MONTITOR_ASSET_CLOSE(job->m_Context->m_VersionIndex, last_asset_index); + } + Longtail_Free(work_mem); SAFE_DISPOSE_STORED_BLOCK(job->m_StoredBlock); LONGTAIL_MONTITOR_BLOCK_COMPLETE(job->m_Context->m_StoreIndex, job->m_BlockIndex, 0); + job->m_Context->m_ConcurrentChunkWriteApi->Flush(job->m_Context->m_ConcurrentChunkWriteApi); return 0; } diff --git a/src/longtail.h b/src/longtail.h index 745564b5..2597b8dc 100644 --- a/src/longtail.h +++ b/src/longtail.h @@ -359,6 +359,7 @@ typedef int (*Longtail_Storage_UnlockFileFunc)(struct Longtail_StorageAPI* stora typedef char* (*Longtail_Storage_GetParentPathFunc)(struct Longtail_StorageAPI* storage_api, const char* path); typedef int (*Longtail_Storage_MapFileFunc)(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HOpenFile f, uint64_t offset, uint64_t length, Longtail_StorageAPI_HFileMap* out_file_map, const void** out_data_ptr); typedef void (*Longtail_Storage_UnmapFileFunc)(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HFileMap m); +typedef int (*Longtail_Storage_OpenAppendFileFunc)(struct Longtail_StorageAPI* storage_api, const char* path, Longtail_StorageAPI_HOpenFile* out_open_file); struct Longtail_StorageAPI { @@ -388,6 +389,7 @@ struct Longtail_StorageAPI Longtail_Storage_GetParentPathFunc GetParentPath; Longtail_Storage_MapFileFunc MapFile; Longtail_Storage_UnmapFileFunc UnMapFile; + Longtail_Storage_OpenAppendFileFunc OpenAppendFile; }; LONGTAIL_EXPORT uint64_t Longtail_GetStorageAPISize(); @@ -419,7 +421,8 @@ LONGTAIL_EXPORT struct Longtail_StorageAPI* Longtail_MakeStorageAPI( Longtail_Storage_UnlockFileFunc unlock_file_func, Longtail_Storage_GetParentPathFunc get_parent_path_func, Longtail_Storage_MapFileFunc map_file_func, - Longtail_Storage_UnmapFileFunc unmap_file_func); + Longtail_Storage_UnmapFileFunc unmap_file_func, + Longtail_Storage_OpenAppendFileFunc open_append_file_func); LONGTAIL_EXPORT int Longtail_Storage_OpenReadFile(struct Longtail_StorageAPI* storage_api, const char* path, Longtail_StorageAPI_HOpenFile* out_open_file); LONGTAIL_EXPORT int Longtail_Storage_GetSize(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HOpenFile f, uint64_t* out_size); @@ -446,16 +449,16 @@ LONGTAIL_EXPORT int Longtail_Storage_UnlockFile(struct Longtail_StorageAPI* stor LONGTAIL_EXPORT char* Longtail_Storage_GetParentPath(struct Longtail_StorageAPI* storage_api, const char* path); LONGTAIL_EXPORT int Longtail_Storage_MapFile(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HOpenFile f, uint64_t offset, uint64_t length, Longtail_StorageAPI_HFileMap* out_file_map, const void** out_data_ptr); LONGTAIL_EXPORT void Longtail_Storage_UnmapFile(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HFileMap m); +LONGTAIL_EXPORT void Longtail_Storage_OpenAppendfile(struct Longtail_StorageAPI* storage_api, const char* path, Longtail_StorageAPI_HOpenFile* out_open_file); ////////////// Longtail_ConcurrentChunkWriteAPI struct Longtail_ConcurrentChunkWriteAPI; -typedef struct Longtail_ConcurrentChunkWriteAPI_OpenFile* Longtail_ConcurrentChunkWriteAPI_HOpenFile; - -typedef int (*Longtail_ConcurrentChunkWrite_CreateDirFunc)(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, const char* path); -typedef int (*Longtail_ConcurrentChunkWrite_OpenFunc)(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, const char* path, uint32_t chunk_write_count, Longtail_ConcurrentChunkWriteAPI_HOpenFile* out_open_file); -typedef int (*Longtail_ConcurrentChunkWrite_WriteFunc)(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, Longtail_ConcurrentChunkWriteAPI_HOpenFile in_open_file, uint64_t offset, uint32_t size, uint32_t chunk_count, const void* input, uint32_t* out_chunks_remaining); +typedef int (*Longtail_ConcurrentChunkWrite_CreateDirFunc)(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index); +typedef int (*Longtail_ConcurrentChunkWrite_OpenFunc)(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index); +typedef void (*Longtail_ConcurrentChunkWrite_CloseFunc)(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index); +typedef int (*Longtail_ConcurrentChunkWrite_WriteFunc)(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index, uint64_t offset, uint32_t size, const void* input); typedef int (*Longtail_ConcurrentChunkWrite_FlushFunc)(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api); struct Longtail_ConcurrentChunkWriteAPI @@ -463,6 +466,7 @@ struct Longtail_ConcurrentChunkWriteAPI struct Longtail_API m_API; Longtail_ConcurrentChunkWrite_CreateDirFunc CreateDir; Longtail_ConcurrentChunkWrite_OpenFunc Open; + Longtail_ConcurrentChunkWrite_CloseFunc Close; Longtail_ConcurrentChunkWrite_WriteFunc Write; Longtail_ConcurrentChunkWrite_FlushFunc Flush; }; @@ -474,12 +478,15 @@ LONGTAIL_EXPORT struct Longtail_ConcurrentChunkWriteAPI* Longtail_MakeConcurrent Longtail_DisposeFunc dispose_func, Longtail_ConcurrentChunkWrite_CreateDirFunc create_dir_func, Longtail_ConcurrentChunkWrite_OpenFunc open_func, + Longtail_ConcurrentChunkWrite_CloseFunc close_func, Longtail_ConcurrentChunkWrite_WriteFunc write_func, - Longtail_ConcurrentChunkWrite_FlushFunc flush_func); + Longtail_ConcurrentChunkWrite_FlushFunc flush_func +); -LONGTAIL_EXPORT int Longtail_ConcurrentChunkWrite_CreateDir(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, const char* path); -LONGTAIL_EXPORT int Longtail_ConcurrentChunkWrite_Open(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, const char* path, uint32_t chunk_write_count, Longtail_ConcurrentChunkWriteAPI_HOpenFile* out_open_file); -LONGTAIL_EXPORT int Longtail_ConcurrentChunkWrite_Write(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, Longtail_ConcurrentChunkWriteAPI_HOpenFile in_open_file, uint64_t offset, uint32_t size, uint32_t chunk_count, const void* input, uint32_t* out_total_chunks_written); +LONGTAIL_EXPORT int Longtail_ConcurrentChunkWrite_CreateDir(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index); +LONGTAIL_EXPORT int Longtail_ConcurrentChunkWrite_Open(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index); +LONGTAIL_EXPORT void Longtail_ConcurrentChunkWrite_Close(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index); +LONGTAIL_EXPORT int Longtail_ConcurrentChunkWrite_Write(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api, uint32_t asset_index, uint64_t offset, uint32_t size, const void* input); LONGTAIL_EXPORT int Longtail_ConcurrentChunkWrite_Flush(struct Longtail_ConcurrentChunkWriteAPI* concurrent_file_write_api); ////////////// Longtail_ProgressAPI @@ -820,16 +827,15 @@ typedef void (*Longtail_MonitorGetStoredBlockPrepare)(const struct Longtail_Stor typedef void (*Longtail_MonitorGetStoredBlockLoad)(const struct Longtail_StoreIndex* store_index, uint32_t block_index); typedef void (*Longtail_MonitorGetStoredBlockLoaded)(const struct Longtail_StoreIndex* store_index, uint32_t block_index, int err); typedef void (*Longtail_MonitorGetStoredBlockComplete)(const struct Longtail_StoreIndex* store_index, uint32_t block_index, int err); -typedef void (*Longtail_MonitorAssetRemove)(const struct Longtail_VersionIndex* source_version_index, uint32_t asset_index); -typedef void (*Longtail_MonitorAssetOpen)(const struct Longtail_VersionIndex* target_version_index, uint32_t asset_index); -typedef void (*Longtail_MonitorAssetWrite)(const struct Longtail_StoreIndex* target_store_index, const struct Longtail_VersionIndex* version_index, uint32_t asset_index, uint64_t write_offset, uint32_t size, uint32_t chunk_index, uint32_t chunk_index_in_block, uint32_t chunk_count_in_block, uint32_t block_index, uint32_t block_data_offset); -typedef void (*Longtail_MonitorChunkRead)(const struct Longtail_StoreIndex* store_index, const struct Longtail_VersionIndex* target_version_index, uint32_t block_index, uint32_t chunk_index, uint32_t chunk_index_in_block); -typedef void (*Longtail_MonitorAssetComplete)(const struct Longtail_VersionIndex* target_version_index, uint32_t asset_index, int err); +typedef void (*Longtail_MonitorAssetRemove)(const struct Longtail_VersionIndex* source_version_index, uint32_t asset_index, int err); +typedef void (*Longtail_MonitorAssetOpen)(const struct Longtail_VersionIndex* target_version_index, uint32_t asset_index, int err); +typedef void (*Longtail_MonitorAssetWrite)(const struct Longtail_StoreIndex* target_store_index, const struct Longtail_VersionIndex* version_index, uint32_t asset_index, uint64_t write_offset, uint32_t size, uint32_t chunk_index, uint32_t chunk_index_in_block, uint32_t chunk_count_in_block, uint32_t block_index, uint32_t block_data_offset, int err); +typedef void (*Longtail_MonitorChunkRead)(const struct Longtail_StoreIndex* store_index, const struct Longtail_VersionIndex* target_version_index, uint32_t block_index, uint32_t chunk_index, uint32_t chunk_index_in_block, int err); typedef void (*Longtail_MonitorBlockCompose)(const struct Longtail_StoreIndex* store_index, uint32_t block_index); typedef void (*Longtail_MonitorBlockSave)(const struct Longtail_StoreIndex* store_index, uint32_t block_index, uint64_t block_size); typedef void (*Longtail_MonitorBlockSaved)(const struct Longtail_StoreIndex* store_index, uint32_t block_index, int err); -typedef void (*Longtail_MonitorAssetRead)(const struct Longtail_StoreIndex* store_index, const struct Longtail_VersionIndex* version_index, uint32_t asset_index, uint64_t read_offset, uint32_t size, TLongtail_Hash chunk_hash, uint32_t block_index, uint32_t block_data_offset); -typedef void (*Longtail_MonitorAssetClose)(const struct Longtail_VersionIndex* version_index, uint32_t asset_index, int err); +typedef void (*Longtail_MonitorAssetRead)(const struct Longtail_StoreIndex* store_index, const struct Longtail_VersionIndex* version_index, uint32_t asset_index, uint64_t read_offset, uint32_t size, TLongtail_Hash chunk_hash, uint32_t block_index, uint32_t block_data_offset, int err); +typedef void (*Longtail_MonitorAssetClose)(const struct Longtail_VersionIndex* version_index, uint32_t asset_index); struct Longtail_Monitor { @@ -842,7 +848,6 @@ struct Longtail_Monitor Longtail_MonitorAssetOpen AssetOpen; Longtail_MonitorAssetWrite AssetWrite; Longtail_MonitorChunkRead ChunkRead; - Longtail_MonitorAssetComplete AssetComplete; Longtail_MonitorBlockCompose BlockCompose; Longtail_MonitorBlockSave BlockSave; Longtail_MonitorBlockSaved BlockSaved; diff --git a/test/test.cpp b/test/test.cpp index ed732b7e..5d191a1b 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -3018,7 +3018,7 @@ TEST(Longtail, Longtail_VersionDiff) Longtail_Free(required_chunk_hashes); required_chunk_hashes = 0; - Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(storage, "old"); + Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(storage, new_vindex, version_diff, "old"); ASSERT_EQ(0, Longtail_ChangeVersion2( block_store_api, storage, @@ -3059,7 +3059,7 @@ TEST(Longtail, Longtail_VersionDiff) store_index = SyncGetExistingContent(block_store_api, required_chunk_count, required_chunk_hashes, 0); - concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(storage, "old"); + concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(storage, new_vindex, version_diff, "old"); ASSERT_EQ(0, Longtail_ChangeVersion2( block_store_api, storage, @@ -5271,7 +5271,7 @@ TEST(Longtail, TestChangeVersionCancelOperation) ASSERT_EQ(0, cancel_api->CreateToken(cancel_api, &cancel_token)); ASSERT_NE((Longtail_CancelAPI_HCancelToken)0, cancel_token); - Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(storage, "old"); + Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(storage, vindex, version_diff, "old"); ASSERT_EQ(ECANCELED, Longtail_ChangeVersion2( block_store_proxy, storage, @@ -5348,7 +5348,7 @@ TEST(Longtail, TestChangeVersionCancelOperation) ASSERT_NE((Longtail_CancelAPI_HCancelToken)0, cancel_token); blockStoreProxy.m_FailCounter = 0x7fffffff; - Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(storage, "old"); + Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(storage, vindex, version_diff, "old"); int err = Longtail_ChangeVersion2( block_store_proxy, storage, @@ -5689,7 +5689,8 @@ struct FailableStorageAPI static int UnlockFile(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HLockFile lock_file) { struct FailableStorageAPI* api = (struct FailableStorageAPI*)storage_api; return api->m_BackingAPI->UnlockFile(api->m_BackingAPI, lock_file);} static char* GetParentPath(struct Longtail_StorageAPI* storage_api, const char* path) { struct FailableStorageAPI* api = (struct FailableStorageAPI*)storage_api; return api->m_BackingAPI->GetParentPath(api->m_BackingAPI, path);} static int MapFile(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HOpenFile f, uint64_t offset, uint64_t length, Longtail_StorageAPI_HFileMap* out_file_map, const void** out_data_ptr) { struct FailableStorageAPI* api = (struct FailableStorageAPI*)storage_api; return api->m_BackingAPI->MapFile(api->m_BackingAPI, f, offset, length, out_file_map, out_data_ptr);} - static void UnmapFile(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HFileMap m) { struct FailableStorageAPI* api = (struct FailableStorageAPI*)storage_api; return api->m_BackingAPI->UnMapFile(api->m_BackingAPI, m);} + static void UnmapFile(struct Longtail_StorageAPI* storage_api, Longtail_StorageAPI_HFileMap m) { struct FailableStorageAPI* api = (struct FailableStorageAPI*)storage_api; return api->m_BackingAPI->UnMapFile(api->m_BackingAPI, m); } + static int OpenAppendFile(struct Longtail_StorageAPI* storage_api, const char* path, Longtail_StorageAPI_HOpenFile* out_open_file) { struct FailableStorageAPI* api = (struct FailableStorageAPI*)storage_api; return api->m_BackingAPI->OpenAppendFile(api->m_BackingAPI, path, out_open_file); } }; struct FailableStorageAPI* CreateFailableStorageAPI(struct Longtail_StorageAPI* backing_api) @@ -5722,7 +5723,8 @@ struct FailableStorageAPI* CreateFailableStorageAPI(struct Longtail_StorageAPI* FailableStorageAPI::UnlockFile, FailableStorageAPI::GetParentPath, FailableStorageAPI::MapFile, - FailableStorageAPI::UnmapFile); + FailableStorageAPI::UnmapFile, + FailableStorageAPI::OpenAppendFile); struct FailableStorageAPI* failable_storage_api = (struct FailableStorageAPI*)api; failable_storage_api->m_BackingAPI = backing_api; failable_storage_api->m_PassCount = 0x7fffffff; @@ -5899,7 +5901,7 @@ TEST(Longtail, TestChangeVersionDiskFull) Longtail_SetLogLevel(LONGTAIL_LOG_LEVEL_OFF); failable_local_storage_api->m_PassCount = 3; failable_local_storage_api->m_WriteError = ENOSPC; - Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(local_storage, "old"); + Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(local_storage, vindex, version_diff, "old"); ASSERT_EQ(ENOSPC, Longtail_ChangeVersion2( cached_compress_store_api, local_storage, @@ -6534,8 +6536,22 @@ static int DownloadFolder( } Longtail_Free(required_chunk_hashes); - Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(storage_api, target_path); - err = Longtail_ChangeVersion2(block_store_api, storage_api, concurrent_chunk_write_api, hash_api, job_api, 0, 0, 0, store_index, current_version_index, version_index, version_diff, target_path, 1); + Longtail_ConcurrentChunkWriteAPI* concurrent_chunk_write_api = Longtail_CreateConcurrentChunkWriteAPI(storage_api, version_index, version_diff, target_path); + err = Longtail_ChangeVersion2( + block_store_api, + storage_api, + concurrent_chunk_write_api, + hash_api, + job_api, + 0, + 0, + 0, + store_index, + current_version_index, + version_index, + version_diff, + target_path, + 1); SAFE_DISPOSE_API(concurrent_chunk_write_api); if (err) {