Skip to content

Commit d8ce69d

Browse files
gotliebgotlieb
authored
Merge pull request #10462 from kaltura/Propus-16.17.0-FOUN-111
FOUN-111
2 parents 624c890 + 88206d3 commit d8ce69d

File tree

6 files changed

+64
-75
lines changed

6 files changed

+64
-75
lines changed

‎batch/batches/Storage/Engine/KFileTransferExportEngine.php‎

Lines changed: 23 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -27,23 +27,24 @@ function __construct($data, $jobSubType) {
2727
function export()
2828
{
2929
$srcTempFile = null;
30-
31-
$srcFile = kFile::realPath($this->srcFile);
32-
$realPathRemote = strpos($srcFile, "http") !== false && kFile::checkFileExists($this->srcFile);
33-
if($realPathRemote || !KBatchBase::pollingFileExists($this->srcFile))
30+
31+
list($isRemote, $remoteUrl, $isDir) = kFile::resolveFilePath($this->srcFile);
32+
$realPathRemote = $isRemote && kFile::checkFileExists($this->srcFile);
33+
if( $realPathRemote || !KBatchBase::pollingFileExists($this->srcFile))
3434
{
35-
$externalUrl = $realPathRemote ? $srcFile : $this->data->externalUrl;
35+
$externalUrl = $isRemote ? $remoteUrl : $this->data->externalUrl;
3636
if($externalUrl == null)
3737
{
3838
throw new kTemporaryException("Source file {$this->srcFile} does not exist, and no external URL provided");
3939
}
40-
41-
$srcTempFile = $this->getAssetFile($externalUrl, $this->data->flavorAssetId);
40+
41+
$srcTempFile = kFile::fetchRemoteToLocal($this->srcFile, $externalUrl, $isDir,
42+
sys_get_temp_dir() . "/imports/", $this->data->flavorAssetId . "_" . basename($this->srcFile));
43+
4244
if(!$srcTempFile)
4345
{
4446
throw new kTemporaryException("Source file {$this->srcFile} does not exist");
4547
}
46-
4748
$this->srcFile = $srcTempFile;
4849
}
4950

@@ -93,6 +94,7 @@ function export()
9394
else if (kFile::isDir($this->srcFile))
9495
{
9596
$filesPaths = kFile::dirList($this->srcFile);
97+
9698
$destDir = $this->destFile;
9799
foreach ($filesPaths as $filePath)
98100
{
@@ -229,56 +231,29 @@ protected function addS3FieldsToStorageData($storageExportData, $externalStorage
229231
return $storageExportData;
230232
}
231233

232-
protected function getAssetFile($externalUrl, $uniqueDownloadId = null)
234+
protected function unlinkFileIfNeeded($tempFile)
233235
{
234-
// Needed arguments
235-
if($externalUrl === null)
236-
{
237-
KalturaLog::info("No external url provided,");
238-
return null;
239-
}
240-
241-
// Create the temporary file path
242-
$tempDirectoryPath = sys_get_temp_dir();
243-
if (!kFile::isDir($tempDirectoryPath))
244-
{
245-
kFile::fullMkfileDir($tempDirectoryPath, 0700, true);
246-
}
247-
248-
if(!$uniqueDownloadId)
236+
if (!$tempFile)
249237
{
250-
$uniqueDownloadId = uniqid(rand(),true);
238+
return;
251239
}
252-
253-
$filePath = $tempDirectoryPath . "/asset_$uniqueDownloadId";
254240

255-
// Retrieve the file
256-
$res = null;
257-
try
241+
if (kFile::isDir($tempFile))
258242
{
259-
$stat = KCurlWrapper::getDataFromFile($externalUrl, $filePath, null, true);
260-
261-
if ($stat)
243+
$dir = dir($tempFile);
244+
if (!$dir)
262245
{
263-
$res = $filePath;
264-
KalturaLog::info("Succeeded to retrieve asset content for assetId: [$assetId]");
246+
return null;
265247
}
266-
else
248+
while ($dirFile = $dir->read())
267249
{
268-
KalturaLog::info("Failed to retrieve asset content for assetId: [$assetId]");
250+
if ($dirFile != "." && $dirFile != "..")
251+
{
252+
unlink($tempFile . DIRECTORY_SEPARATOR . $dirFile);
253+
}
269254
}
270255
}
271-
catch(Exception $e)
272-
{
273-
KalturaLog::info("Can't serve asset id [$assetId] from [$externalUrl] " . $e->getMessage());
274-
}
275-
276-
return $res;
277-
}
278-
279-
protected function unlinkFileIfNeeded($tempFile)
280-
{
281-
if($tempFile)
256+
else
282257
{
283258
unlink($tempFile);
284259
}

‎infra/storage/kFile.class.php‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,15 @@ public static function listDir($path, $pathPrefix = '')
6161
}
6262

6363
// TODO - implement recursion
64-
static public function dirList($directory, $return_directory_as_prefix = true, $should_recurse = false)
64+
static public function dirList($directory, $return_directory_as_prefix = true)
6565
{
66+
if (kFile::isSharedPath($directory))
67+
{
68+
$sharedFsMgr = kSharedFileSystemMgr::getInstanceFromPath($directory);
69+
$pathPrefix = $return_directory_as_prefix ? $directory . "/" : '';
70+
return $sharedFsMgr->listFiles($directory . "/" , $pathPrefix, false, true);
71+
}
72+
6673
// create an array to hold directory list
6774
$results = array();
6875

@@ -74,10 +81,9 @@ static public function dirList($directory, $return_directory_as_prefix = true, $
7481
// keep going until all files in directory have been read
7582
while($file = readdir($handler))
7683
{
77-
7884
// if $file isn't this directory or its parent,
7985
// add it to the results array
80-
if($file != '.' && $file != '..')
86+
if($file != '.' && $file != '..' )
8187
{
8288
$results[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file;
8389
}

‎infra/storage/shared_file_system_managers/kNfsSharedFileSystemMgr.php‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ protected function doGetUploadMaxSize()
221221
return 2000000000;
222222
}
223223

224-
protected function doListFiles($filePath, $pathPrefix = '')
224+
protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false)
225225
{
226226
$fileList = array();
227227
$path = str_ireplace(DIRECTORY_SEPARATOR, '/', $filePath);
@@ -238,12 +238,15 @@ protected function doListFiles($filePath, $pathPrefix = '')
238238
if (is_dir($fullPath))
239239
{
240240
$tmpPrefix = $tmpPrefix.'/';
241-
$fileList[] = array($tmpPrefix, 'dir', self::fileSize($fullPath));
242-
$fileList = array_merge($fileList, self::listDir($fullPath, $tmpPrefix));
241+
$fileList[] = $fileNamesOnly ? $tmpPrefix : array($tmpPrefix, 'dir', self::fileSize($fullPath));
242+
if ($recursive)
243+
{
244+
$fileList = array_merge($fileList, self::doListFiles($fullPath, $tmpPrefix));
245+
}
243246
}
244247
else
245248
{
246-
$fileList[] = array($tmpPrefix, 'file', self::fileSize($fullPath));
249+
$fileList[] = $fileNamesOnly ? $tmpPrefix : array($tmpPrefix, 'file', self::fileSize($fullPath));
247250
}
248251
}
249252
}

‎infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr.php‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -578,52 +578,55 @@ public function completeMultiPartUpload($destFilePath, $uploadId, $parts)
578578
}
579579
return $result['Location'];
580580
}
581-
582-
protected function doListFiles($filePath, $pathPrefix = '')
581+
582+
protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false)
583583
{
584584
$dirList = array();
585585
list($bucket, $filePath) = $this->getBucketAndFilePath($filePath);
586-
586+
587587
try
588588
{
589589
$dirListObjectsRaw = $this->s3Client->getIterator('ListObjects', array(
590590
'Bucket' => $bucket,
591591
'Prefix' => $filePath
592592
));
593-
593+
594594
$originalFilePath = $bucket . '/' . $filePath . '/';
595595
foreach ($dirListObjectsRaw as $dirListObject)
596596
{
597597
$fullPath = '/' . $bucket . '/' . $dirListObject['Key'];
598598
$fileName = $pathPrefix.basename($fullPath);
599599
if($originalFilePath == $fullPath)
600600
continue;
601-
601+
602602
$fileType = "file";
603603
if($dirListObject['Size'] == 0 && substr_compare($fullPath, '/', -strlen('/')) === 0)
604604
{
605605
$fileType = 'dir';
606606
}
607-
607+
608608
if ($fileType == 'dir')
609609
{
610-
$dirList[] = array($fileName, 'dir', $dirListObject['Size']);
611-
$dirList = array_merge($dirList, self::doListFiles($fullPath, $pathPrefix));
610+
$dirList[] = $fileNamesOnly ? $fileName : array($fileName, 'dir', $dirListObject['Size']);
611+
if( $recursive)
612+
{
613+
$dirList = array_merge($dirList, self::doListFiles($fullPath, $pathPrefix, $fileNamesOnly));
614+
}
612615
}
613616
else
614617
{
615-
$dirList[] = array($fileName, 'file', $dirListObject['Size']);
618+
$dirList[] = $fileNamesOnly ? $fileName : array($fileName, 'file', $dirListObject['Size']);
616619
}
617620
}
618621
}
619622
catch ( Exception $e )
620623
{
621624
self::safeLog("Couldn't list file objects for remote path, [$filePath] from bucket [$bucket]: {$e->getMessage()}");
622625
}
623-
626+
624627
return $dirList;
625628
}
626-
629+
627630
protected function doGetMaximumPartsNum()
628631
{
629632
return self::MAX_PARTS_NUMBER;

‎infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr_V3_SDK.php‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -538,18 +538,18 @@ public function completeMultiPartUpload($destFilePath, $uploadId, $parts)
538538
}
539539
return $result['Location'];
540540
}
541-
542-
protected function doListFiles($filePath, $pathPrefix = '')
541+
542+
protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false)
543543
{
544544
$results = $this->s3Call('listObjects', null, $filePath);
545-
545+
546546
if(!$results)
547547
{
548548
return false;
549549
}
550550
return $results;
551551
}
552-
552+
553553
protected function doGetMaximumPartsNum()
554554
{
555555
return self::MAX_PARTS_NUMBER;

‎infra/storage/shared_file_system_managers/kSharedFileSystemMgr.php‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -251,15 +251,17 @@ abstract protected function doGetUploadMinimumSize();
251251
* @return int
252252
*/
253253
abstract protected function doGetUploadMaxSize();
254-
254+
255255
/**
256256
* returns list of files under given file path
257257
*
258258
* @param $filePath file path to list dir content for
259-
*
259+
* @param string $pathPrefix
260+
* @param bool $recursive
261+
* @param bool $fileNamesOnly
260262
* @return array
261263
*/
262-
abstract protected function doListFiles($filePath, $pathPrefix = '');
264+
abstract protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false);
263265

264266
/**
265267
* returns true/false if the givven file path exists and is a regular file
@@ -517,10 +519,10 @@ function copySingleFile($from, $to, $deleteSrc)
517519
return $this->doRename($from, $to);
518520
}
519521

520-
public function listFiles($filePath, $pathPrefix = '')
522+
public function listFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false)
521523
{
522524
$filePath = kFileBase::fixPath($filePath);
523-
return $this->doListFiles($filePath, $pathPrefix);
525+
return $this->doListFiles($filePath, $pathPrefix, $recursive, $fileNamesOnly);
524526
}
525527

526528
public function isFile($filePath)

0 commit comments

Comments
 (0)