diff --git a/lib/Command/Generate.php b/lib/Command/Generate.php index 735ceb9..aef2e81 100644 --- a/lib/Command/Generate.php +++ b/lib/Command/Generate.php @@ -38,6 +38,7 @@ use OCP\Files\StorageInvalidException; use OCP\Files\StorageNotAvailableException; use OCP\IConfig; +use OCP\IDBConnection; use OCP\IPreview; use OCP\IUser; use OCP\IUserManager; @@ -53,12 +54,27 @@ class Generate extends Command { /* @return array{width: int, height: int, crop: bool} */ protected array $specifications; + /** @var ?GlobalStoragesService */ protected ?GlobalStoragesService $globalService; + /** @var IUserManager */ protected IUserManager $userManager; + + /** @var IRootFolder */ protected IRootFolder $rootFolder; + + /** @var IPreview */ protected IPreview $previewGenerator; + + /** @var IConfig */ protected IConfig $config; + + /** @var IDBConnection */ + protected $connection; + + /** @var OutputInterface */ protected OutputInterface $output; + + /** @var IManager */ protected IManager $encryptionManager; protected SizeHelper $sizeHelper; @@ -66,6 +82,7 @@ public function __construct(IRootFolder $rootFolder, IUserManager $userManager, IPreview $previewGenerator, IConfig $config, + IDBConnection $connection, IManager $encryptionManager, ContainerInterface $container, SizeHelper $sizeHelper) { @@ -75,6 +92,7 @@ public function __construct(IRootFolder $rootFolder, $this->rootFolder = $rootFolder; $this->previewGenerator = $previewGenerator; $this->config = $config; + $this->connection = $connection; $this->encryptionManager = $encryptionManager; $this->sizeHelper = $sizeHelper; @@ -176,7 +194,7 @@ private function generatePathPreviews(IUser $user, string $path): void { } $pathFolder = $userFolder->get($relativePath); $noPreviewMountPaths = $this->getNoPreviewMountPaths($user); - $this->parseFolder($pathFolder, $noPreviewMountPaths); + $this->parseFolder($pathFolder, $noPreviewMountPaths, $user); } private function generateUserPreviews(IUser $user): void { @@ -185,10 +203,10 @@ private function generateUserPreviews(IUser $user): void { $userFolder = $this->rootFolder->getUserFolder($user->getUID()); $noPreviewMountPaths = $this->getNoPreviewMountPaths($user); - $this->parseFolder($userFolder, $noPreviewMountPaths); + $this->parseFolder($userFolder, $noPreviewMountPaths, $user); } - private function parseFolder(Folder $folder, array $noPreviewMountPaths): void { + private function parseFolder(Folder $folder, array $noPreviewMountPaths, IUser $user): void { try { $folderPath = $folder->getPath(); @@ -206,8 +224,44 @@ private function parseFolder(Folder $folder, array $noPreviewMountPaths): void { foreach ($nodes as $node) { if ($node instanceof Folder) { $this->parseFolder($node, $noPreviewMountPaths); - } elseif ($node instanceof File) { - $this->parseFile($node); + } else if ($node instanceof File) { + $is_locked = false; + $qb = $this->connection->getQueryBuilder(); + $row = $qb->select('*') + ->from('preview_generation') + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($node->getId()))) + ->setMaxResults(1) + ->execute() + ->fetch(); + if ($row !== false) { + if ($row['locked'] == 1) { + // already being processed + $is_locked = true; + } else { + $qb->update('preview_generation') + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($node->getId()))) + ->set('locked', $qb->createNamedParameter(true)) + ->execute(); + } + } else { + $qb->insert('preview_generation') + ->values([ + 'uid' => $qb->createNamedParameter($user->getUID()), + 'file_id' => $qb->createNamedParameter($node->getId()), + 'locked' => $qb->createNamedParameter(true), + ]) + ->execute(); + } + + if ($is_locked === false) { + try { + $this->parseFile($node); + } finally { + $qb->delete('preview_generation') + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($node->getId()))) + ->execute(); + } + } } } } catch (StorageNotAvailableException|StorageInvalidException $e) { diff --git a/lib/Command/PreGenerate.php b/lib/Command/PreGenerate.php index b7d0380..e42fa79 100644 --- a/lib/Command/PreGenerate.php +++ b/lib/Command/PreGenerate.php @@ -56,8 +56,6 @@ class PreGenerate extends Command { protected OutputInterface $output; protected IManager $encryptionManager; protected ITimeFactory $time; - protected NoMediaService $noMediaService; - protected SizeHelper $sizeHelper; /** * @param string $appName @@ -96,7 +94,7 @@ public function __construct(string $appName, protected function configure(): void { $this ->setName('preview:pre-generate') - ->setDescription('Pre generate only images that have been added or changed since the last regular run'); + ->setDescription('Pre generate previews'); } protected function execute(InputInterface $input, OutputInterface $output): int { @@ -104,13 +102,15 @@ protected function execute(InputInterface $input, OutputInterface $output): int $output->writeln('Encryption is enabled. Aborted.'); return 1; } - + /* + this locks it to only be run once if ($this->checkAlreadyRunning()) { $output->writeln('Command is already running.'); return 2; } $this->setPID(); + */ // Set timestamp output $formatter = new TimestampFormatter($this->config, $output->getFormatter()); @@ -123,37 +123,41 @@ protected function execute(InputInterface $input, OutputInterface $output): int } $this->startProcessing(); + /* $this->clearPID(); + */ return 0; } private function startProcessing(): void { + // random sleep between 0 and 50ms to avoid collision between 2 processes + usleep(rand(0,50000)); + while (true) { $qb = $this->connection->getQueryBuilder(); - $qb->select('*') + $row = $qb->select('*') ->from('preview_generation') ->orderBy('id') - ->setMaxResults(1000); - $cursor = $qb->execute(); - $rows = $cursor->fetchAll(); - $cursor->closeCursor(); + ->where($qb->expr()->eq('locked', $qb->createNamedParameter(false))) + ->setMaxResults(1) + ->execute() + ->fetch(); - if ($rows === []) { + if ($row === false) { break; } - foreach ($rows as $row) { - /* - * First delete the row so that if preview generation fails for some reason - * the next run can just continue - */ - $qb = $this->connection->getQueryBuilder(); - $qb->delete('preview_generation') - ->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id']))); - $qb->execute(); - + $qb->update('preview_generation') + ->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id']))) + ->set('locked', $qb->createNamedParameter(true)) + ->execute(); + try { $this->processRow($row); + } finally { + $qb->delete('preview_generation') + ->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id']))) + ->execute(); } } } @@ -209,6 +213,10 @@ private function processFile(File $file): void { $this->previewGenerator->generatePreviews($file, $this->specifications); } catch (NotFoundException $e) { // Maybe log that previews could not be generated? + if ($this->output->getVerbosity() > OutputInterface::VERBOSITY_VERBOSE) { + $error = $e->getMessage(); + $this->output->writeln("${error} " . $file->getPath() . " not found."); + } } catch (\InvalidArgumentException|GenericFileException $e) { $class = $e::class; $error = $e->getMessage(); diff --git a/lib/Migration/Version020200Date20190608205303.php b/lib/Migration/Version020200Date20190608205303.php new file mode 100644 index 0000000..cc560d2 --- /dev/null +++ b/lib/Migration/Version020200Date20190608205303.php @@ -0,0 +1,53 @@ + + * + * @author Ignacio Nunez + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\PreviewGenerator\Migration; + +use OCP\DB\ISchemaWrapper; +use OCP\Migration\SimpleMigrationStep; +use OCP\Migration\IOutput; +use Doctrine\DBAL\Types\Type; + +class Version020200Date20190608205303 extends SimpleMigrationStep { + + /** + * @param IOutput $output + * @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options) { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + $table = $schema->getTable('preview_generation'); + + if (!$table->hasColumn('locked')) { + $table->addColumn('locked', Type::BOOLEAN, [ + 'notnull' => true, + 'default' => 0, + ]); + } + return $schema; + } +} \ No newline at end of file