Skip to content

Commit a84fb39

Browse files
lastlinklastlink
committed
v3 follow up
1 parent 4a7891e commit a84fb39

File tree

3 files changed

+140
-25
lines changed

3 files changed

+140
-25
lines changed

‎lib/Command/Generate.php

Lines changed: 59 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
use OCP\Files\StorageInvalidException;
3939
use OCP\Files\StorageNotAvailableException;
4040
use OCP\IConfig;
41+
use OCP\IDBConnection;
4142
use OCP\IPreview;
4243
use OCP\IUser;
4344
use OCP\IUserManager;
@@ -53,19 +54,35 @@ class Generate extends Command {
5354
/* @return array{width: int, height: int, crop: bool} */
5455
protected array $specifications;
5556

57+
/** @var ?GlobalStoragesService */
5658
protected ?GlobalStoragesService $globalService;
59+
/** @var IUserManager */
5760
protected IUserManager $userManager;
61+
62+
/** @var IRootFolder */
5863
protected IRootFolder $rootFolder;
64+
65+
/** @var IPreview */
5966
protected IPreview $previewGenerator;
67+
68+
/** @var IConfig */
6069
protected IConfig $config;
70+
71+
/** @var IDBConnection */
72+
protected $connection;
73+
74+
/** @var OutputInterface */
6175
protected OutputInterface $output;
76+
77+
/** @var IManager */
6278
protected IManager $encryptionManager;
6379
protected SizeHelper $sizeHelper;
6480

6581
public function __construct(IRootFolder $rootFolder,
6682
IUserManager $userManager,
6783
IPreview $previewGenerator,
6884
IConfig $config,
85+
IDBConnection $connection,
6986
IManager $encryptionManager,
7087
ContainerInterface $container,
7188
SizeHelper $sizeHelper) {
@@ -75,6 +92,7 @@ public function __construct(IRootFolder $rootFolder,
7592
$this->rootFolder = $rootFolder;
7693
$this->previewGenerator = $previewGenerator;
7794
$this->config = $config;
95+
$this->connection = $connection;
7896
$this->encryptionManager = $encryptionManager;
7997
$this->sizeHelper = $sizeHelper;
8098

@@ -176,7 +194,7 @@ private function generatePathPreviews(IUser $user, string $path): void {
176194
}
177195
$pathFolder = $userFolder->get($relativePath);
178196
$noPreviewMountPaths = $this->getNoPreviewMountPaths($user);
179-
$this->parseFolder($pathFolder, $noPreviewMountPaths);
197+
$this->parseFolder($pathFolder, $noPreviewMountPaths, $user);
180198
}
181199

182200
private function generateUserPreviews(IUser $user): void {
@@ -185,10 +203,10 @@ private function generateUserPreviews(IUser $user): void {
185203

186204
$userFolder = $this->rootFolder->getUserFolder($user->getUID());
187205
$noPreviewMountPaths = $this->getNoPreviewMountPaths($user);
188-
$this->parseFolder($userFolder, $noPreviewMountPaths);
206+
$this->parseFolder($userFolder, $noPreviewMountPaths, $user);
189207
}
190208

191-
private function parseFolder(Folder $folder, array $noPreviewMountPaths): void {
209+
private function parseFolder(Folder $folder, array $noPreviewMountPaths, IUser $user): void {
192210
try {
193211
$folderPath = $folder->getPath();
194212

@@ -206,8 +224,44 @@ private function parseFolder(Folder $folder, array $noPreviewMountPaths): void {
206224
foreach ($nodes as $node) {
207225
if ($node instanceof Folder) {
208226
$this->parseFolder($node, $noPreviewMountPaths);
209-
} elseif ($node instanceof File) {
210-
$this->parseFile($node);
227+
} else if ($node instanceof File) {
228+
$is_locked = false;
229+
$qb = $this->connection->getQueryBuilder();
230+
$row = $qb->select('*')
231+
->from('preview_generation')
232+
->where($qb->expr()->eq('file_id', $qb->createNamedParameter($node->getId())))
233+
->setMaxResults(1)
234+
->execute()
235+
->fetch();
236+
if ($row !== false) {
237+
if ($row['locked'] == 1) {
238+
// already being processed
239+
$is_locked = true;
240+
} else {
241+
$qb->update('preview_generation')
242+
->where($qb->expr()->eq('file_id', $qb->createNamedParameter($node->getId())))
243+
->set('locked', $qb->createNamedParameter(true))
244+
->execute();
245+
}
246+
} else {
247+
$qb->insert('preview_generation')
248+
->values([
249+
'uid' => $qb->createNamedParameter($user->getUID()),
250+
'file_id' => $qb->createNamedParameter($node->getId()),
251+
'locked' => $qb->createNamedParameter(true),
252+
])
253+
->execute();
254+
}
255+
256+
if ($is_locked === false) {
257+
try {
258+
$this->parseFile($node);
259+
} finally {
260+
$qb->delete('preview_generation')
261+
->where($qb->expr()->eq('file_id', $qb->createNamedParameter($node->getId())))
262+
->execute();
263+
}
264+
}
211265
}
212266
}
213267
} catch (StorageNotAvailableException|StorageInvalidException $e) {

‎lib/Command/PreGenerate.php

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@ class PreGenerate extends Command {
5656
protected OutputInterface $output;
5757
protected IManager $encryptionManager;
5858
protected ITimeFactory $time;
59-
protected NoMediaService $noMediaService;
60-
protected SizeHelper $sizeHelper;
6159

6260
/**
6361
* @param string $appName
@@ -96,21 +94,23 @@ public function __construct(string $appName,
9694
protected function configure(): void {
9795
$this
9896
->setName('preview:pre-generate')
99-
->setDescription('Pre generate only images that have been added or changed since the last regular run');
97+
->setDescription('Pre generate previews');
10098
}
10199

102100
protected function execute(InputInterface $input, OutputInterface $output): int {
103101
if ($this->encryptionManager->isEnabled()) {
104102
$output->writeln('Encryption is enabled. Aborted.');
105103
return 1;
106104
}
107-
105+
/*
106+
this locks it to only be run once
108107
if ($this->checkAlreadyRunning()) {
109108
$output->writeln('Command is already running.');
110109
return 2;
111110
}
112111
113112
$this->setPID();
113+
*/
114114

115115
// Set timestamp output
116116
$formatter = new TimestampFormatter($this->config, $output->getFormatter());
@@ -123,37 +123,41 @@ protected function execute(InputInterface $input, OutputInterface $output): int
123123
}
124124
$this->startProcessing();
125125

126+
/*
126127
$this->clearPID();
128+
*/
127129

128130
return 0;
129131
}
130132

131133
private function startProcessing(): void {
134+
// random sleep between 0 and 50ms to avoid collision between 2 processes
135+
usleep(rand(0,50000));
136+
132137
while (true) {
133138
$qb = $this->connection->getQueryBuilder();
134-
$qb->select('*')
139+
$row = $qb->select('*')
135140
->from('preview_generation')
136141
->orderBy('id')
137-
->setMaxResults(1000);
138-
$cursor = $qb->execute();
139-
$rows = $cursor->fetchAll();
140-
$cursor->closeCursor();
142+
->where($qb->expr()->eq('locked', $qb->createNamedParameter(false)))
143+
->setMaxResults(1)
144+
->execute()
145+
->fetch();
141146

142-
if ($rows === []) {
147+
if ($row === false) {
143148
break;
144149
}
145150

146-
foreach ($rows as $row) {
147-
/*
148-
* First delete the row so that if preview generation fails for some reason
149-
* the next run can just continue
150-
*/
151-
$qb = $this->connection->getQueryBuilder();
152-
$qb->delete('preview_generation')
153-
->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])));
154-
$qb->execute();
155-
151+
$qb->update('preview_generation')
152+
->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])))
153+
->set('locked', $qb->createNamedParameter(true))
154+
->execute();
155+
try {
156156
$this->processRow($row);
157+
} finally {
158+
$qb->delete('preview_generation')
159+
->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])))
160+
->execute();
157161
}
158162
}
159163
}
@@ -209,6 +213,10 @@ private function processFile(File $file): void {
209213
$this->previewGenerator->generatePreviews($file, $this->specifications);
210214
} catch (NotFoundException $e) {
211215
// Maybe log that previews could not be generated?
216+
if ($this->output->getVerbosity() > OutputInterface::VERBOSITY_VERBOSE) {
217+
$error = $e->getMessage();
218+
$this->output->writeln("<error>${error} " . $file->getPath() . " not found.</error>");
219+
}
212220
} catch (\InvalidArgumentException|GenericFileException $e) {
213221
$class = $e::class;
214222
$error = $e->getMessage();

‎lib/Migration/Version020200Date20190608205303.php

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
<?php
2+
declare(strict_types=1);
3+
/**
4+
* @copyright Copyleft (c) 2019, Ignacio Nunez <[email protected]>
5+
*
6+
* @author Ignacio Nunez <[email protected]>
7+
*
8+
* @license GNU AGPL version 3 or any later version
9+
*
10+
* This program is free software: you can redistribute it and/or modify
11+
* it under the terms of the GNU Affero General Public License as
12+
* published by the Free Software Foundation, either version 3 of the
13+
* License, or (at your option) any later version.
14+
*
15+
* This program is distributed in the hope that it will be useful,
16+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
17+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18+
* GNU Affero General Public License for more details.
19+
*
20+
* You should have received a copy of the GNU Affero General Public License
21+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
22+
*
23+
*/
24+
25+
namespace OCA\PreviewGenerator\Migration;
26+
27+
use OCP\DB\ISchemaWrapper;
28+
use OCP\Migration\SimpleMigrationStep;
29+
use OCP\Migration\IOutput;
30+
use Doctrine\DBAL\Types\Type;
31+
32+
class Version020200Date20190608205303 extends SimpleMigrationStep {
33+
34+
/**
35+
* @param IOutput $output
36+
* @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
37+
* @param array $options
38+
* @return null|ISchemaWrapper
39+
*/
40+
public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options) {
41+
/** @var ISchemaWrapper $schema */
42+
$schema = $schemaClosure();
43+
$table = $schema->getTable('preview_generation');
44+
45+
if (!$table->hasColumn('locked')) {
46+
$table->addColumn('locked', Type::BOOLEAN, [
47+
'notnull' => true,
48+
'default' => 0,
49+
]);
50+
}
51+
return $schema;
52+
}
53+
}

0 commit comments

Comments
 (0)