-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
7 changed files
with
275 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
name: CI-CD | ||
|
||
on: [push] | ||
|
||
jobs: | ||
ci: | ||
runs-on: ubuntu-latest | ||
env: | ||
APP_ENV: testing | ||
BROADCAST_DRIVER: log | ||
services: | ||
database: | ||
image: ankane/pgvector:latest | ||
env: | ||
POSTGRES_PASSWORD: password | ||
POSTGRES_USER: root | ||
POSTGRES_DB: testing | ||
ports: | ||
- 5432:5432 | ||
options: >- | ||
--health-cmd pg_isready | ||
--health-interval 10s | ||
--health-timeout 5s | ||
--health-retries 5 | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
php-versions: ["8.1"] | ||
steps: | ||
- uses: actions/checkout@v2 | ||
|
||
- name: Setup PHP, with composer and extensions | ||
uses: shivammathur/setup-php@v2 #https://github.com/shivammathur/setup-php | ||
with: | ||
php-version: ${{ matrix.php-versions }} | ||
tools: phpmd | ||
extensions: mbstring, dom, fileinfo, grpc | ||
coverage: xdebug #optional | ||
|
||
- name: Get composer cache directory | ||
id: composer-cache | ||
run: echo "::set-output name=dir::$(composer config cache-files-dir)" | ||
|
||
- name: Cache composer dependencies | ||
uses: actions/cache@v2 | ||
with: | ||
path: ${{ steps.composer-cache.outputs.dir }} | ||
# Use composer.json for key, if composer.lock is not committed. | ||
# key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.json') }} | ||
key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} | ||
restore-keys: ${{ runner.os }}-composer- | ||
|
||
- name: Install Composer dependencies | ||
run: | | ||
composer install --no-progress --prefer-dist --optimize-autoloader | ||
- name: PHP Code Style (phpcs) | ||
run: | | ||
composer fix | ||
- name: PHP Code Style (phpcs) | ||
run: | | ||
composer stan | ||
- name: Setup application | ||
run: | | ||
cp .env.github .env | ||
php artisan storage:link | ||
- name: Test with phpunit | ||
run: | | ||
npm install && npm run build | ||
XDEBUG_MODE=coverage php artisan test --coverage --min=50 | ||
cd: | ||
runs-on: ubuntu-latest | ||
needs: ci | ||
if: github.ref == 'refs/heads/main' | ||
steps: | ||
- name: Checkout code | ||
uses: actions/checkout@v2 | ||
- name: Setup PHP | ||
uses: shivammathur/setup-php@v2 | ||
with: | ||
php-version: 8.1 | ||
tools: composer:v2 | ||
coverage: none | ||
- name: Require Vapor CLI | ||
run: composer global require laravel/vapor-cli | ||
- name: Install Project Dependencies | ||
run: | | ||
curl https://envoyer.io/deploy/${{ secrets.ENVOYER }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
<?php | ||
|
||
namespace App\Domains\Collections; | ||
|
||
use App\Helpers\EnumHelperTrait; | ||
|
||
enum CollectionStatusEnum: string | ||
{ | ||
use EnumHelperTrait; | ||
|
||
case PENDING = 'pending'; | ||
case PROCESSING = 'processing'; | ||
case PROCESSED = 'processed'; | ||
case FAILED = 'failed'; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
<?php | ||
|
||
namespace App\Events; | ||
|
||
use App\Domains\Collections\CollectionStatusEnum; | ||
use App\Models\Collection; | ||
use Illuminate\Broadcasting\InteractsWithSockets; | ||
use Illuminate\Broadcasting\PrivateChannel; | ||
use Illuminate\Contracts\Broadcasting\ShouldBroadcast; | ||
use Illuminate\Foundation\Events\Dispatchable; | ||
use Illuminate\Queue\SerializesModels; | ||
|
||
class CollectionStatusEvent implements ShouldBroadcast | ||
{ | ||
use Dispatchable, InteractsWithSockets, SerializesModels; | ||
|
||
/** | ||
* Create a new event instance. | ||
*/ | ||
public function __construct(public Collection $collection, public CollectionStatusEnum $status) | ||
{ | ||
// | ||
} | ||
|
||
/** | ||
* Get the channels the event should broadcast on. | ||
* | ||
* @return array<int, \Illuminate\Broadcasting\Channel> | ||
*/ | ||
public function broadcastOn(): array | ||
{ | ||
return [ | ||
new PrivateChannel('collection.'.$this->collection->id), | ||
]; | ||
} | ||
|
||
/** | ||
* The event's broadcast name. | ||
*/ | ||
public function broadcastAs(): string | ||
{ | ||
return 'status'; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
<?php | ||
|
||
namespace App\Jobs; | ||
|
||
use App\Domains\Documents\Transformers\PdfTransformer; | ||
use App\Models\Document; | ||
use Illuminate\Bus\Batchable; | ||
use Illuminate\Bus\Queueable; | ||
use Illuminate\Contracts\Queue\ShouldQueue; | ||
use Illuminate\Foundation\Bus\Dispatchable; | ||
use Illuminate\Queue\InteractsWithQueue; | ||
use Illuminate\Queue\SerializesModels; | ||
|
||
class ParsePdfFileJob implements ShouldQueue | ||
{ | ||
use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels; | ||
|
||
/** | ||
* Create a new job instance. | ||
*/ | ||
public function __construct(public Document $document) | ||
{ | ||
// | ||
} | ||
|
||
/** | ||
* Execute the job. | ||
*/ | ||
public function handle(): void | ||
{ | ||
if ($this->batch()->cancelled()) { | ||
// Determine if the batch has been cancelled... | ||
|
||
return; | ||
} | ||
|
||
(new PdfTransformer())->handle($this->document); | ||
|
||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
<?php | ||
|
||
namespace App\Jobs; | ||
|
||
use App\Domains\Collections\CollectionStatusEnum; | ||
use App\Events\CollectionStatusEvent; | ||
use App\Models\Document; | ||
use Illuminate\Bus\Batch; | ||
use Illuminate\Bus\Queueable; | ||
use Illuminate\Contracts\Queue\ShouldQueue; | ||
use Illuminate\Foundation\Bus\Dispatchable; | ||
use Illuminate\Queue\InteractsWithQueue; | ||
use Illuminate\Queue\SerializesModels; | ||
use Illuminate\Support\Facades\Bus; | ||
|
||
class ProcessFileJob implements ShouldQueue | ||
{ | ||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; | ||
|
||
/** | ||
* Create a new job instance. | ||
*/ | ||
public function __construct(public Document $document) | ||
{ | ||
// | ||
} | ||
|
||
/** | ||
* Execute the job. | ||
*/ | ||
public function handle(): void | ||
{ | ||
//This will trigger a batch job chain that will do the following for all the document pages | ||
// it will save this on a Collections Watchers so we can see it in the UI. | ||
//1) Chunk up the data | ||
//2) vectorize the data | ||
//3) Summarize the data | ||
//4) Tag the data | ||
|
||
$batch = Bus::batch([ | ||
new ParsePdfFileJob($this->document), | ||
//new VectorizeDataJob($this->document), | ||
//new SummarizeDataJob($this->document), | ||
//new TagDataJob($this->document), | ||
//then mark it all as done and notify the ui | ||
]) | ||
->name('OptOutRequests') | ||
->finally(function (Batch $batch) { | ||
/** | ||
* @TODO | ||
* make a job that does that and also | ||
* closes up the batch on the run watcher | ||
*/ | ||
CollectionStatusEvent::dispatch( | ||
$this->document->collection, | ||
CollectionStatusEnum::PROCESSED); | ||
}) | ||
->allowFailures() | ||
->dispatch(); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
<?php | ||
|
||
namespace Tests\Feature\Jobs; | ||
|
||
use Tests\TestCase; | ||
|
||
class ChunkFileJobTest extends TestCase | ||
{ | ||
/** | ||
* A basic feature test example. | ||
*/ | ||
public function test_example(): void | ||
{ | ||
$response = $this->get('/'); | ||
|
||
$response->assertStatus(200); | ||
} | ||
} |