First commit
This commit is contained in:
Saufi
67
app/Jobs/LogChatInteractionJob.php
Normal file
67
app/Jobs/LogChatInteractionJob.php
Normal file
@@ -0,0 +1,67 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\ChatLog;
|
||||
use App\Models\ChatSession;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
|
||||
/**
|
||||
* LogChatInteractionJob
|
||||
*
|
||||
* Log pertanyaan + jawapan chatbot secara asynchronous.
|
||||
* Hantar ke queue supaya response chatbot tidak ditangguh oleh operasi DB.
|
||||
*/
|
||||
class LogChatInteractionJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public int $tries = 3;
|
||||
public int $timeout = 30;
|
||||
|
||||
public function __construct(
|
||||
private readonly string $sessionToken,
|
||||
private readonly ?int $userId,
|
||||
private readonly ?int $categoryId,
|
||||
private readonly string $question,
|
||||
private readonly string $answer,
|
||||
private readonly array $sources,
|
||||
private readonly array $contextChunks,
|
||||
private readonly string $modelUsed,
|
||||
private readonly ?int $tokensUsed,
|
||||
private readonly float $responseTime,
|
||||
private readonly bool $hasAnswer,
|
||||
) {
|
||||
$this->onQueue(config('knowledgebase.queue.chat_log', 'default'));
|
||||
}
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
$session = ChatSession::where('session_token', $this->sessionToken)->first();
|
||||
|
||||
if (!$session) {
|
||||
return;
|
||||
}
|
||||
|
||||
ChatLog::create([
|
||||
'chat_session_id' => $session->id,
|
||||
'user_id' => $this->userId,
|
||||
'category_id' => $this->categoryId,
|
||||
'question' => $this->question,
|
||||
'answer' => $this->answer,
|
||||
'sources_used' => $this->sources,
|
||||
'context_chunks' => $this->contextChunks,
|
||||
'model_used' => $this->modelUsed,
|
||||
'tokens_used' => $this->tokensUsed,
|
||||
'response_time' => $this->responseTime,
|
||||
'has_answer' => $this->hasAnswer,
|
||||
'is_flagged' => false,
|
||||
]);
|
||||
|
||||
$session->update(['last_activity_at' => now()]);
|
||||
}
|
||||
}
|
||||
65
app/Jobs/ProcessUploadedDocumentJob.php
Normal file
65
app/Jobs/ProcessUploadedDocumentJob.php
Normal file
@@ -0,0 +1,65 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\DocumentVersion;
|
||||
use App\Services\KnowledgeBase\IngestionService;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* ProcessUploadedDocumentJob
|
||||
*
|
||||
* Job utama untuk proses dokumen selepas upload.
|
||||
* Melaksanakan: extract PDF → chunk → embed → sync Qdrant
|
||||
*
|
||||
* Dihantar ke queue supaya upload tidak timeout.
|
||||
* Retry: 2 kali jika gagal, dengan 60s delay.
|
||||
*/
|
||||
class ProcessUploadedDocumentJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public int $tries = 2;
|
||||
public int $backoff = 60; // saat antara retry
|
||||
public int $timeout = 600; // 10 minit — PDF besar mungkin ambil masa
|
||||
|
||||
public function __construct(
|
||||
private readonly int $documentVersionId
|
||||
) {
|
||||
$this->onQueue(config('knowledgebase.queue.ingestion', 'default'));
|
||||
}
|
||||
|
||||
public function handle(IngestionService $ingestionService): void
|
||||
{
|
||||
$version = DocumentVersion::with(['document.category'])->findOrFail($this->documentVersionId);
|
||||
|
||||
Log::info("ProcessUploadedDocumentJob mula untuk version {$this->documentVersionId}");
|
||||
|
||||
$ingestionService->processDocumentVersion($version);
|
||||
|
||||
Log::info("ProcessUploadedDocumentJob selesai untuk version {$this->documentVersionId}");
|
||||
}
|
||||
|
||||
public function failed(Throwable $exception): void
|
||||
{
|
||||
Log::error("ProcessUploadedDocumentJob GAGAL untuk version {$this->documentVersionId}", [
|
||||
'error' => $exception->getMessage(),
|
||||
'trace' => $exception->getTraceAsString(),
|
||||
]);
|
||||
|
||||
// Kemaskini status ke failed dalam database
|
||||
$version = DocumentVersion::find($this->documentVersionId);
|
||||
if ($version) {
|
||||
$version->updateStatus(
|
||||
DocumentVersion::STATUS_FAILED,
|
||||
'Job gagal: ' . $exception->getMessage()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
171
app/Jobs/ReindexChunkJob.php
Normal file
171
app/Jobs/ReindexChunkJob.php
Normal file
@@ -0,0 +1,171 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\DocumentChunk;
|
||||
use App\Models\ProcessingLog;
|
||||
use App\Services\Ollama\OllamaService;
|
||||
use App\Services\Qdrant\QdrantService;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Queue\Queueable;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Str;
|
||||
use RuntimeException;
|
||||
|
||||
/**
|
||||
* ReindexChunkJob
|
||||
*
|
||||
* Reindex satu chunk sahaja:
|
||||
* 1. Embed semula getEmbeddableText() (final_text > cleaned_text > content)
|
||||
* 2. Upsert point ke Qdrant (guna semula qdrant_point_id lama jika ada)
|
||||
* 3. Kemaskini chunk: markAsEmbedded(), needs_reindex=false
|
||||
*
|
||||
* Berbeza dari ReindexDocumentJob yang reindex SELURUH dokumen.
|
||||
* Job ini digunakan untuk:
|
||||
* - Edit final_text oleh admin
|
||||
* - Include semula chunk yang excluded
|
||||
* - Manual trigger reindex
|
||||
* - Child chunks hasil split (perlu embed buat pertama kali)
|
||||
*/
|
||||
class ReindexChunkJob implements ShouldQueue
|
||||
{
|
||||
use Queueable;
|
||||
|
||||
public int $tries = 3;
|
||||
public int $backoff = 30; // Tunggu 30s sebelum retry
|
||||
public int $timeout = 120; // 2 minit maksimum per chunk
|
||||
|
||||
public function __construct(
|
||||
public readonly int $chunkId,
|
||||
) {
|
||||
$this->onQueue(config('knowledgebase.queue.ingestion', 'default'));
|
||||
}
|
||||
|
||||
public function handle(OllamaService $ollama, QdrantService $qdrant): void
|
||||
{
|
||||
// Load chunk dengan semua relasi yang diperlukan untuk toQdrantPayload()
|
||||
$chunk = DocumentChunk::with(['document.category', 'documentVersion'])
|
||||
->find($this->chunkId);
|
||||
|
||||
if (! $chunk) {
|
||||
Log::warning("ReindexChunkJob: Chunk #{$this->chunkId} tidak dijumpai. Job dilangkau.");
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Guard: Jangan reindex chunk yang tidak sepatutnya ──────────────
|
||||
|
||||
if ($chunk->isSuperseded()) {
|
||||
Log::info("ReindexChunkJob: Chunk #{$this->chunkId} adalah superseded. Job dilangkau.", [
|
||||
'chunk_index' => $chunk->chunk_index,
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
if ($chunk->exclude_from_index) {
|
||||
Log::info("ReindexChunkJob: Chunk #{$this->chunkId} dikecualikan dari index. Job dilangkau.", [
|
||||
'chunk_index' => $chunk->chunk_index,
|
||||
'chunk_status' => $chunk->chunk_status,
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Ambil teks untuk embedding ─────────────────────────────────────
|
||||
|
||||
$textToEmbed = $chunk->getEmbeddableText();
|
||||
|
||||
if (empty(trim($textToEmbed))) {
|
||||
$chunk->update(['chunk_status' => DocumentChunk::STATUS_FAILED_EMBEDDING]);
|
||||
|
||||
Log::error("ReindexChunkJob: Chunk #{$this->chunkId} mempunyai teks kosong.", [
|
||||
'has_final_text' => ! is_null($chunk->final_text),
|
||||
'has_cleaned_text' => ! is_null($chunk->cleaned_text),
|
||||
'content_length' => mb_strlen($chunk->content),
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Log: mula proses ───────────────────────────────────────────────
|
||||
|
||||
ProcessingLog::record(
|
||||
DocumentChunk::class,
|
||||
$chunk->id,
|
||||
ProcessingLog::STAGE_EMBED,
|
||||
ProcessingLog::STATUS_STARTED,
|
||||
null,
|
||||
[
|
||||
'chunk_index' => $chunk->chunk_index,
|
||||
'text_source' => $chunk->final_text ? 'final_text'
|
||||
: ($chunk->cleaned_text ? 'cleaned_text' : 'content'),
|
||||
'text_length' => mb_strlen($textToEmbed),
|
||||
'is_reindex' => true,
|
||||
]
|
||||
);
|
||||
|
||||
try {
|
||||
// ── Embed teks ─────────────────────────────────────────────────
|
||||
$vector = $ollama->embed($textToEmbed);
|
||||
|
||||
// ── Tentukan point ID ──────────────────────────────────────────
|
||||
// Guna semula qdrant_point_id lama jika ada → upsert akan overwrite
|
||||
// Ini mengelakkan "ghost points" yang tidak dirujuk oleh mana-mana chunk
|
||||
$pointId = $chunk->qdrant_point_id ?? (string) Str::uuid();
|
||||
|
||||
// ── Upsert ke Qdrant ───────────────────────────────────────────
|
||||
$qdrant->ensureCollectionExists();
|
||||
$qdrant->upsertPoint($pointId, $vector, $chunk->toQdrantPayload());
|
||||
|
||||
// ── Kemaskini chunk ────────────────────────────────────────────
|
||||
$chunk->markAsEmbedded($pointId);
|
||||
|
||||
ProcessingLog::record(
|
||||
DocumentChunk::class,
|
||||
$chunk->id,
|
||||
ProcessingLog::STAGE_QDRANT,
|
||||
ProcessingLog::STATUS_COMPLETED,
|
||||
null,
|
||||
[
|
||||
'point_id' => $pointId,
|
||||
'is_reindex' => true,
|
||||
]
|
||||
);
|
||||
|
||||
Log::info("ReindexChunkJob: Chunk #{$this->chunkId} berjaya direindex.", [
|
||||
'chunk_index' => $chunk->chunk_index,
|
||||
'point_id' => $pointId,
|
||||
]);
|
||||
|
||||
} catch (RuntimeException $e) {
|
||||
// Tandakan failed — job akan cuba semula (mengikut $tries)
|
||||
$chunk->update(['chunk_status' => DocumentChunk::STATUS_FAILED_EMBEDDING]);
|
||||
|
||||
ProcessingLog::record(
|
||||
DocumentChunk::class,
|
||||
$chunk->id,
|
||||
ProcessingLog::STAGE_EMBED,
|
||||
ProcessingLog::STATUS_FAILED,
|
||||
$e->getMessage(),
|
||||
['attempt' => $this->attempts()]
|
||||
);
|
||||
|
||||
Log::error("ReindexChunkJob: Gagal reindex chunk #{$this->chunkId}.", [
|
||||
'error' => $e->getMessage(),
|
||||
'attempt' => $this->attempts(),
|
||||
]);
|
||||
|
||||
throw $e; // Allow retry
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dipanggil selepas semua retry habis.
|
||||
*/
|
||||
public function failed(\Throwable $e): void
|
||||
{
|
||||
DocumentChunk::where('id', $this->chunkId)
|
||||
->update(['chunk_status' => DocumentChunk::STATUS_FAILED_EMBEDDING]);
|
||||
|
||||
Log::error("ReindexChunkJob: Chunk #{$this->chunkId} gagal selepas semua cubaan semula.", [
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
80
app/Jobs/ReindexDocumentJob.php
Normal file
80
app/Jobs/ReindexDocumentJob.php
Normal file
@@ -0,0 +1,80 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\DocumentVersion;
|
||||
use App\Services\KnowledgeBase\AuditService;
|
||||
use App\Services\KnowledgeBase\IngestionService;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* ReindexDocumentJob
|
||||
*
|
||||
* Reindex (reprocess embedding) untuk document version tertentu.
|
||||
* Berguna bila:
|
||||
* - model embedding ditukar
|
||||
* - chunking strategy dikemaskini
|
||||
* - Qdrant collection diset semula
|
||||
*
|
||||
* Proses: Deactivate chunk lama → Delete chunk lama → Re-chunk → Re-embed → Qdrant
|
||||
*/
|
||||
class ReindexDocumentJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public int $tries = 2;
|
||||
public int $backoff = 60;
|
||||
public int $timeout = 600;
|
||||
|
||||
public function __construct(
|
||||
private readonly int $documentVersionId
|
||||
) {
|
||||
$this->onQueue(config('knowledgebase.queue.ingestion', 'default'));
|
||||
}
|
||||
|
||||
public function handle(IngestionService $ingestionService, AuditService $auditService): void
|
||||
{
|
||||
$version = DocumentVersion::with(['document.category'])->findOrFail($this->documentVersionId);
|
||||
|
||||
Log::info("ReindexDocumentJob mula untuk version {$this->documentVersionId}");
|
||||
|
||||
// Deactivate point lama dalam Qdrant
|
||||
$ingestionService->deactivateVersionInQdrant($version);
|
||||
|
||||
// Padam chunk lama dari MySQL untuk reprocess
|
||||
$version->chunks()->delete();
|
||||
|
||||
// Reset status dan mula proses semula
|
||||
$version->update([
|
||||
'processing_status' => DocumentVersion::STATUS_PENDING,
|
||||
'processing_error' => null,
|
||||
'processing_completed_at' => null,
|
||||
]);
|
||||
|
||||
// Reprocess
|
||||
$ingestionService->processDocumentVersion($version);
|
||||
|
||||
$auditService->documentReindexed($version->document, $version);
|
||||
|
||||
Log::info("ReindexDocumentJob selesai untuk version {$this->documentVersionId}");
|
||||
}
|
||||
|
||||
public function failed(Throwable $exception): void
|
||||
{
|
||||
Log::error("ReindexDocumentJob GAGAL untuk version {$this->documentVersionId}", [
|
||||
'error' => $exception->getMessage(),
|
||||
]);
|
||||
|
||||
$version = DocumentVersion::find($this->documentVersionId);
|
||||
$version?->updateStatus(
|
||||
DocumentVersion::STATUS_FAILED,
|
||||
'Reindex gagal: ' . $exception->getMessage()
|
||||
);
|
||||
}
|
||||
}
|
||||
50
app/Jobs/ReindexKnowledgeItemJob.php
Normal file
50
app/Jobs/ReindexKnowledgeItemJob.php
Normal file
@@ -0,0 +1,50 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\KnowledgeItem;
|
||||
use App\Services\KnowledgeBase\IngestionService;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* ReindexKnowledgeItemJob
|
||||
*
|
||||
* Reindex (re-embed) satu knowledge item ke Qdrant.
|
||||
* Berguna bila kandungan FAQ/polisi dikemaskini.
|
||||
*/
|
||||
class ReindexKnowledgeItemJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public int $tries = 3;
|
||||
public int $backoff = 30;
|
||||
public int $timeout = 120;
|
||||
|
||||
public function __construct(
|
||||
private readonly int $knowledgeItemId
|
||||
) {
|
||||
$this->onQueue(config('knowledgebase.queue.embedding', 'default'));
|
||||
}
|
||||
|
||||
public function handle(IngestionService $ingestionService): void
|
||||
{
|
||||
$item = KnowledgeItem::with('category')->findOrFail($this->knowledgeItemId);
|
||||
|
||||
Log::info("ReindexKnowledgeItemJob untuk item {$this->knowledgeItemId}");
|
||||
|
||||
$ingestionService->processKnowledgeItem($item);
|
||||
}
|
||||
|
||||
public function failed(Throwable $exception): void
|
||||
{
|
||||
Log::error("ReindexKnowledgeItemJob GAGAL untuk item {$this->knowledgeItemId}", [
|
||||
'error' => $exception->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user