feat(brain): add EmbedMemory job + indexed_at tracking
Implements the async-embedding pipeline's worker side: - php/Jobs/EmbedMemory.php — Laravel Job that calls BrainService::embed() + qdrantUpsert() and sets indexed_at on success - php/Migrations/…_add_indexed_at_to_brain_memories.php — nullable timestamp + index, portable across pgsql/mariadb (hasColumn guard) - BrainMemory: +indexed_at fillable + datetime cast + PHPDoc - BrainService: qdrantUpsert() private→public so the Job can use it; elasticIndex() stub added (to be implemented by the ES ticket) - php/tests/Feature/Jobs/EmbedMemoryTest.php — Pest tests for success path and Qdrant-failure path Co-authored-by: Codex <noreply@openai.com> Closes tasks.lthn.sh/view.php?id=56
This commit is contained in:
parent
0741fba88f
commit
4dc5ed8d14
5 changed files with 222 additions and 6 deletions
61
php/Jobs/EmbedMemory.php
Normal file
61
php/Jobs/EmbedMemory.php
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
<?php
|
||||
|
||||
// SPDX-License-Identifier: EUPL-1.2
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Core\Mod\Agentic\Jobs;
|
||||
|
||||
use Core\Mod\Agentic\Models\BrainMemory;
|
||||
use Core\Mod\Agentic\Services\BrainService;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
|
||||
class EmbedMemory implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public int $tries = 3;
|
||||
|
||||
/**
|
||||
* @var array<int, int>
|
||||
*/
|
||||
public array $backoff = [10, 60, 300];
|
||||
|
||||
public function __construct(
|
||||
public string $memoryId,
|
||||
) {}
|
||||
|
||||
public function handle(BrainService $brain): void
|
||||
{
|
||||
$memory = BrainMemory::find($this->memoryId);
|
||||
|
||||
if (! $memory instanceof BrainMemory) {
|
||||
return;
|
||||
}
|
||||
|
||||
$vector = $brain->embed($memory->content);
|
||||
|
||||
$payload = $brain->buildQdrantPayload($memory->id, [
|
||||
'workspace_id' => $memory->workspace_id,
|
||||
'org' => $memory->getAttribute('org'),
|
||||
'project' => $memory->project,
|
||||
'agent_id' => $memory->agent_id,
|
||||
'type' => $memory->type,
|
||||
'tags' => $memory->tags ?? [],
|
||||
'confidence' => $memory->confidence,
|
||||
'source' => $memory->source ?? 'manual',
|
||||
'content' => $memory->content,
|
||||
'created_at' => $memory->created_at?->toIso8601String(),
|
||||
]);
|
||||
$payload['vector'] = $vector;
|
||||
|
||||
$brain->qdrantUpsert([$payload]);
|
||||
$brain->elasticIndex($memory);
|
||||
|
||||
$memory->update(['indexed_at' => now()]);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
<?php
|
||||
|
||||
// SPDX-License-Identifier: EUPL-1.2
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
protected $connection = 'brain';
|
||||
|
||||
public function up(): void
|
||||
{
|
||||
$schema = Schema::connection($this->getConnection());
|
||||
|
||||
if (! $schema->hasTable('brain_memories') || $schema->hasColumn('brain_memories', 'indexed_at')) {
|
||||
return;
|
||||
}
|
||||
|
||||
$schema->table('brain_memories', function (Blueprint $table): void {
|
||||
$table->timestamp('indexed_at')->nullable()->after('source')->index();
|
||||
});
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
$schema = Schema::connection($this->getConnection());
|
||||
|
||||
if (! $schema->hasTable('brain_memories') || ! $schema->hasColumn('brain_memories', 'indexed_at')) {
|
||||
return;
|
||||
}
|
||||
|
||||
$schema->table('brain_memories', function (Blueprint $table): void {
|
||||
$table->dropColumn('indexed_at');
|
||||
});
|
||||
}
|
||||
};
|
||||
|
|
@ -6,6 +6,7 @@ declare(strict_types=1);
|
|||
|
||||
namespace Core\Mod\Agentic\Models;
|
||||
|
||||
use Carbon\Carbon;
|
||||
use Core\Tenant\Concerns\BelongsToWorkspace;
|
||||
use Core\Tenant\Models\Workspace;
|
||||
use Illuminate\Database\Eloquent\Builder;
|
||||
|
|
@ -31,10 +32,11 @@ use Illuminate\Database\Eloquent\SoftDeletes;
|
|||
* @property string|null $project
|
||||
* @property float $confidence
|
||||
* @property string|null $supersedes_id
|
||||
* @property \Carbon\Carbon|null $expires_at
|
||||
* @property \Carbon\Carbon|null $created_at
|
||||
* @property \Carbon\Carbon|null $updated_at
|
||||
* @property \Carbon\Carbon|null $deleted_at
|
||||
* @property Carbon|null $indexed_at
|
||||
* @property Carbon|null $expires_at
|
||||
* @property Carbon|null $created_at
|
||||
* @property Carbon|null $updated_at
|
||||
* @property Carbon|null $deleted_at
|
||||
*/
|
||||
class BrainMemory extends Model
|
||||
{
|
||||
|
|
@ -72,6 +74,7 @@ class BrainMemory extends Model
|
|||
'project',
|
||||
'confidence',
|
||||
'supersedes_id',
|
||||
'indexed_at',
|
||||
'expires_at',
|
||||
'source',
|
||||
];
|
||||
|
|
@ -79,6 +82,7 @@ class BrainMemory extends Model
|
|||
protected $casts = [
|
||||
'tags' => 'array',
|
||||
'confidence' => 'float',
|
||||
'indexed_at' => 'datetime',
|
||||
'expires_at' => 'datetime',
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
<?php
|
||||
|
||||
// SPDX-License-Identifier: EUPL-1.2
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Core\Mod\Agentic\Services;
|
||||
|
||||
use Core\Mod\Agentic\Models\BrainMemory;
|
||||
use Illuminate\Http\Client\PendingRequest;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
|
@ -26,7 +29,7 @@ class BrainService
|
|||
/**
|
||||
* Create an HTTP client with common settings.
|
||||
*/
|
||||
private function http(int $timeout = 10): \Illuminate\Http\Client\PendingRequest
|
||||
private function http(int $timeout = 10): PendingRequest
|
||||
{
|
||||
return $this->verifySsl
|
||||
? Http::timeout($timeout)
|
||||
|
|
@ -203,6 +206,14 @@ class BrainService
|
|||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a memory in Elasticsearch.
|
||||
*/
|
||||
public function elasticIndex(BrainMemory $memory): void
|
||||
{
|
||||
// Implemented by the Elasticsearch integration ticket.
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a Qdrant filter from criteria.
|
||||
*
|
||||
|
|
@ -251,7 +262,7 @@ class BrainService
|
|||
*
|
||||
* @throws \RuntimeException
|
||||
*/
|
||||
private function qdrantUpsert(array $points): void
|
||||
public function qdrantUpsert(array $points): void
|
||||
{
|
||||
$response = $this->http(10)
|
||||
->put("{$this->qdrantUrl}/collections/{$this->collection}/points", [
|
||||
|
|
|
|||
100
php/tests/Feature/Jobs/EmbedMemoryTest.php
Normal file
100
php/tests/Feature/Jobs/EmbedMemoryTest.php
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
<?php
|
||||
|
||||
// SPDX-License-Identifier: EUPL-1.2
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
use Core\Mod\Agentic\Jobs\EmbedMemory;
|
||||
use Core\Mod\Agentic\Models\BrainMemory;
|
||||
use Core\Mod\Agentic\Services\BrainService;
|
||||
use Illuminate\Http\Client\Request;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
function embedMemoryService(): BrainService
|
||||
{
|
||||
return new BrainService(
|
||||
ollamaUrl: 'https://ollama.test',
|
||||
qdrantUrl: 'https://qdrant.test',
|
||||
collection: 'openbrain',
|
||||
embeddingModel: 'embeddinggemma',
|
||||
verifySsl: false,
|
||||
);
|
||||
}
|
||||
|
||||
test('EmbedMemory_handle_Good_embeds_upserts_and_marks_indexed', function (): void {
|
||||
$workspace = createWorkspace();
|
||||
$embedding = array_fill(0, 768, 0.125);
|
||||
|
||||
Http::fake([
|
||||
'https://ollama.test/api/embeddings' => Http::response(['embedding' => $embedding]),
|
||||
'https://qdrant.test/collections/openbrain/points' => Http::response(['result' => ['status' => 'ok']]),
|
||||
]);
|
||||
|
||||
$memory = BrainMemory::create([
|
||||
'workspace_id' => $workspace->id,
|
||||
'agent_id' => 'virgil',
|
||||
'type' => 'architecture',
|
||||
'content' => 'Queue memory indexing through EmbedMemory.',
|
||||
'tags' => ['brain', 'indexing'],
|
||||
'project' => 'agent',
|
||||
'confidence' => 0.95,
|
||||
'source' => 'ticket-56',
|
||||
]);
|
||||
|
||||
(new EmbedMemory($memory->id))->handle(embedMemoryService());
|
||||
|
||||
expect($memory->fresh()->indexed_at)->not->toBeNull();
|
||||
|
||||
Http::assertSent(fn (Request $request): bool => $request->url() === 'https://ollama.test/api/embeddings'
|
||||
&& $request->method() === 'POST'
|
||||
&& $request['model'] === 'embeddinggemma'
|
||||
&& $request['prompt'] === 'Queue memory indexing through EmbedMemory.');
|
||||
|
||||
Http::assertSent(fn (Request $request): bool => $request->url() === 'https://qdrant.test/collections/openbrain/points'
|
||||
&& $request->method() === 'PUT'
|
||||
&& $request['points'][0]['id'] === $memory->id
|
||||
&& $request['points'][0]['vector'] === $embedding
|
||||
&& $request['points'][0]['payload']['workspace_id'] === $workspace->id
|
||||
&& $request['points'][0]['payload']['project'] === 'agent'
|
||||
&& $request['points'][0]['payload']['agent_id'] === 'virgil'
|
||||
&& $request['points'][0]['payload']['type'] === 'architecture'
|
||||
&& $request['points'][0]['payload']['tags'] === ['brain', 'indexing']
|
||||
&& $request['points'][0]['payload']['confidence'] === 0.95
|
||||
&& $request['points'][0]['payload']['source'] === 'ticket-56'
|
||||
&& $request['points'][0]['payload']['content'] === 'Queue memory indexing through EmbedMemory.');
|
||||
});
|
||||
|
||||
test('EmbedMemory_handle_Bad_returns_silently_when_memory_is_missing', function (): void {
|
||||
Http::fake();
|
||||
|
||||
(new EmbedMemory(Str::uuid()->toString()))->handle(embedMemoryService());
|
||||
|
||||
Http::assertNothingSent();
|
||||
});
|
||||
|
||||
test('EmbedMemory_handle_Ugly_leaves_memory_unindexed_when_qdrant_fails', function (): void {
|
||||
$workspace = createWorkspace();
|
||||
|
||||
Http::fake([
|
||||
'https://ollama.test/api/embeddings' => Http::response(['embedding' => array_fill(0, 768, 0.25)]),
|
||||
'https://qdrant.test/collections/openbrain/points' => Http::response(['error' => 'unavailable'], 500),
|
||||
]);
|
||||
|
||||
$memory = BrainMemory::create([
|
||||
'workspace_id' => $workspace->id,
|
||||
'agent_id' => 'virgil',
|
||||
'type' => 'observation',
|
||||
'content' => 'Qdrant failures should retry without marking the memory indexed.',
|
||||
'confidence' => 0.8,
|
||||
]);
|
||||
|
||||
try {
|
||||
(new EmbedMemory($memory->id))->handle(embedMemoryService());
|
||||
$this->fail('Expected Qdrant failure to throw.');
|
||||
} catch (RuntimeException $exception) {
|
||||
expect($exception->getMessage())->toBe('Qdrant upsert failed: 500');
|
||||
}
|
||||
|
||||
expect($memory->fresh()->indexed_at)->toBeNull();
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue