From 4dc5ed8d14f27a95840148c95ddabc13dc8ef8c7 Mon Sep 17 00:00:00 2001 From: Snider Date: Thu, 23 Apr 2026 12:47:10 +0100 Subject: [PATCH] feat(brain): add EmbedMemory job + indexed_at tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the async-embedding pipeline's worker side: - php/Jobs/EmbedMemory.php — Laravel Job that calls BrainService::embed() + qdrantUpsert() and sets indexed_at on success - php/Migrations/…_add_indexed_at_to_brain_memories.php — nullable timestamp + index, portable across pgsql/mariadb (hasColumn guard) - BrainMemory: +indexed_at fillable + datetime cast + PHPDoc - BrainService: qdrantUpsert() private→public so the Job can use it; elasticIndex() stub added (to be implemented by the ES ticket) - php/tests/Feature/Jobs/EmbedMemoryTest.php — Pest tests for success path and Qdrant-failure path Co-authored-by: Codex Closes tasks.lthn.sh/view.php?id=56 --- php/Jobs/EmbedMemory.php | 61 +++++++++++ ...00001_add_indexed_at_to_brain_memories.php | 40 +++++++ php/Models/BrainMemory.php | 12 ++- php/Services/BrainService.php | 15 ++- php/tests/Feature/Jobs/EmbedMemoryTest.php | 100 ++++++++++++++++++ 5 files changed, 222 insertions(+), 6 deletions(-) create mode 100644 php/Jobs/EmbedMemory.php create mode 100644 php/Migrations/2026_04_23_000001_add_indexed_at_to_brain_memories.php create mode 100644 php/tests/Feature/Jobs/EmbedMemoryTest.php diff --git a/php/Jobs/EmbedMemory.php b/php/Jobs/EmbedMemory.php new file mode 100644 index 0000000..d1f3520 --- /dev/null +++ b/php/Jobs/EmbedMemory.php @@ -0,0 +1,61 @@ + + */ + public array $backoff = [10, 60, 300]; + + public function __construct( + public string $memoryId, + ) {} + + public function handle(BrainService $brain): void + { + $memory = BrainMemory::find($this->memoryId); + + if (! $memory instanceof BrainMemory) { + return; + } + + $vector = $brain->embed($memory->content); + + $payload = $brain->buildQdrantPayload($memory->id, [ + 'workspace_id' => $memory->workspace_id, + 'org' => $memory->getAttribute('org'), + 'project' => $memory->project, + 'agent_id' => $memory->agent_id, + 'type' => $memory->type, + 'tags' => $memory->tags ?? [], + 'confidence' => $memory->confidence, + 'source' => $memory->source ?? 'manual', + 'content' => $memory->content, + 'created_at' => $memory->created_at?->toIso8601String(), + ]); + $payload['vector'] = $vector; + + $brain->qdrantUpsert([$payload]); + $brain->elasticIndex($memory); + + $memory->update(['indexed_at' => now()]); + } +} diff --git a/php/Migrations/2026_04_23_000001_add_indexed_at_to_brain_memories.php b/php/Migrations/2026_04_23_000001_add_indexed_at_to_brain_memories.php new file mode 100644 index 0000000..f5b438c --- /dev/null +++ b/php/Migrations/2026_04_23_000001_add_indexed_at_to_brain_memories.php @@ -0,0 +1,40 @@ +getConnection()); + + if (! $schema->hasTable('brain_memories') || $schema->hasColumn('brain_memories', 'indexed_at')) { + return; + } + + $schema->table('brain_memories', function (Blueprint $table): void { + $table->timestamp('indexed_at')->nullable()->after('source')->index(); + }); + } + + public function down(): void + { + $schema = Schema::connection($this->getConnection()); + + if (! $schema->hasTable('brain_memories') || ! $schema->hasColumn('brain_memories', 'indexed_at')) { + return; + } + + $schema->table('brain_memories', function (Blueprint $table): void { + $table->dropColumn('indexed_at'); + }); + } +}; diff --git a/php/Models/BrainMemory.php b/php/Models/BrainMemory.php index 3f23b38..9fa9096 100644 --- a/php/Models/BrainMemory.php +++ b/php/Models/BrainMemory.php @@ -6,6 +6,7 @@ declare(strict_types=1); namespace Core\Mod\Agentic\Models; +use Carbon\Carbon; use Core\Tenant\Concerns\BelongsToWorkspace; use Core\Tenant\Models\Workspace; use Illuminate\Database\Eloquent\Builder; @@ -31,10 +32,11 @@ use Illuminate\Database\Eloquent\SoftDeletes; * @property string|null $project * @property float $confidence * @property string|null $supersedes_id - * @property \Carbon\Carbon|null $expires_at - * @property \Carbon\Carbon|null $created_at - * @property \Carbon\Carbon|null $updated_at - * @property \Carbon\Carbon|null $deleted_at + * @property Carbon|null $indexed_at + * @property Carbon|null $expires_at + * @property Carbon|null $created_at + * @property Carbon|null $updated_at + * @property Carbon|null $deleted_at */ class BrainMemory extends Model { @@ -72,6 +74,7 @@ class BrainMemory extends Model 'project', 'confidence', 'supersedes_id', + 'indexed_at', 'expires_at', 'source', ]; @@ -79,6 +82,7 @@ class BrainMemory extends Model protected $casts = [ 'tags' => 'array', 'confidence' => 'float', + 'indexed_at' => 'datetime', 'expires_at' => 'datetime', ]; diff --git a/php/Services/BrainService.php b/php/Services/BrainService.php index 2c6940b..bd61002 100644 --- a/php/Services/BrainService.php +++ b/php/Services/BrainService.php @@ -1,10 +1,13 @@ verifySsl ? Http::timeout($timeout) @@ -203,6 +206,14 @@ class BrainService ]; } + /** + * Index a memory in Elasticsearch. + */ + public function elasticIndex(BrainMemory $memory): void + { + // Implemented by the Elasticsearch integration ticket. + } + /** * Build a Qdrant filter from criteria. * @@ -251,7 +262,7 @@ class BrainService * * @throws \RuntimeException */ - private function qdrantUpsert(array $points): void + public function qdrantUpsert(array $points): void { $response = $this->http(10) ->put("{$this->qdrantUrl}/collections/{$this->collection}/points", [ diff --git a/php/tests/Feature/Jobs/EmbedMemoryTest.php b/php/tests/Feature/Jobs/EmbedMemoryTest.php new file mode 100644 index 0000000..e20269d --- /dev/null +++ b/php/tests/Feature/Jobs/EmbedMemoryTest.php @@ -0,0 +1,100 @@ + Http::response(['embedding' => $embedding]), + 'https://qdrant.test/collections/openbrain/points' => Http::response(['result' => ['status' => 'ok']]), + ]); + + $memory = BrainMemory::create([ + 'workspace_id' => $workspace->id, + 'agent_id' => 'virgil', + 'type' => 'architecture', + 'content' => 'Queue memory indexing through EmbedMemory.', + 'tags' => ['brain', 'indexing'], + 'project' => 'agent', + 'confidence' => 0.95, + 'source' => 'ticket-56', + ]); + + (new EmbedMemory($memory->id))->handle(embedMemoryService()); + + expect($memory->fresh()->indexed_at)->not->toBeNull(); + + Http::assertSent(fn (Request $request): bool => $request->url() === 'https://ollama.test/api/embeddings' + && $request->method() === 'POST' + && $request['model'] === 'embeddinggemma' + && $request['prompt'] === 'Queue memory indexing through EmbedMemory.'); + + Http::assertSent(fn (Request $request): bool => $request->url() === 'https://qdrant.test/collections/openbrain/points' + && $request->method() === 'PUT' + && $request['points'][0]['id'] === $memory->id + && $request['points'][0]['vector'] === $embedding + && $request['points'][0]['payload']['workspace_id'] === $workspace->id + && $request['points'][0]['payload']['project'] === 'agent' + && $request['points'][0]['payload']['agent_id'] === 'virgil' + && $request['points'][0]['payload']['type'] === 'architecture' + && $request['points'][0]['payload']['tags'] === ['brain', 'indexing'] + && $request['points'][0]['payload']['confidence'] === 0.95 + && $request['points'][0]['payload']['source'] === 'ticket-56' + && $request['points'][0]['payload']['content'] === 'Queue memory indexing through EmbedMemory.'); +}); + +test('EmbedMemory_handle_Bad_returns_silently_when_memory_is_missing', function (): void { + Http::fake(); + + (new EmbedMemory(Str::uuid()->toString()))->handle(embedMemoryService()); + + Http::assertNothingSent(); +}); + +test('EmbedMemory_handle_Ugly_leaves_memory_unindexed_when_qdrant_fails', function (): void { + $workspace = createWorkspace(); + + Http::fake([ + 'https://ollama.test/api/embeddings' => Http::response(['embedding' => array_fill(0, 768, 0.25)]), + 'https://qdrant.test/collections/openbrain/points' => Http::response(['error' => 'unavailable'], 500), + ]); + + $memory = BrainMemory::create([ + 'workspace_id' => $workspace->id, + 'agent_id' => 'virgil', + 'type' => 'observation', + 'content' => 'Qdrant failures should retry without marking the memory indexed.', + 'confidence' => 0.8, + ]); + + try { + (new EmbedMemory($memory->id))->handle(embedMemoryService()); + $this->fail('Expected Qdrant failure to throw.'); + } catch (RuntimeException $exception) { + expect($exception->getMessage())->toBe('Qdrant upsert failed: 500'); + } + + expect($memory->fresh()->indexed_at)->toBeNull(); +});