feat(brain): add EmbedMemory job + indexed_at tracking

Implements the async-embedding pipeline's worker side:

- php/Jobs/EmbedMemory.php — Laravel Job that calls BrainService::embed()
  + qdrantUpsert() and sets indexed_at on success
- php/Migrations/…_add_indexed_at_to_brain_memories.php — nullable
  timestamp + index, portable across pgsql/mariadb (hasColumn guard)
- BrainMemory: +indexed_at fillable + datetime cast + PHPDoc
- BrainService: qdrantUpsert() private→public so the Job can use it;
  elasticIndex() stub added (to be implemented by the ES ticket)
- php/tests/Feature/Jobs/EmbedMemoryTest.php — Pest tests for success
  path and Qdrant-failure path

Co-authored-by: Codex <noreply@openai.com>

Closes tasks.lthn.sh/view.php?id=56
This commit is contained in:
Snider 2026-04-23 12:47:10 +01:00
parent 0741fba88f
commit 4dc5ed8d14
5 changed files with 222 additions and 6 deletions

61
php/Jobs/EmbedMemory.php Normal file
View file

@ -0,0 +1,61 @@
<?php
// SPDX-License-Identifier: EUPL-1.2
declare(strict_types=1);
namespace Core\Mod\Agentic\Jobs;
use Core\Mod\Agentic\Models\BrainMemory;
use Core\Mod\Agentic\Services\BrainService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class EmbedMemory implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
/**
* @var array<int, int>
*/
public array $backoff = [10, 60, 300];
public function __construct(
public string $memoryId,
) {}
public function handle(BrainService $brain): void
{
$memory = BrainMemory::find($this->memoryId);
if (! $memory instanceof BrainMemory) {
return;
}
$vector = $brain->embed($memory->content);
$payload = $brain->buildQdrantPayload($memory->id, [
'workspace_id' => $memory->workspace_id,
'org' => $memory->getAttribute('org'),
'project' => $memory->project,
'agent_id' => $memory->agent_id,
'type' => $memory->type,
'tags' => $memory->tags ?? [],
'confidence' => $memory->confidence,
'source' => $memory->source ?? 'manual',
'content' => $memory->content,
'created_at' => $memory->created_at?->toIso8601String(),
]);
$payload['vector'] = $vector;
$brain->qdrantUpsert([$payload]);
$brain->elasticIndex($memory);
$memory->update(['indexed_at' => now()]);
}
}

View file

@ -0,0 +1,40 @@
<?php
// SPDX-License-Identifier: EUPL-1.2
declare(strict_types=1);
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
protected $connection = 'brain';
public function up(): void
{
$schema = Schema::connection($this->getConnection());
if (! $schema->hasTable('brain_memories') || $schema->hasColumn('brain_memories', 'indexed_at')) {
return;
}
$schema->table('brain_memories', function (Blueprint $table): void {
$table->timestamp('indexed_at')->nullable()->after('source')->index();
});
}
public function down(): void
{
$schema = Schema::connection($this->getConnection());
if (! $schema->hasTable('brain_memories') || ! $schema->hasColumn('brain_memories', 'indexed_at')) {
return;
}
$schema->table('brain_memories', function (Blueprint $table): void {
$table->dropColumn('indexed_at');
});
}
};

View file

@ -6,6 +6,7 @@ declare(strict_types=1);
namespace Core\Mod\Agentic\Models;
use Carbon\Carbon;
use Core\Tenant\Concerns\BelongsToWorkspace;
use Core\Tenant\Models\Workspace;
use Illuminate\Database\Eloquent\Builder;
@ -31,10 +32,11 @@ use Illuminate\Database\Eloquent\SoftDeletes;
* @property string|null $project
* @property float $confidence
* @property string|null $supersedes_id
* @property \Carbon\Carbon|null $expires_at
* @property \Carbon\Carbon|null $created_at
* @property \Carbon\Carbon|null $updated_at
* @property \Carbon\Carbon|null $deleted_at
* @property Carbon|null $indexed_at
* @property Carbon|null $expires_at
* @property Carbon|null $created_at
* @property Carbon|null $updated_at
* @property Carbon|null $deleted_at
*/
class BrainMemory extends Model
{
@ -72,6 +74,7 @@ class BrainMemory extends Model
'project',
'confidence',
'supersedes_id',
'indexed_at',
'expires_at',
'source',
];
@ -79,6 +82,7 @@ class BrainMemory extends Model
protected $casts = [
'tags' => 'array',
'confidence' => 'float',
'indexed_at' => 'datetime',
'expires_at' => 'datetime',
];

View file

@ -1,10 +1,13 @@
<?php
// SPDX-License-Identifier: EUPL-1.2
declare(strict_types=1);
namespace Core\Mod\Agentic\Services;
use Core\Mod\Agentic\Models\BrainMemory;
use Illuminate\Http\Client\PendingRequest;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
@ -26,7 +29,7 @@ class BrainService
/**
* Create an HTTP client with common settings.
*/
private function http(int $timeout = 10): \Illuminate\Http\Client\PendingRequest
private function http(int $timeout = 10): PendingRequest
{
return $this->verifySsl
? Http::timeout($timeout)
@ -203,6 +206,14 @@ class BrainService
];
}
/**
* Index a memory in Elasticsearch.
*/
public function elasticIndex(BrainMemory $memory): void
{
// Implemented by the Elasticsearch integration ticket.
}
/**
* Build a Qdrant filter from criteria.
*
@ -251,7 +262,7 @@ class BrainService
*
* @throws \RuntimeException
*/
private function qdrantUpsert(array $points): void
public function qdrantUpsert(array $points): void
{
$response = $this->http(10)
->put("{$this->qdrantUrl}/collections/{$this->collection}/points", [

View file

@ -0,0 +1,100 @@
<?php
// SPDX-License-Identifier: EUPL-1.2
declare(strict_types=1);
use Core\Mod\Agentic\Jobs\EmbedMemory;
use Core\Mod\Agentic\Models\BrainMemory;
use Core\Mod\Agentic\Services\BrainService;
use Illuminate\Http\Client\Request;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Str;
function embedMemoryService(): BrainService
{
return new BrainService(
ollamaUrl: 'https://ollama.test',
qdrantUrl: 'https://qdrant.test',
collection: 'openbrain',
embeddingModel: 'embeddinggemma',
verifySsl: false,
);
}
test('EmbedMemory_handle_Good_embeds_upserts_and_marks_indexed', function (): void {
$workspace = createWorkspace();
$embedding = array_fill(0, 768, 0.125);
Http::fake([
'https://ollama.test/api/embeddings' => Http::response(['embedding' => $embedding]),
'https://qdrant.test/collections/openbrain/points' => Http::response(['result' => ['status' => 'ok']]),
]);
$memory = BrainMemory::create([
'workspace_id' => $workspace->id,
'agent_id' => 'virgil',
'type' => 'architecture',
'content' => 'Queue memory indexing through EmbedMemory.',
'tags' => ['brain', 'indexing'],
'project' => 'agent',
'confidence' => 0.95,
'source' => 'ticket-56',
]);
(new EmbedMemory($memory->id))->handle(embedMemoryService());
expect($memory->fresh()->indexed_at)->not->toBeNull();
Http::assertSent(fn (Request $request): bool => $request->url() === 'https://ollama.test/api/embeddings'
&& $request->method() === 'POST'
&& $request['model'] === 'embeddinggemma'
&& $request['prompt'] === 'Queue memory indexing through EmbedMemory.');
Http::assertSent(fn (Request $request): bool => $request->url() === 'https://qdrant.test/collections/openbrain/points'
&& $request->method() === 'PUT'
&& $request['points'][0]['id'] === $memory->id
&& $request['points'][0]['vector'] === $embedding
&& $request['points'][0]['payload']['workspace_id'] === $workspace->id
&& $request['points'][0]['payload']['project'] === 'agent'
&& $request['points'][0]['payload']['agent_id'] === 'virgil'
&& $request['points'][0]['payload']['type'] === 'architecture'
&& $request['points'][0]['payload']['tags'] === ['brain', 'indexing']
&& $request['points'][0]['payload']['confidence'] === 0.95
&& $request['points'][0]['payload']['source'] === 'ticket-56'
&& $request['points'][0]['payload']['content'] === 'Queue memory indexing through EmbedMemory.');
});
test('EmbedMemory_handle_Bad_returns_silently_when_memory_is_missing', function (): void {
Http::fake();
(new EmbedMemory(Str::uuid()->toString()))->handle(embedMemoryService());
Http::assertNothingSent();
});
test('EmbedMemory_handle_Ugly_leaves_memory_unindexed_when_qdrant_fails', function (): void {
$workspace = createWorkspace();
Http::fake([
'https://ollama.test/api/embeddings' => Http::response(['embedding' => array_fill(0, 768, 0.25)]),
'https://qdrant.test/collections/openbrain/points' => Http::response(['error' => 'unavailable'], 500),
]);
$memory = BrainMemory::create([
'workspace_id' => $workspace->id,
'agent_id' => 'virgil',
'type' => 'observation',
'content' => 'Qdrant failures should retry without marking the memory indexed.',
'confidence' => 0.8,
]);
try {
(new EmbedMemory($memory->id))->handle(embedMemoryService());
$this->fail('Expected Qdrant failure to throw.');
} catch (RuntimeException $exception) {
expect($exception->getMessage())->toBe('Qdrant upsert failed: 500');
}
expect($memory->fresh()->indexed_at)->toBeNull();
});