diff --git a/pkg/display/preload.go b/pkg/display/preload.go index d2b16924..d3fb45b8 100644 --- a/pkg/display/preload.go +++ b/pkg/display/preload.go @@ -835,49 +835,139 @@ func buildElectronShimScript() string { return () => {}; }; - root.electron = root.electron || { - clipboard: { - readText() { - return invoke('clipboard:read').then((result) => { - if (result && typeof result === 'object' && 'text' in result) { - return String(result.text ?? ''); + class CoreElectronNotification { + constructor(options = {}) { + this.options = options && typeof options === 'object' ? { ...options } : {}; + this.onclick = null; + this.onshow = null; + this.onclose = null; + this.onerror = null; + this._fallback = null; + } + + static isSupported() { + return Boolean(root.__coreGUIBridge) || Boolean(fallbackNotification()); + } + + show() { + const payload = { + title: String(this.options.title ?? ''), + message: String(this.options.body ?? this.options.message ?? ''), + subtitle: typeof this.options.subtitle === 'string' ? this.options.subtitle : '', + icon: typeof this.options.icon === 'string' ? this.options.icon : '', + silent: Boolean(this.options.silent), + actions: Array.isArray(this.options.actions) ? this.options.actions : [], + }; + + return invoke('notification:show', payload) + .catch((error) => { + const NativeNotification = fallbackNotification(); + if (!NativeNotification) { + throw error; } - return String(result ?? ''); + + const nativeNotification = new NativeNotification(payload.title || 'Notification', { + body: payload.message, + icon: payload.icon || undefined, + silent: payload.silent, + }); + + this._fallback = nativeNotification; + if (typeof nativeNotification.addEventListener === 'function') { + nativeNotification.addEventListener('click', () => { + if (typeof this.onclick === 'function') { + this.onclick({ target: nativeNotification }); + } + }); + nativeNotification.addEventListener('close', () => { + if (typeof this.onclose === 'function') { + this.onclose({ target: nativeNotification }); + } + }); + } else { + nativeNotification.onclick = (event) => { + if (typeof this.onclick === 'function') { + this.onclick(event); + } + }; + } + + return nativeNotification; + }) + .then((result) => { + if (typeof this.onshow === 'function') { + this.onshow({ target: result }); + } + return result; + }) + .catch((error) => { + if (typeof this.onerror === 'function') { + this.onerror(error); + } + throw error; }); - }, - writeText(text) { - return invoke('clipboard:write', { text: String(text ?? '') }); - }, + } + + close() { + if (this._fallback && typeof this._fallback.close === 'function') { + this._fallback.close(); + } + if (typeof this.onclose === 'function') { + this.onclose({ target: this }); + } + } + } + + const fallbackNotification = () => { + if (typeof root.Notification === 'function' && root.Notification !== CoreElectronNotification) { + return root.Notification; + } + return null; + }; + + root.electron = root.electron || {}; + root.electron.Notification = root.electron.Notification || CoreElectronNotification; + root.electron.clipboard = root.electron.clipboard || { + readText() { + return invoke('clipboard:read').then((result) => { + if (result && typeof result === 'object' && 'text' in result) { + return String(result.text ?? ''); + } + return String(result ?? ''); + }); }, - dialog: { - showMessageBox(options) { - return invoke('dialog:message', options ?? {}); - }, - showOpenDialog(options) { - return invoke('dialog:open-file', options ?? {}); - }, - showSaveDialog(options) { - return invoke('dialog:save-file', options ?? {}); - }, + writeText(text) { + return invoke('clipboard:write', { text: String(text ?? '') }); }, - ipcRenderer: { - invoke(channel, payload) { - return invoke('core.ipc.query', { channel, payload }); - }, - on(channel, handler) { - return subscribe(channel, handler); - }, - send(channel, payload) { - return invoke('core.ipc.action', { channel, payload }); - }, + }; + root.electron.dialog = root.electron.dialog || { + showMessageBox(options) { + return invoke('dialog:message', options ?? {}); }, - shell: { - openExternal(target) { - return invoke('browser:open-url', { url: String(target ?? '') }); - }, - openPath(target) { - return invoke('browser:open-file', { path: String(target ?? '') }); - }, + showOpenDialog(options) { + return invoke('dialog:open-file', options ?? {}); + }, + showSaveDialog(options) { + return invoke('dialog:save-file', options ?? {}); + }, + }; + root.electron.ipcRenderer = root.electron.ipcRenderer || { + invoke(channel, payload) { + return invoke('core.ipc.query', { channel, payload }); + }, + on(channel, handler) { + return subscribe(channel, handler); + }, + send(channel, payload) { + return invoke('core.ipc.action', { channel, payload }); + }, + }; + root.electron.shell = root.electron.shell || { + openExternal(target) { + return invoke('browser:open-url', { url: String(target ?? '') }); + }, + openPath(target) { + return invoke('browser:open-file', { path: String(target ?? '') }); }, }; diff --git a/pkg/display/preload_test.go b/pkg/display/preload_test.go index b565773d..d2d1fb2c 100644 --- a/pkg/display/preload_test.go +++ b/pkg/display/preload_test.go @@ -46,6 +46,8 @@ func TestPreloadScript_Good(t *testing.T) { assert.Contains(t, script, "window.__appPreloadLoaded = true;") assert.Contains(t, script, "root.core.ml.generate") assert.Contains(t, script, "root.electron = root.electron ||") + assert.Contains(t, script, "root.electron.Notification") + assert.Contains(t, script, "notification:show") assert.Contains(t, script, `"theme":"dark"`) assert.Contains(t, script, `"session_id":"abc123"`) } @@ -60,6 +62,7 @@ func TestInjectPreload_Good(t *testing.T) { assert.Contains(t, target.script, "root.core.ml.generate") assert.Contains(t, target.script, "root.core.storage") + assert.Contains(t, target.script, "root.electron.Notification") } func TestBrowserStoragePersistenceAndSearch_Good(t *testing.T) { diff --git a/ui/src/app/dashboard.component.ts b/ui/src/app/dashboard.component.ts index 27eda923..0bcbe6ae 100644 --- a/ui/src/app/dashboard.component.ts +++ b/ui/src/app/dashboard.component.ts @@ -14,7 +14,12 @@ import { ChatService, Conversation, ImageAttachment, + SUPPORTED_CHAT_IMAGE_ACCEPT, + SUPPORTED_CHAT_IMAGE_LABEL, ToolInvocation, + conversationSearchText, + isSupportedChatImageFile, + isSupportedChatImageMimeType, } from '../services/chat.service'; import { UiStateService } from '../services/ui-state.service'; @@ -346,7 +351,7 @@ interface ConversationGroup { this.chat.selectedModelEntry()?.supportsVision !== false, ); @@ -1019,8 +1025,7 @@ export class DashboardComponent implements AfterViewChecked { for (const conversation of this.chat.conversations()) { if (query) { - const haystack = `${conversation.title} ${conversation.messages.map((message) => message.content).join(' ')}`.toLowerCase(); - if (!haystack.includes(query)) { + if (!conversationSearchText(conversation).includes(query)) { continue; } } @@ -1343,11 +1348,29 @@ export class DashboardComponent implements AfterViewChecked { } return; } + + const supportedFiles = imageFiles.filter((file) => isSupportedChatImageFile(file)); + const skippedFiles = imageFiles.length - supportedFiles.length; + if (supportedFiles.length === 0) { + this.showComposerNotice(`Supported image formats: ${SUPPORTED_CHAT_IMAGE_LABEL}.`); + return; + } if (!this.selectedModelSupportsVision()) { this.showComposerNotice('The selected model does not support image input.'); return; } - await this.chat.addAttachments(imageFiles); + + try { + await this.chat.addAttachments(supportedFiles); + if (skippedFiles > 0) { + this.showComposerNotice( + `Skipped ${skippedFiles} unsupported attachment${skippedFiles === 1 ? '' : 's'}. Supported formats: ${SUPPORTED_CHAT_IMAGE_LABEL}.`, + ); + } + } catch (error) { + const message = error instanceof Error ? error.message : `Supported image formats: ${SUPPORTED_CHAT_IMAGE_LABEL}.`; + this.showComposerNotice(message); + } } private showComposerNotice(message: string): void { @@ -1407,7 +1430,7 @@ function hasImageFiles(dataTransfer: DataTransfer | null): boolean { if (!dataTransfer) { return false; } - return Array.from(dataTransfer.items).some((item) => item.type.startsWith('image/')); + return Array.from(dataTransfer.items).some((item) => isSupportedChatImageMimeType(item.type)); } function renderMarkdownContent(content: string): string { diff --git a/ui/src/services/chat.service.ts b/ui/src/services/chat.service.ts index 09c70d5e..8401c8ee 100644 --- a/ui/src/services/chat.service.ts +++ b/ui/src/services/chat.service.ts @@ -68,6 +68,47 @@ export interface Conversation { } const STORAGE_KEY = 'core.gui.chat.state'; +export const SUPPORTED_CHAT_IMAGE_MIME_TYPES = [ + 'image/png', + 'image/jpeg', + 'image/webp', + 'image/gif', +] as const; +export const SUPPORTED_CHAT_IMAGE_LABEL = 'PNG, JPEG, WebP, or GIF'; +export const SUPPORTED_CHAT_IMAGE_ACCEPT = SUPPORTED_CHAT_IMAGE_MIME_TYPES.join(','); + +interface ChatToolDefinition { + name: string; + description: string; + parameters: Record; +} + +const REGISTERED_CHAT_TOOLS: ChatToolDefinition[] = [ + { + name: 'gui.chat.settings.load', + description: 'Load the persisted inference defaults for the chat shell.', + parameters: {}, + }, + { + name: 'gui.chat.models', + description: 'List the locally available chat models and which model is selected.', + parameters: {}, + }, + { + name: 'gui.chat.conversations.search', + description: 'Search saved conversation history by title, content, tool calls, and attachments.', + parameters: { + q: 'Search string', + }, + }, + { + name: 'gui.route.store', + description: 'Search the local CoreGUI store surface for matching chat data.', + parameters: { + q: 'Search string', + }, + }, +]; function defaultSettings(): ChatSettings { return { @@ -113,6 +154,69 @@ function defaultModels(): ModelEntry[] { ]; } +export function normaliseChatImageMimeType(mimeType: string): string { + return mimeType.trim().toLowerCase(); +} + +function inferChatImageMimeType(fileName: string): string { + const normalized = fileName.trim().toLowerCase(); + if (normalized.endsWith('.png')) { + return 'image/png'; + } + if (normalized.endsWith('.jpg') || normalized.endsWith('.jpeg')) { + return 'image/jpeg'; + } + if (normalized.endsWith('.webp')) { + return 'image/webp'; + } + if (normalized.endsWith('.gif')) { + return 'image/gif'; + } + return ''; +} + +function resolveChatImageMimeType(file: Pick): string { + const mimeType = normaliseChatImageMimeType(file.type ?? ''); + if (mimeType) { + return mimeType; + } + return inferChatImageMimeType(file.name ?? ''); +} + +export function isSupportedChatImageMimeType(mimeType: string): boolean { + return SUPPORTED_CHAT_IMAGE_MIME_TYPES.includes( + normaliseChatImageMimeType(mimeType) as (typeof SUPPORTED_CHAT_IMAGE_MIME_TYPES)[number], + ); +} + +export function isSupportedChatImageFile(file: Pick): boolean { + return isSupportedChatImageMimeType(resolveChatImageMimeType(file)); +} + +export function conversationSearchText(conversation: Conversation): string { + return [ + conversation.title, + conversation.model, + ...conversation.messages.flatMap((message) => [ + message.role, + message.content, + message.thinking?.content ?? '', + ...(message.attachments ?? []).flatMap((attachment) => [ + attachment.filename, + attachment.mimeType, + ]), + ...(message.toolCalls ?? []).flatMap((toolCall) => [ + toolCall.name, + JSON.stringify(toolCall.arguments ?? {}), + toolCall.result, + toolCall.error ?? '', + ]), + ]), + ] + .join(' ') + .toLowerCase(); +} + @Injectable({ providedIn: 'root' }) export class ChatService { private readonly storage = window.localStorage; @@ -247,12 +351,17 @@ export class ChatService { } async addAttachment(file: File): Promise { + const mimeType = resolveChatImageMimeType(file); + if (!isSupportedChatImageMimeType(mimeType)) { + throw new Error(`Supported image formats: ${SUPPORTED_CHAT_IMAGE_LABEL}.`); + } + const dataUrl = await readAsDataURL(file); const dimensions = await readImageSize(dataUrl); const attachment: ImageAttachment = { id: crypto.randomUUID(), filename: file.name, - mimeType: file.type || 'application/octet-stream', + mimeType, data: dataUrl, width: dimensions.width, height: dimensions.height, @@ -694,8 +803,7 @@ function describeConversationMatches(conversations: Conversation[], query: strin if (!needle) { return true; } - const haystack = `${conversation.title} ${conversation.messages.map((message) => message.content).join(' ')}`.toLowerCase(); - return haystack.includes(needle); + return conversationSearchText(conversation).includes(needle); }); if (matches.length === 0) { @@ -733,16 +841,30 @@ function describeStoreMatches(conversations: Conversation[], query: string): str } function buildToolAwarePrompt(prompt: string, toolOutputs: string[]): string { - if (toolOutputs.length === 0) { - return prompt; + const sections = [ + 'System tool manifest:', + buildToolManifest(), + '', + 'User prompt:', + prompt, + ]; + + if (toolOutputs.length > 0) { + sections.push('', 'Tool context:', ...toolOutputs.map((output) => `- ${output}`)); } - return [ - prompt, - '', - 'Tool context:', - ...toolOutputs.map((output) => `- ${output}`), - ].join('\n'); + return sections.join('\n'); +} + +function buildToolManifest(): string { + return REGISTERED_CHAT_TOOLS.map((tool) => { + const params = Object.entries(tool.parameters) + .map(([name, description]) => `${name}: ${description}`) + .join(', '); + return params + ? `- ${tool.name}: ${tool.description} Parameters: ${params}.` + : `- ${tool.name}: ${tool.description}`; + }).join('\n'); } function readAsDataURL(file: File): Promise {