feat: use native MLX backend when --model-path is set on Apple Silicon
Build-tagged backend selection: MLX on darwin/arm64/mlx, HTTP elsewhere. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
6f2d9f8de4
commit
f8d8bd6556
3 changed files with 37 additions and 3 deletions
|
|
@ -79,9 +79,12 @@ type usageInfo struct {
|
|||
}
|
||||
|
||||
func runServe(cmd *cli.Command, args []string) error {
|
||||
// Create a backend — use HTTP backend pointing to configured API URL.
|
||||
// On macOS with MLX build tag, this will use the native MLX backend instead.
|
||||
backend := ml.NewHTTPBackend(apiURL, modelName)
|
||||
// Try native MLX backend first (macOS arm64 with mlx tag + model-path set),
|
||||
// fall back to HTTP proxy backend.
|
||||
backend, err := createServeBackend()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
|
|
|
|||
9
internal/cmd/ml/serve_backend_default.go
Normal file
9
internal/cmd/ml/serve_backend_default.go
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
//go:build !(darwin && arm64 && mlx)
|
||||
|
||||
package ml
|
||||
|
||||
import "forge.lthn.ai/core/cli/pkg/ml"
|
||||
|
||||
func createServeBackend() (ml.Backend, error) {
|
||||
return ml.NewHTTPBackend(apiURL, modelName), nil
|
||||
}
|
||||
22
internal/cmd/ml/serve_backend_mlx.go
Normal file
22
internal/cmd/ml/serve_backend_mlx.go
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
//go:build darwin && arm64 && mlx
|
||||
|
||||
package ml
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"forge.lthn.ai/core/cli/pkg/ml"
|
||||
)
|
||||
|
||||
func createServeBackend() (ml.Backend, error) {
|
||||
if serveModelPath != "" {
|
||||
slog.Info("ml serve: loading native MLX backend", "path", serveModelPath)
|
||||
b, err := ml.NewMLXBackend(serveModelPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mlx backend: %w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
return ml.NewHTTPBackend(apiURL, modelName), nil
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue