Skip to content

Commit

Permalink
feat(ui): prompt for chat, support vision, enhancements (#2259)
Browse files Browse the repository at this point in the history 
* feat(ui): allow to set system prompt for chat

Make also the models in the index clickable, and display as table

Fixes #2257

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* feat(vision): support also png with base64 input

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* feat(ui): support vision and upload of files

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* display the processed image

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* make trust remote code stand out

Signed-off-by: mudler <mudler@localai.io>

* feat(ui): track in progress job across index/model gallery

Signed-off-by: mudler <mudler@localai.io>

* minor fixups

Signed-off-by: mudler <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Signed-off-by: mudler <mudler@localai.io>
  • Loading branch information
@mudler
mudler authored May 7, 2024
1 parent 02ec546 commit 6559ac1
Show file tree
Hide file tree
Showing 11 changed files with 344 additions and 82 deletions.
54 changes: 38 additions & 16 deletions core/http/elements/gallery.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/chasefleming/elem-go"
"github.com/chasefleming/elem-go/attrs"
"github.com/go-skynet/LocalAI/core/services"
"github.com/go-skynet/LocalAI/pkg/gallery"
"github.com/go-skynet/LocalAI/pkg/xsync"
)
Expand Down Expand Up @@ -72,12 +73,13 @@ func StartProgressBar(uid, progress, text string) string {
if progress == "" {
progress = "0"
}
return elem.Div(attrs.Props{
"hx-trigger": "done",
"hx-get": "/browse/job/" + uid,
"hx-swap": "innerHTML",
"hx-target": "this",
},
return elem.Div(
attrs.Props{
"hx-trigger": "done",
"hx-get": "/browse/job/" + uid,
"hx-swap": "innerHTML",
"hx-target": "this",
},
elem.H3(
attrs.Props{
"role": "status",
Expand Down Expand Up @@ -223,7 +225,7 @@ func deleteButton(modelName string) elem.Node {
)
}

func ListModels(models []*gallery.GalleryModel, installing *xsync.SyncedMap[string, string]) string {
func ListModels(models []*gallery.GalleryModel, processing *xsync.SyncedMap[string, string], galleryService *services.GalleryService) string {
//StartProgressBar(uid, "0")
modelsElements := []elem.Node{}
// span := func(s string) elem.Node {
Expand Down Expand Up @@ -258,7 +260,15 @@ func ListModels(models []*gallery.GalleryModel, installing *xsync.SyncedMap[stri

actionDiv := func(m *gallery.GalleryModel) elem.Node {
galleryID := fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name)
currentlyInstalling := installing.Exists(galleryID)
currentlyProcessing := processing.Exists(galleryID)
isDeletionOp := false
if currentlyProcessing {
status := galleryService.GetStatus(galleryID)
if status != nil && status.Deletion {
isDeletionOp = true
}
// if status == nil : "Waiting"
}

nodes := []elem.Node{
cardSpan("Repository: "+m.Gallery.Name, "fa-brands fa-git-alt"),
Expand Down Expand Up @@ -292,6 +302,11 @@ func ListModels(models []*gallery.GalleryModel, installing *xsync.SyncedMap[stri
)
}

progressMessage := "Installation"
if isDeletionOp {
progressMessage = "Deletion"
}

return elem.Div(
attrs.Props{
"class": "px-6 pt-4 pb-2",
Expand All @@ -303,9 +318,9 @@ func ListModels(models []*gallery.GalleryModel, installing *xsync.SyncedMap[stri
nodes...,
),
elem.If(
currentlyInstalling,
currentlyProcessing,
elem.Node( // If currently installing, show progress bar
elem.Raw(StartProgressBar(installing.Get(galleryID), "0", "Installing")),
elem.Raw(StartProgressBar(processing.Get(galleryID), "0", progressMessage)),
), // Otherwise, show install button (if not installed) or display "Installed"
elem.If(m.Installed,
elem.Node(elem.Div(
Expand All @@ -331,12 +346,6 @@ func ListModels(models []*gallery.GalleryModel, installing *xsync.SyncedMap[stri
"class": "flex justify-center items-center",
}

_, trustRemoteCodeExists := m.Overrides["trust_remote_code"]
if trustRemoteCodeExists {
// should this be checking for trust_remote_code: false? I don't think we ever use that value.
divProperties["class"] = divProperties["class"] + " remote-code"
}

elems = append(elems,

elem.Div(divProperties,
Expand All @@ -352,6 +361,19 @@ func ListModels(models []*gallery.GalleryModel, installing *xsync.SyncedMap[stri
),
))

_, trustRemoteCodeExists := m.Overrides["trust_remote_code"]
if trustRemoteCodeExists {
elems = append(elems, elem.Div(
attrs.Props{
"class": "flex justify-center items-center bg-red-500 text-white p-2 rounded-lg mt-2",
},
elem.I(attrs.Props{
"class": "fa-solid fa-circle-exclamation pr-2",
}),
elem.Text("Attention: Trust Remote Code is required for this model"),
))
}

elems = append(elems, descriptionDiv(m), actionDiv(m))
modelsElements = append(modelsElements,
elem.Div(
Expand Down
7 changes: 6 additions & 1 deletion core/http/endpoints/localai/welcome.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
)

func WelcomeEndpoint(appConfig *config.ApplicationConfig,
cl *config.BackendConfigLoader, ml *model.ModelLoader) func(*fiber.Ctx) error {
cl *config.BackendConfigLoader, ml *model.ModelLoader, modelStatus func() (map[string]string, map[string]string)) func(*fiber.Ctx) error {
return func(c *fiber.Ctx) error {
models, _ := ml.ListModels()
backendConfigs := cl.GetAllBackendConfigs()
Expand All @@ -24,13 +24,18 @@ func WelcomeEndpoint(appConfig *config.ApplicationConfig,
galleryConfigs[m.Name] = cfg
}

// Get model statuses to display in the UI the operation in progress
processingModels, taskTypes := modelStatus()

summary := fiber.Map{
"Title": "LocalAI API - " + internal.PrintableVersion(),
"Version": internal.PrintableVersion(),
"Models": models,
"ModelsConfig": backendConfigs,
"GalleryConfig": galleryConfigs,
"ApplicationConfig": appConfig,
"ProcessingModels": processingModels,
"TaskTypes": taskTypes,
}

if string(c.Context().Request.Header.ContentType()) == "application/json" || len(c.Accepts("html")) == 0 {
Expand Down
12 changes: 8 additions & 4 deletions core/http/endpoints/openai/request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,14 @@ func getBase64Image(s string) (string, error) {
return encoded, nil
}

// if the string instead is prefixed with "data:image/jpeg;base64,", drop it
if strings.HasPrefix(s, "data:image/jpeg;base64,") {
return strings.ReplaceAll(s, "data:image/jpeg;base64,", ""), nil
// if the string instead is prefixed with "data:image/...;base64,", drop it
dropPrefix := []string{"data:image/jpeg;base64,", "data:image/png;base64,"}
for _, prefix := range dropPrefix {
if strings.HasPrefix(s, prefix) {
return strings.ReplaceAll(s, prefix, ""), nil
}
}

return "", fmt.Errorf("not valid string")
}

Expand Down Expand Up @@ -181,7 +185,7 @@ func updateRequestConfig(config *config.BackendConfig, input *schema.OpenAIReque
input.Messages[i].StringContent = fmt.Sprintf("[img-%d]", index) + input.Messages[i].StringContent
index++
} else {
fmt.Print("Failed encoding image", err)
log.Error().Msgf("Failed encoding image: %s", err)
}
}
}
Expand Down
70 changes: 46 additions & 24 deletions core/http/routes/ui.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,35 @@ func RegisterUIRoutes(app *fiber.App,
galleryService *services.GalleryService,
auth func(*fiber.Ctx) error) {

app.Get("/", auth, localai.WelcomeEndpoint(appConfig, cl, ml))

// keeps the state of models that are being installed from the UI
var installingModels = xsync.NewSyncedMap[string, string]()
var processingModels = xsync.NewSyncedMap[string, string]()

// modelStatus returns the current status of the models being processed (installation or deletion)
// it is called asynchonously from the UI
modelStatus := func() (map[string]string, map[string]string) {
processingModelsData := processingModels.Map()

taskTypes := map[string]string{}

for k, v := range processingModelsData {
status := galleryService.GetStatus(v)
taskTypes[k] = "Installation"
if status != nil && status.Deletion {
taskTypes[k] = "Deletion"
} else if status == nil {
taskTypes[k] = "Waiting"
}
}

return processingModelsData, taskTypes
}

app.Get("/", auth, localai.WelcomeEndpoint(appConfig, cl, ml, modelStatus))

// Show the Models page (all models)
app.Get("/browse", auth, func(c *fiber.Ctx) error {
term := c.Query("term")

models, _ := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.ModelPath)

// Get all available tags
Expand All @@ -47,12 +69,22 @@ func RegisterUIRoutes(app *fiber.App,
tags = append(tags, t)
}
sort.Strings(tags)

if term != "" {
models = gallery.GalleryModels(models).Search(term)
}

// Get model statuses
processingModelsData, taskTypes := modelStatus()

summary := fiber.Map{
"Title": "LocalAI - Models",
"Version": internal.PrintableVersion(),
"Models": template.HTML(elements.ListModels(models, installingModels)),
"Repositories": appConfig.Galleries,
"AllTags": tags,
"Title": "LocalAI - Models",
"Version": internal.PrintableVersion(),
"Models": template.HTML(elements.ListModels(models, processingModels, galleryService)),
"Repositories": appConfig.Galleries,
"AllTags": tags,
"ProcessingModels": processingModelsData,
"TaskTypes": taskTypes,
// "ApplicationConfig": appConfig,
}

Expand All @@ -72,17 +104,7 @@ func RegisterUIRoutes(app *fiber.App,

models, _ := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.ModelPath)

filteredModels := []*gallery.GalleryModel{}
for _, m := range models {
if strings.Contains(m.Name, form.Search) ||
strings.Contains(m.Description, form.Search) ||
strings.Contains(m.Gallery.Name, form.Search) ||
strings.Contains(strings.Join(m.Tags, ","), form.Search) {
filteredModels = append(filteredModels, m)
}
}

return c.SendString(elements.ListModels(filteredModels, installingModels))
return c.SendString(elements.ListModels(gallery.GalleryModels(models).Search(form.Search), processingModels, galleryService))
})

/*
Expand All @@ -103,7 +125,7 @@ func RegisterUIRoutes(app *fiber.App,

uid := id.String()

installingModels.Set(galleryID, uid)
processingModels.Set(galleryID, uid)

op := gallery.GalleryOp{
Id: uid,
Expand All @@ -129,7 +151,7 @@ func RegisterUIRoutes(app *fiber.App,

uid := id.String()

installingModels.Set(galleryID, uid)
processingModels.Set(galleryID, uid)

op := gallery.GalleryOp{
Id: uid,
Expand Down Expand Up @@ -174,10 +196,10 @@ func RegisterUIRoutes(app *fiber.App,
status := galleryService.GetStatus(c.Params("uid"))

galleryID := ""
for _, k := range installingModels.Keys() {
if installingModels.Get(k) == c.Params("uid") {
for _, k := range processingModels.Keys() {
if processingModels.Get(k) == c.Params("uid") {
galleryID = k
installingModels.Delete(k)
processingModels.Delete(k)
}
}

Expand Down
Loading

0 comments on commit 6559ac1

Please sign in to comment.