From cb5a37abed9e30d3fcc2fd3b6e6ea615225c30ed Mon Sep 17 00:00:00 2001 From: feitianbubu Date: Thu, 27 Nov 2025 12:39:17 +0800 Subject: [PATCH 1/4] feat: gemini image support edit --- relay/channel/api_request.go | 2 ++ relay/channel/gemini/adaptor.go | 22 ++++++++++-- relay/channel/gemini/relay-gemini.go | 7 ++-- relay/common/relay_utils.go | 54 ++++++++++++++++++++++++++++ relay/helper/valid_request.go | 1 + 5 files changed, 81 insertions(+), 5 deletions(-) diff --git a/relay/channel/api_request.go b/relay/channel/api_request.go index 1ff1e2392..22426c69e 100644 --- a/relay/channel/api_request.go +++ b/relay/channel/api_request.go @@ -27,6 +27,8 @@ import ( func SetupApiRequestHeader(info *common.RelayInfo, c *gin.Context, req *http.Header) { if info.RelayMode == constant.RelayModeAudioTranscription || info.RelayMode == constant.RelayModeAudioTranslation { // multipart/form-data + } else if info.RelayMode == constant.RelayModeImagesEdits { + // multipart/form-data } else if info.RelayMode == constant.RelayModeRealtime { // websocket } else { diff --git a/relay/channel/gemini/adaptor.go b/relay/channel/gemini/adaptor.go index 021ed0623..fcc99b662 100644 --- a/relay/channel/gemini/adaptor.go +++ b/relay/channel/gemini/adaptor.go @@ -142,11 +142,29 @@ func processSizeParameters(size, quality string) ImageConfig { } func (a *Adaptor) ConvertImageRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.ImageRequest) (any, error) { - if strings.HasPrefix(info.UpstreamModelName, "gemini-3-pro-image") { + if model_setting.IsGeminiModelSupportImagine(info.UpstreamModelName) { + var content any + if base64Data, err := relaycommon.GetImageBase64sFromForm(c); err == nil { + content = []any{ + dto.MediaContent{ + Type: dto.ContentTypeText, + Text: request.Prompt, + }, + dto.MediaContent{ + Type: dto.ContentTypeFile, + File: &dto.MessageFile{ + FileData: base64Data.String(), + }, + }, + } + } else { + content = request.Prompt + } + chatRequest := dto.GeneralOpenAIRequest{ Model: request.Model, Messages: []dto.Message{ - {Role: "user", Content: request.Prompt}, + {Role: "user", Content: content}, }, N: int(request.N), } diff --git a/relay/channel/gemini/relay-gemini.go b/relay/channel/gemini/relay-gemini.go index 7776847be..9da33b308 100644 --- a/relay/channel/gemini/relay-gemini.go +++ b/relay/channel/gemini/relay-gemini.go @@ -183,7 +183,7 @@ func ThinkingAdaptor(geminiRequest *dto.GeminiChatRequest, info *relaycommon.Rel } // Setting safety to the lowest possible values since Gemini is already powerless enough -func CovertOpenAI2Gemini(c *gin.Context, textRequest dto.GeneralOpenAIRequest, info *relaycommon.RelayInfo) (*dto.GeminiChatRequest, error) { +func CovertOpenAI2Gemini(c *gin.Context, textRequest dto.GeneralOpenAIRequest, info *relaycommon.RelayInfo, base64Data ...*relaycommon.Base64Data) (*dto.GeminiChatRequest, error) { geminiRequest := dto.GeminiChatRequest{ Contents: make([]dto.GeminiChatContent, 0, len(textRequest.Messages)), @@ -464,10 +464,11 @@ func CovertOpenAI2Gemini(c *gin.Context, textRequest dto.GeneralOpenAIRequest, i }) } } else if part.Type == dto.ContentTypeFile { - if part.GetFile().FileId != "" { + file := part.GetFile() + if file.FileId != "" { return nil, fmt.Errorf("only base64 file is supported in gemini") } - format, base64String, err := service.DecodeBase64FileData(part.GetFile().FileData) + format, base64String, err := service.DecodeBase64FileData(file.FileData) if err != nil { return nil, fmt.Errorf("decode base64 file data failed: %s", err.Error()) } diff --git a/relay/common/relay_utils.go b/relay/common/relay_utils.go index b662f9053..b97583024 100644 --- a/relay/common/relay_utils.go +++ b/relay/common/relay_utils.go @@ -1,7 +1,10 @@ package common import ( + "encoding/base64" + "errors" "fmt" + "io" "net/http" "strconv" "strings" @@ -226,3 +229,54 @@ func ValidateBasicTaskRequest(c *gin.Context, info *RelayInfo, action string) *d storeTaskRequest(c, info, action, req) return nil } +func GetImagesBase64sFromForm(c *gin.Context) ([]*Base64Data, error) { + return GetBase64sFromForm(c, "image") +} +func GetImageBase64sFromForm(c *gin.Context) (*Base64Data, error) { + base64s, err := GetImagesBase64sFromForm(c) + if err != nil { + return nil, err + } + return base64s[0], nil +} + +type Base64Data struct { + MimeType string + Data string +} + +func (m Base64Data) String() string { + return fmt.Sprintf("data:%s;base64,%s", m.MimeType, m.Data) +} +func GetBase64sFromForm(c *gin.Context, fieldName string) ([]*Base64Data, error) { + mf := c.Request.MultipartForm + if mf == nil { + if _, err := c.MultipartForm(); err != nil { + return nil, fmt.Errorf("failed to parse image edit form request: %w", err) + } + mf = c.Request.MultipartForm + } + imageFiles, exists := mf.File[fieldName] + if !exists || len(imageFiles) == 0 { + return nil, errors.New("field " + fieldName + "\" is not found or empty") + } + var imageBase64s []*Base64Data + for _, file := range imageFiles { + image, err := file.Open() + if err != nil { + return nil, errors.New("failed to open image file") + } + imageData, err := io.ReadAll(image) + if err != nil { + return nil, errors.New("failed to read image file") + } + mimeType := http.DetectContentType(imageData) + base64Data := base64.StdEncoding.EncodeToString(imageData) + imageBase64s = append(imageBase64s, &Base64Data{ + MimeType: mimeType, + Data: base64Data, + }) + image.Close() + } + return imageBase64s, nil +} diff --git a/relay/helper/valid_request.go b/relay/helper/valid_request.go index 3bdfa6ff4..e6e8dc989 100644 --- a/relay/helper/valid_request.go +++ b/relay/helper/valid_request.go @@ -141,6 +141,7 @@ func GetAndValidOpenAIImageRequest(c *gin.Context, relayMode int) (*dto.ImageReq imageRequest.N = uint(common.String2Int(formData.Get("n"))) imageRequest.Quality = formData.Get("quality") imageRequest.Size = formData.Get("size") + imageRequest.ResponseFormat = formData.Get("response_format") if imageValue := formData.Get("image"); imageValue != "" { imageRequest.Image, _ = json.Marshal(imageValue) } From a0982996a44cebcd828fbc3ff31f7cbbe68cb0f6 Mon Sep 17 00:00:00 2001 From: IcedTangerine Date: Thu, 27 Nov 2025 17:56:59 +0800 Subject: [PATCH 2/4] Use defer to close image file after opening Ensure image file is closed using defer after opening. --- relay/common/relay_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/common/relay_utils.go b/relay/common/relay_utils.go index b97583024..53e792974 100644 --- a/relay/common/relay_utils.go +++ b/relay/common/relay_utils.go @@ -263,6 +263,7 @@ func GetBase64sFromForm(c *gin.Context, fieldName string) ([]*Base64Data, error) var imageBase64s []*Base64Data for _, file := range imageFiles { image, err := file.Open() + defer image.Close() if err != nil { return nil, errors.New("failed to open image file") } @@ -276,7 +277,6 @@ func GetBase64sFromForm(c *gin.Context, fieldName string) ([]*Base64Data, error) MimeType: mimeType, Data: base64Data, }) - image.Close() } return imageBase64s, nil } From 4d00dad002c7ab42ed9da3910346a75f12e7daf1 Mon Sep 17 00:00:00 2001 From: IcedTangerine Date: Thu, 27 Nov 2025 17:59:38 +0800 Subject: [PATCH 3/4] Fix error message formatting in relay_utils.go --- relay/common/relay_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/common/relay_utils.go b/relay/common/relay_utils.go index 53e792974..3a4eeee8d 100644 --- a/relay/common/relay_utils.go +++ b/relay/common/relay_utils.go @@ -258,7 +258,7 @@ func GetBase64sFromForm(c *gin.Context, fieldName string) ([]*Base64Data, error) } imageFiles, exists := mf.File[fieldName] if !exists || len(imageFiles) == 0 { - return nil, errors.New("field " + fieldName + "\" is not found or empty") + return nil, errors.New("field " + fieldName + " is not found or empty") } var imageBase64s []*Base64Data for _, file := range imageFiles { From 420c6e58f2af94916552610efa31c7e123547dbe Mon Sep 17 00:00:00 2001 From: IcedTangerine Date: Thu, 27 Nov 2025 18:01:34 +0800 Subject: [PATCH 4/4] Fix defer placement for image file closure --- relay/common/relay_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/common/relay_utils.go b/relay/common/relay_utils.go index 3a4eeee8d..1159298c4 100644 --- a/relay/common/relay_utils.go +++ b/relay/common/relay_utils.go @@ -263,10 +263,10 @@ func GetBase64sFromForm(c *gin.Context, fieldName string) ([]*Base64Data, error) var imageBase64s []*Base64Data for _, file := range imageFiles { image, err := file.Open() - defer image.Close() if err != nil { return nil, errors.New("failed to open image file") } + defer image.Close() imageData, err := io.ReadAll(image) if err != nil { return nil, errors.New("failed to read image file")