feat: jimeng images base64 limit (#2032)

feat: jimeng use openai sdk input_reference i2v

.github/workflows/docker-image-arm64.yml

@@ -33,7 +33,8 @@ jobs:
       - name: Resolve tag & write VERSION
         run: |
           git fetch --tags --force --depth=1
-          echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+          TAG=${GITHUB_REF#refs/tags/}
+          echo "TAG=$TAG" >> $GITHUB_ENV
           echo "$TAG" > VERSION
           echo "Building tag: $TAG for ${{ matrix.arch }}"
 

.github/workflows/electron-build.yml

@@ -130,13 +130,10 @@ jobs:
       - name: Download all artifacts
         uses: actions/download-artifact@v4
 
-      - name: Create Release
+      - name: Upload to Release
         uses: softprops/action-gh-release@v2
         with:
           files: |
             windows-build/*
-          draft: false
-          prerelease: false
-          overwrite_files: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -54,8 +54,6 @@ jobs:
         with:
           files: |
             new-api-*
-          draft: true
-          generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -93,8 +91,6 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
         with:
           files: new-api-macos-*
-          draft: true
-          generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -134,8 +130,6 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
         with:
           files: new-api-*.exe
-          draft: true
-          generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

controller/task_video.go

@@ -137,14 +137,19 @@ func updateVideoSingleTask(ctx context.Context, adaptor channel.TaskAdaptor, cha
 				if modelName, ok := taskData["model"].(string); ok && modelName != "" {
 					// 获取模型价格和倍率
 					modelRatio, hasRatioSetting, _ := ratio_setting.GetModelRatio(modelName)
-
 					// 只有配置了倍率(非固定价格)时才按 token 重新计费
 					if hasRatioSetting && modelRatio > 0 {
 						// 获取用户和组的倍率信息
-						user, err := model.GetUserById(task.UserId, false)
-						if err == nil {
-							groupRatio := ratio_setting.GetGroupRatio(user.Group)
-							userGroupRatio, hasUserGroupRatio := ratio_setting.GetGroupGroupRatio(user.Group, user.Group)
+						group := task.Group
+						if group == "" {
+							user, err := model.GetUserById(task.UserId, false)
+							if err == nil {
+								group = user.Group
+							}
+						}
+						if group != "" {
+							groupRatio := ratio_setting.GetGroupRatio(group)
+							userGroupRatio, hasUserGroupRatio := ratio_setting.GetGroupGroupRatio(group, group)
 
 							var finalGroupRatio float64
 							if hasUserGroupRatio {
@@ -214,6 +219,7 @@ func updateVideoSingleTask(ctx context.Context, adaptor channel.TaskAdaptor, cha
 			}
 		}
 	case model.TaskStatusFailure:
+		preStatus := task.Status
 		task.Status = model.TaskStatusFailure
 		task.Progress = "100%"
 		if task.FinishTime == 0 {
@@ -222,12 +228,18 @@ func updateVideoSingleTask(ctx context.Context, adaptor channel.TaskAdaptor, cha
 		task.FailReason = taskResult.Reason
 		logger.LogInfo(ctx, fmt.Sprintf("Task %s failed: %s", task.TaskID, task.FailReason))
 		quota := task.Quota
+		taskResult.Progress = "100%"
 		if quota != 0 {
-			if err := model.IncreaseUserQuota(task.UserId, quota, false); err != nil {
-				logger.LogError(ctx, "Failed to increase user quota: "+err.Error())
+			if preStatus != model.TaskStatusFailure {
+				// 任务失败且之前状态不是失败才退还额度，防止重复退还
+				if err := model.IncreaseUserQuota(task.UserId, quota, false); err != nil {
+					logger.LogWarn(ctx, "Failed to increase user quota: "+err.Error())
+				}
+				logContent := fmt.Sprintf("Video async task failed %s, refund %s", task.TaskID, logger.LogQuota(quota))
+				model.RecordLog(task.UserId, model.LogTypeSystem, logContent)
+			} else {
+				logger.LogWarn(ctx, fmt.Sprintf("Task %s already in failure status, skip refund", task.TaskID))
 			}
-			logContent := fmt.Sprintf("Video async task failed %s, refund %s", task.TaskID, logger.LogQuota(quota))
-			model.RecordLog(task.UserId, model.LogTypeSystem, logContent)
 		}
 	default:
 		return fmt.Errorf("unknown task status %s for task %s", taskResult.Status, taskId)

dto/openai_video.go

@@ -1,4 +1,4 @@
-package common
+package dto
 
 import (
 	"strconv"

model/task.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/QuantumNous/new-api/constant"
+	"github.com/QuantumNous/new-api/dto"
 	commonRelay "github.com/QuantumNous/new-api/relay/common"
 )
 
@@ -15,15 +16,15 @@ func (t TaskStatus) ToVideoStatus() string {
 	var status string
 	switch t {
 	case TaskStatusQueued, TaskStatusSubmitted:
-		status = commonRelay.VideoStatusQueued
+		status = dto.VideoStatusQueued
 	case TaskStatusInProgress:
-		status = commonRelay.VideoStatusInProgress
+		status = dto.VideoStatusInProgress
 	case TaskStatusSuccess:
-		status = commonRelay.VideoStatusCompleted
+		status = dto.VideoStatusCompleted
 	case TaskStatusFailure:
-		status = commonRelay.VideoStatusFailed
+		status = dto.VideoStatusFailed
 	default:
-		status = commonRelay.VideoStatusUnknown // Default fallback
+		status = dto.VideoStatusUnknown // Default fallback
 	}
 	return status
 }
@@ -45,6 +46,7 @@ type Task struct {
 	TaskID     string                `json:"task_id" gorm:"type:varchar(191);index"` // 第三方id，不一定有/ song id\ Task id
 	Platform   constant.TaskPlatform `json:"platform" gorm:"type:varchar(30);index"` // 平台
 	UserId     int                   `json:"user_id" gorm:"index"`
+	Group      string                `json:"group" gorm:"type:varchar(50)"` // 修正计费用
 	ChannelId  int                   `json:"channel_id" gorm:"index"`
 	Quota      int                   `json:"quota"`
 	Action     string                `json:"action" gorm:"type:varchar(40);index"` // 任务类型, song, lyrics, description-mode
@@ -98,6 +100,7 @@ type SyncTaskQueryParams struct {
 func InitTask(platform constant.TaskPlatform, relayInfo *commonRelay.RelayInfo) *Task {
 	t := &Task{
 		UserId:     relayInfo.UserId,
+		Group:      relayInfo.UsingGroup,
 		SubmitTime: time.Now().Unix(),
 		Status:     TaskStatusNotStart,
 		Progress:   "0%",

relay/channel/adapter.go

@@ -53,5 +53,5 @@ type TaskAdaptor interface {
 }
 
 type OpenAIVideoConverter interface {
-	ConvertToOpenAIVideo(originTask *model.Task) (*relaycommon.OpenAIVideo, error)
+	ConvertToOpenAIVideo(originTask *model.Task) (*dto.OpenAIVideo, error)
 }

relay/channel/gemini/relay-gemini.go

@@ -211,7 +211,16 @@ func CovertGemini2OpenAI(c *gin.Context, textRequest dto.GeneralOpenAIRequest, i
 			// eg. {"google":{"thinking_config":{"thinking_budget":5324,"include_thoughts":true}}}
 			if googleBody, ok := extraBody["google"].(map[string]interface{}); ok {
 				adaptorWithExtraBody = true
+				// check error param name like thinkingConfig, should be thinking_config
+				if _, hasErrorParam := googleBody["thinkingConfig"]; hasErrorParam {
+					return nil, errors.New("extra_body.google.thinkingConfig is not supported, use extra_body.google.thinking_config instead")
+				}
+
 				if thinkingConfig, ok := googleBody["thinking_config"].(map[string]interface{}); ok {
+					// check error param name like thinkingBudget, should be thinking_budget
+					if _, hasErrorParam := thinkingConfig["thinkingBudget"]; hasErrorParam {
+						return nil, errors.New("extra_body.google.thinking_config.thinkingBudget is not supported, use extra_body.google.thinking_config.thinking_budget instead")
+					}
 					if budget, ok := thinkingConfig["thinking_budget"].(float64); ok {
 						budgetInt := int(budget)
 						geminiRequest.GenerationConfig.ThinkingConfig = &dto.GeminiThinkingConfig{

relay/channel/task/jimeng/adaptor.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/hmac"
 	"crypto/sha256"
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -64,6 +65,11 @@ type responseTask struct {
 	TimeElapsed string `json:"time_elapsed"`
 }
 
+const (
+	// 即梦限制单个文件最大4.7MB https://www.volcengine.com/docs/85621/1747301
+	MaxFileSize int64 = 4*1024*1024 + 700*1024 // 4.7MB (4MB + 724KB)
+)
+
 // ============================
 // Adaptor implementation
 // ============================
@@ -89,7 +95,6 @@ func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
 
 // ValidateRequestAndSetAction parses body, validates fields and sets default action.
 func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *dto.TaskError) {
-	// Accept only POST /v1/video/generations as "generate" action.
 	return relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionGenerate)
 }
 
@@ -113,13 +118,49 @@ func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info
 	return nil
 }
 
-// BuildRequestBody converts request into Jimeng specific format.
 func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
 	v, exists := c.Get("task_request")
 	if !exists {
 		return nil, fmt.Errorf("request not found in context")
 	}
-	req := v.(relaycommon.TaskSubmitReq)
+	req, ok := v.(relaycommon.TaskSubmitReq)
+	if !ok {
+		return nil, fmt.Errorf("invalid request type in context")
+	}
+	// 支持openai sdk的图片上传方式
+	if mf, err := c.MultipartForm(); err == nil {
+		if files, exists := mf.File["input_reference"]; exists && len(files) > 0 {
+			if len(files) == 1 {
+				info.Action = constant.TaskActionGenerate
+			} else if len(files) > 1 {
+				info.Action = constant.TaskActionFirstTailGenerate
+			}
+
+			// 将上传的文件转换为base64格式
+			var images []string
+
+			for _, fileHeader := range files {
+				// 检查文件大小
+				if fileHeader.Size > MaxFileSize {
+					return nil, fmt.Errorf("文件 %s 大小超过限制，最大允许 %d MB", fileHeader.Filename, MaxFileSize/(1024*1024))
+				}
+
+				file, err := fileHeader.Open()
+				if err != nil {
+					continue
+				}
+				fileBytes, err := io.ReadAll(file)
+				file.Close()
+				if err != nil {
+					continue
+				}
+				// 将文件内容转换为base64
+				base64Str := base64.StdEncoding.EncodeToString(fileBytes)
+				images = append(images, base64Str)
+			}
+			req.Images = images
+		}
+	}
 
 	body, err := a.convertToRequestPayload(&req)
 	if err != nil {
@@ -158,7 +199,7 @@ func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *rela
 		return
 	}
 
-	ov := relaycommon.NewOpenAIVideo()
+	ov := dto.NewOpenAIVideo()
 	ov.ID = jResp.Data.TaskID
 	ov.TaskID = jResp.Data.TaskID
 	ov.CreatedAt = time.Now().Unix()
@@ -364,10 +405,10 @@ func (a *TaskAdaptor) convertToRequestPayload(req *relaycommon.TaskSubmitReq) (*
 	// 即梦视频3.0 ReqKey转换
 	// https://www.volcengine.com/docs/85621/1792707
 	if strings.Contains(r.ReqKey, "jimeng_v30") {
-		if len(r.ImageUrls) > 1 {
+		if len(req.Images) > 1 {
 			// 多张图片：首尾帧生成
 			r.ReqKey = strings.Replace(r.ReqKey, "jimeng_v30", "jimeng_i2v_first_tail_v30", 1)
-		} else if len(r.ImageUrls) == 1 {
+		} else if len(req.Images) == 1 {
 			// 单张图片：图生视频
 			r.ReqKey = strings.Replace(r.ReqKey, "jimeng_v30", "jimeng_i2v_first_v30", 1)
 		} else {
@@ -405,13 +446,13 @@ func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, e
 	return &taskResult, nil
 }
 
-func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) (*relaycommon.OpenAIVideo, error) {
+func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) (*dto.OpenAIVideo, error) {
 	var jimengResp responseTask
 	if err := json.Unmarshal(originTask.Data, &jimengResp); err != nil {
 		return nil, errors.Wrap(err, "unmarshal jimeng task data failed")
 	}
 
-	openAIVideo := relaycommon.NewOpenAIVideo()
+	openAIVideo := dto.NewOpenAIVideo()
 	openAIVideo.ID = originTask.TaskID
 	openAIVideo.Status = originTask.Status.ToVideoStatus()
 	openAIVideo.SetProgressStr(originTask.Progress)
@@ -420,7 +461,7 @@ func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) (*relaycommon
 	openAIVideo.CompletedAt = originTask.UpdatedAt
 
 	if jimengResp.Code != 10000 {
-		openAIVideo.Error = &relaycommon.OpenAIVideoError{
+		openAIVideo.Error = &dto.OpenAIVideoError{
 			Message: jimengResp.Message,
 			Code:    fmt.Sprintf("%d", jimengResp.Code),
 		}

relay/channel/task/kling/adaptor.go

@@ -188,7 +188,7 @@ func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *rela
 		taskErr = service.TaskErrorWrapperLocal(fmt.Errorf(kResp.Message), "task_failed", http.StatusBadRequest)
 		return
 	}
-	ov := relaycommon.NewOpenAIVideo()
+	ov := dto.NewOpenAIVideo()
 	ov.ID = kResp.Data.TaskId
 	ov.TaskID = kResp.Data.TaskId
 	ov.CreatedAt = time.Now().Unix()
@@ -367,13 +367,13 @@ func isNewAPIRelay(apiKey string) bool {
 	return strings.HasPrefix(apiKey, "sk-")
 }
 
-func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) (*relaycommon.OpenAIVideo, error) {
+func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) (*dto.OpenAIVideo, error) {
 	var klingResp responsePayload
 	if err := json.Unmarshal(originTask.Data, &klingResp); err != nil {
 		return nil, errors.Wrap(err, "unmarshal kling task data failed")
 	}
 
-	openAIVideo := relaycommon.NewOpenAIVideo()
+	openAIVideo := dto.NewOpenAIVideo()
 	openAIVideo.ID = originTask.TaskID
 	openAIVideo.Status = originTask.Status.ToVideoStatus()
 	openAIVideo.SetProgressStr(originTask.Progress)
@@ -391,7 +391,7 @@ func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) (*relaycommon
 	}
 
 	if klingResp.Code != 0 && klingResp.Message != "" {
-		openAIVideo.Error = &relaycommon.OpenAIVideoError{
+		openAIVideo.Error = &dto.OpenAIVideoError{
 			Message: klingResp.Message,
 			Code:    fmt.Sprintf("%d", klingResp.Code),
 		}

relay/channel/task/sora/adaptor.go

@@ -186,8 +186,8 @@ func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, e
 	return &taskResult, nil
 }
 
-func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) (*relaycommon.OpenAIVideo, error) {
-	openAIVideo := &relaycommon.OpenAIVideo{}
+func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) (*dto.OpenAIVideo, error) {
+	openAIVideo := &dto.OpenAIVideo{}
 	err := json.Unmarshal(task.Data, openAIVideo)
 	if err != nil {
 		return nil, errors.Wrap(err, "unmarshal to OpenAIVideo failed")

relay/channel/task/vidu/adaptor.go

@@ -155,7 +155,7 @@ func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *rela
 		return
 	}
 
-	ov := relaycommon.NewOpenAIVideo()
+	ov := dto.NewOpenAIVideo()
 	ov.ID = vResp.TaskId
 	ov.TaskID = vResp.TaskId
 	ov.CreatedAt = time.Now().Unix()
@@ -263,13 +263,13 @@ func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, e
 	return taskInfo, nil
 }
 
-func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) (*relaycommon.OpenAIVideo, error) {
+func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) (*dto.OpenAIVideo, error) {
 	var viduResp taskResultResponse
 	if err := json.Unmarshal(originTask.Data, &viduResp); err != nil {
 		return nil, errors.Wrap(err, "unmarshal vidu task data failed")
 	}
 
-	openAIVideo := relaycommon.NewOpenAIVideo()
+	openAIVideo := dto.NewOpenAIVideo()
 	openAIVideo.ID = originTask.TaskID
 	openAIVideo.Status = originTask.Status.ToVideoStatus()
 	openAIVideo.SetProgressStr(originTask.Progress)
@@ -281,7 +281,7 @@ func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) (*relaycommon
 	}
 
 	if viduResp.State == "failed" && viduResp.ErrCode != "" {
-		openAIVideo.Error = &relaycommon.OpenAIVideoError{
+		openAIVideo.Error = &dto.OpenAIVideoError{
 			Message: viduResp.ErrCode,
 			Code:    viduResp.ErrCode,
 		}