mirror of
https://github.com/QuantumNous/new-api.git
synced 2026-04-19 11:58:38 +00:00
feat: add ali wan video (#2141)
* feat: add ali wan video * refactor: use same UnmarshalBodyReusable * feat: enhance request body metadata * feat: opt wan convertToOpenAIVideo * feat: add wan support other param via json metadata * refactor: remove unused code * fix ali --------- Co-authored-by: feitianbubu <feitianbubu@qq.com>
This commit is contained in:
Seefs
360
relay/channel/task/ali/adaptor.go
Normal file
360
relay/channel/task/ali/adaptor.go
Normal file
@@ -0,0 +1,360 @@
|
||||
package ali
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ============================
|
||||
// Request / Response structures
|
||||
// ============================
|
||||
|
||||
// AliVideoRequest 阿里通义万相视频生成请求
|
||||
type AliVideoRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input AliVideoInput `json:"input"`
|
||||
Parameters *AliVideoParameters `json:"parameters,omitempty"`
|
||||
}
|
||||
|
||||
// AliVideoInput 视频输入参数
|
||||
type AliVideoInput struct {
|
||||
Prompt string `json:"prompt,omitempty"` // 文本提示词
|
||||
ImgURL string `json:"img_url,omitempty"` // 首帧图像URL或Base64（图生视频）
|
||||
FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频）
|
||||
LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频）
|
||||
AudioURL string `json:"audio_url,omitempty"` // 音频URL（wan2.5支持）
|
||||
NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词
|
||||
Template string `json:"template,omitempty"` // 视频特效模板
|
||||
}
|
||||
|
||||
// AliVideoParameters 视频参数
|
||||
type AliVideoParameters struct {
|
||||
Resolution string `json:"resolution,omitempty"` // 分辨率: 480P/720P/1080P（图生视频、首尾帧生视频）
|
||||
Size string `json:"size,omitempty"` // 尺寸: 如 "832*480"（文生视频）
|
||||
Duration int `json:"duration,omitempty"` // 时长: 3-10秒
|
||||
PromptExtend bool `json:"prompt_extend,omitempty"` // 是否开启prompt智能改写
|
||||
Watermark bool `json:"watermark,omitempty"` // 是否添加水印
|
||||
Audio *bool `json:"audio,omitempty"` // 是否添加音频（wan2.5）
|
||||
Seed int `json:"seed,omitempty"` // 随机数种子
|
||||
}
|
||||
|
||||
// AliVideoResponse 阿里通义万相响应
|
||||
type AliVideoResponse struct {
|
||||
Output AliVideoOutput `json:"output"`
|
||||
RequestID string `json:"request_id"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Usage *AliUsage `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
// AliVideoOutput 输出信息
|
||||
type AliVideoOutput struct {
|
||||
TaskID string `json:"task_id"`
|
||||
TaskStatus string `json:"task_status"`
|
||||
SubmitTime string `json:"submit_time,omitempty"`
|
||||
ScheduledTime string `json:"scheduled_time,omitempty"`
|
||||
EndTime string `json:"end_time,omitempty"`
|
||||
OrigPrompt string `json:"orig_prompt,omitempty"`
|
||||
ActualPrompt string `json:"actual_prompt,omitempty"`
|
||||
VideoURL string `json:"video_url,omitempty"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
// AliUsage 使用统计
|
||||
type AliUsage struct {
|
||||
Duration int `json:"duration,omitempty"`
|
||||
VideoCount int `json:"video_count,omitempty"`
|
||||
SR int `json:"SR,omitempty"`
|
||||
}
|
||||
|
||||
type AliMetadata struct {
|
||||
// Input 相关
|
||||
AudioURL string `json:"audio_url,omitempty"` // 音频URL
|
||||
ImgURL string `json:"img_url,omitempty"` // 图片URL（图生视频）
|
||||
FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频）
|
||||
LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频）
|
||||
NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词
|
||||
Template string `json:"template,omitempty"` // 视频特效模板
|
||||
|
||||
// Parameters 相关
|
||||
Resolution *string `json:"resolution,omitempty"` // 分辨率: 480P/720P/1080P
|
||||
Size *string `json:"size,omitempty"` // 尺寸: 如 "832*480"
|
||||
Duration *int `json:"duration,omitempty"` // 时长
|
||||
PromptExtend *bool `json:"prompt_extend,omitempty"` // 是否开启prompt智能改写
|
||||
Watermark *bool `json:"watermark,omitempty"` // 是否添加水印
|
||||
Audio *bool `json:"audio,omitempty"` // 是否添加音频
|
||||
Seed *int `json:"seed,omitempty"` // 随机数种子
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Adaptor implementation
|
||||
// ============================
|
||||
|
||||
type TaskAdaptor struct {
|
||||
ChannelType int
|
||||
apiKey string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
a.baseURL = info.ChannelBaseUrl
|
||||
a.apiKey = info.ApiKey
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *dto.TaskError) {
|
||||
// 阿里通义万相支持 JSON 格式，不使用 multipart
|
||||
return relaycommon.ValidateMultipartDirect(c, info)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
|
||||
return fmt.Sprintf("%s/api/v1/services/aigc/video-generation/video-synthesis", a.baseURL), nil
|
||||
}
|
||||
|
||||
// BuildRequestHeader sets required headers for Ali API
|
||||
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
|
||||
req.Header.Set("Authorization", "Bearer "+a.apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-DashScope-Async", "enable") // 阿里异步任务必须设置
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
var taskReq relaycommon.TaskSubmitReq
|
||||
if err := common.UnmarshalBodyReusable(c, &taskReq); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal_task_request_failed")
|
||||
}
|
||||
aliReq := a.convertToAliRequest(taskReq)
|
||||
|
||||
bodyBytes, err := common.Marshal(aliReq)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "marshal_ali_request_failed")
|
||||
}
|
||||
|
||||
return bytes.NewReader(bodyBytes), nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) convertToAliRequest(req relaycommon.TaskSubmitReq) *AliVideoRequest {
|
||||
aliReq := &AliVideoRequest{
|
||||
Model: req.Model,
|
||||
Input: AliVideoInput{
|
||||
Prompt: req.Prompt,
|
||||
ImgURL: req.InputReference,
|
||||
},
|
||||
Parameters: &AliVideoParameters{
|
||||
PromptExtend: true, // 默认开启智能改写
|
||||
Watermark: false,
|
||||
},
|
||||
}
|
||||
|
||||
// 处理分辨率映射
|
||||
if req.Size != "" {
|
||||
resolution := strings.ToUpper(req.Size)
|
||||
// 支持 480p, 720p, 1080p 或 480P, 720P, 1080P
|
||||
if !strings.HasSuffix(resolution, "P") {
|
||||
resolution = resolution + "P"
|
||||
}
|
||||
aliReq.Parameters.Resolution = resolution
|
||||
} else {
|
||||
// 根据模型设置默认分辨率
|
||||
if strings.HasPrefix(req.Model, "wan2.5") {
|
||||
aliReq.Parameters.Resolution = "1080P"
|
||||
} else if strings.HasPrefix(req.Model, "wan2.2-i2v-flash") {
|
||||
aliReq.Parameters.Resolution = "720P"
|
||||
} else if strings.HasPrefix(req.Model, "wan2.2-i2v-plus") {
|
||||
aliReq.Parameters.Resolution = "1080P"
|
||||
} else {
|
||||
aliReq.Parameters.Resolution = "720P"
|
||||
}
|
||||
}
|
||||
|
||||
// 处理时长
|
||||
if req.Duration > 0 {
|
||||
aliReq.Parameters.Duration = req.Duration
|
||||
} else {
|
||||
aliReq.Parameters.Duration = 5 // 默认5秒
|
||||
}
|
||||
|
||||
// 从 metadata 中提取额外参数
|
||||
if req.Metadata != nil {
|
||||
if metadataBytes, err := common.Marshal(req.Metadata); err == nil {
|
||||
_ = common.Unmarshal(metadataBytes, aliReq)
|
||||
}
|
||||
}
|
||||
|
||||
return aliReq
|
||||
}
|
||||
|
||||
// DoRequest delegates to common helper
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
// DoResponse handles upstream response
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *dto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
// 解析阿里响应
|
||||
var aliResp AliVideoResponse
|
||||
if err := common.Unmarshal(responseBody, &aliResp); err != nil {
|
||||
taskErr = service.TaskErrorWrapper(errors.Wrapf(err, "body: %s", responseBody), "unmarshal_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// 检查错误
|
||||
if aliResp.Code != "" {
|
||||
taskErr = service.TaskErrorWrapper(fmt.Errorf("%s: %s", aliResp.Code, aliResp.Message), "ali_api_error", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
if aliResp.Output.TaskID == "" {
|
||||
taskErr = service.TaskErrorWrapper(fmt.Errorf("task_id is empty"), "invalid_response", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// 转换为 OpenAI 格式响应
|
||||
openAIResp := dto.NewOpenAIVideo()
|
||||
openAIResp.ID = aliResp.Output.TaskID
|
||||
openAIResp.Model = c.GetString("model")
|
||||
if openAIResp.Model == "" && info != nil {
|
||||
openAIResp.Model = info.OriginModelName
|
||||
}
|
||||
openAIResp.Status = convertAliStatus(aliResp.Output.TaskStatus)
|
||||
openAIResp.CreatedAt = common.GetTimestamp()
|
||||
|
||||
// 返回 OpenAI 格式
|
||||
c.JSON(http.StatusOK, openAIResp)
|
||||
|
||||
return aliResp.Output.TaskID, responseBody, nil
|
||||
}
|
||||
|
||||
// FetchTask 查询任务状态
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any) (*http.Response, error) {
|
||||
taskID, ok := body["task_id"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid task_id")
|
||||
}
|
||||
|
||||
uri := fmt.Sprintf("%s/api/v1/tasks/%s", baseUrl, taskID)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, uri, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
|
||||
return service.GetHttpClient().Do(req)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return ModelList
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetChannelName() string {
|
||||
return ChannelName
|
||||
}
|
||||
|
||||
// ParseTaskResult 解析任务结果
|
||||
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
|
||||
var aliResp AliVideoResponse
|
||||
if err := common.Unmarshal(respBody, &aliResp); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal task result failed")
|
||||
}
|
||||
|
||||
taskResult := relaycommon.TaskInfo{
|
||||
Code: 0,
|
||||
}
|
||||
|
||||
// 状态映射
|
||||
switch aliResp.Output.TaskStatus {
|
||||
case "PENDING":
|
||||
taskResult.Status = model.TaskStatusQueued
|
||||
case "RUNNING":
|
||||
taskResult.Status = model.TaskStatusInProgress
|
||||
case "SUCCEEDED":
|
||||
taskResult.Status = model.TaskStatusSuccess
|
||||
// 阿里直接返回视频URL，不需要额外的代理端点
|
||||
taskResult.Url = aliResp.Output.VideoURL
|
||||
case "FAILED", "CANCELED", "UNKNOWN":
|
||||
taskResult.Status = model.TaskStatusFailure
|
||||
if aliResp.Message != "" {
|
||||
taskResult.Reason = aliResp.Message
|
||||
} else if aliResp.Output.Message != "" {
|
||||
taskResult.Reason = fmt.Sprintf("task failed, code: %s , message: %s", aliResp.Output.Code, aliResp.Output.Message)
|
||||
} else {
|
||||
taskResult.Reason = "task failed"
|
||||
}
|
||||
default:
|
||||
taskResult.Status = model.TaskStatusQueued
|
||||
}
|
||||
|
||||
return &taskResult, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) ([]byte, error) {
|
||||
var aliResp AliVideoResponse
|
||||
if err := common.Unmarshal(task.Data, &aliResp); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal ali response failed")
|
||||
}
|
||||
|
||||
openAIResp := dto.NewOpenAIVideo()
|
||||
openAIResp.ID = task.TaskID
|
||||
openAIResp.Status = convertAliStatus(aliResp.Output.TaskStatus)
|
||||
openAIResp.Model = task.Properties.OriginModelName
|
||||
openAIResp.SetProgressStr(task.Progress)
|
||||
openAIResp.CreatedAt = task.CreatedAt
|
||||
openAIResp.CompletedAt = task.UpdatedAt
|
||||
|
||||
// 设置视频URL（核心字段）
|
||||
openAIResp.SetMetadata("url", aliResp.Output.VideoURL)
|
||||
|
||||
// 错误处理
|
||||
if aliResp.Code != "" {
|
||||
openAIResp.Error = &dto.OpenAIVideoError{
|
||||
Code: aliResp.Code,
|
||||
Message: aliResp.Message,
|
||||
}
|
||||
} else if aliResp.Output.Code != "" {
|
||||
openAIResp.Error = &dto.OpenAIVideoError{
|
||||
Code: aliResp.Output.Code,
|
||||
Message: aliResp.Output.Message,
|
||||
}
|
||||
}
|
||||
|
||||
return common.Marshal(openAIResp)
|
||||
}
|
||||
|
||||
func convertAliStatus(aliStatus string) string {
|
||||
switch aliStatus {
|
||||
case "PENDING":
|
||||
return dto.VideoStatusQueued
|
||||
case "RUNNING":
|
||||
return dto.VideoStatusInProgress
|
||||
case "SUCCEEDED":
|
||||
return dto.VideoStatusCompleted
|
||||
case "FAILED", "CANCELED", "UNKNOWN":
|
||||
return dto.VideoStatusFailed
|
||||
default:
|
||||
return dto.VideoStatusUnknown
|
||||
}
|
||||
}
|
||||
11
relay/channel/task/ali/constants.go
Normal file
11
relay/channel/task/ali/constants.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package ali
|
||||
|
||||
var ModelList = []string{
|
||||
"wan2.5-i2v-preview", // 万相2.5 preview（有声视频）推荐
|
||||
"wan2.2-i2v-flash", // 万相2.2极速版（无声视频）
|
||||
"wan2.2-i2v-plus", // 万相2.2专业版（无声视频）
|
||||
"wanx2.1-i2v-plus", // 万相2.1专业版（无声视频）
|
||||
"wanx2.1-i2v-turbo", // 万相2.1极速版（无声视频）
|
||||
}
|
||||
|
||||
var ChannelName = "ali"
|
||||
Reference in New Issue
Block a user