豆包支持chat支持图片询问

dao/im.go

@@ -23,7 +23,7 @@ type Message struct {
 	ToID       int    `gorm:"column:to_id"`            //接收者,可以为用户或者模型,如果为模型，则为模型id，根据type判断
 	Msg        string `gorm:"column:msg"`              //消息内容
 	FunctionID int    `gorm:"column:function_id"`      //功能id
-	Status     int    `gorm:"column:status"`           //0为未读，1为已读
+	Status     int    `gorm:"column:status"`           //0为未读，1为已读，3为图片消息，4为文件消息
 }
 
 func CreateSession(userID int, name string) (error, uint) {

handler/im.go

@@ -238,8 +238,12 @@ func doReceiveGenChatMessage(userId int, sessionID *uint, data *proto.WSMessageR
 	imContext := proto.IMParamContext{UserID: userId, SessionID: int(*sessionID), FunctionID: int(funcs[0].ID), ModelID: int(model.ID), Question: data.Msg, Channel: chanel, ModelType: model.Type}
 
 	var userMsgID uint
+	status := proto.MsgHasRead
+	if data.IsImage {
+		status = proto.UserToModelImageMsgType //图片类型
+	}
 	//将消息存入数据库
-	err, userMsgID = service.CreateMessage(proto.UserToModelMsgType, int(*sessionID), userId, int(model.ID), data.Msg, proto.MsgHasRead, int(funcs[0].ID))
+	err, userMsgID = service.CreateMessage(proto.UserToModelMsgType, int(*sessionID), userId, int(model.ID), data.Msg, status, int(funcs[0].ID))
 	log.Println("create user message id:", userMsgID)
 	if err != nil {
 		return err

proto/im.go

@@ -25,6 +25,7 @@ type WSMessageReq struct {
 	SessionID int    `json:"session_id"` //应用层会话id
 	ToID      int    `json:"to_id"`      //接收者id
 	ModelID   uint   `json:"model_id"`   //模型id
+	IsImage   bool   `json:"is_image"`   //是否为图片(图片消息)，如果是图片消息，则msg为图片消息结构为{"img_url":"","text":""}
 }
 
 type ModelParam struct {
@@ -75,3 +76,9 @@ type IMParamContext struct {
 	Question   string `json:"question"`    //问题
 	Channel    string `json:"channel"`     //消息队列
 }
+
+// 用户向模型发送图片的消息结构
+type UserImageMsg struct {
+	ImgUrl string `json:"img_url"` //图片url
+	Text   string `json:"text"`    //问题文本
+}

proto/status.go

@@ -99,6 +99,11 @@ const (
 	UserToModelMsgType = 3
 	//模型发到用户
 	ModelToUserMsgType = 4
+
+	//用户发送图片对话
+	UserToModelImageMsgType = 3
+	//模型发送文件对话
+	ModelToUserFileMsgType = 4
 )
 
 // 豆包返回的数据停止原因

service/doubao.go

@@ -194,14 +194,44 @@ func GetDouBaoSessionHistoryMsg(sessionID int, systemPrompt string, messages *[]
 		var message model.ChatCompletionMessage
 		if v.Type == proto.UserToModelMsgType {
 			message.Role = model.ChatMessageRoleUser
+			//用户消息为有图片类型
+			if v.Status == proto.UserToModelImageMsgType {
+				var imgMsg proto.UserImageMsg
+				err2 := json.Unmarshal([]byte(v.Msg), &imgMsg)
+				if err2 != nil {
+					log.Println("unmarshal user image message error:", err2)
+					return err2
+				}
+				//用户消息为图片类型
+				message.Content = &model.ChatCompletionMessageContent{
+					ListValue: []*model.ChatCompletionMessageContentPart{
+						{
+							Type: model.ChatCompletionMessageContentPartTypeText,
+							Text: imgMsg.Text,
+						},
+						{
+							Type: model.ChatCompletionMessageContentPartTypeImageURL,
+							ImageURL: &model.ChatMessageImageURL{
+								URL: imgMsg.ImgUrl,
+							},
+						},
+					},
+				}
+			} else {
+				//用户消息为文本类型
+				message.Content = &model.ChatCompletionMessageContent{
+					StringValue: volcengine.String(v.Msg),
+				}
+			}
 		} else if v.Type == proto.ModelToUserMsgType {
 			message.Role = model.ChatMessageRoleAssistant
+			message.Content = &model.ChatCompletionMessageContent{
+				StringValue: volcengine.String(v.Msg),
+			}
 		} else {
 			continue
 		}
-		message.Content = &model.ChatCompletionMessageContent{
+
-			StringValue: volcengine.String(v.Msg),
-		}
 		*messages = append(*messages, &message)
 	}
 	//添加本次请求消息(本次消息已在上面添加)