package agent import ( "context" "encoding/csv" "fmt" "github.com/suyuan32/simple-admin-common/msg/errormsg" "mime/multipart" agentModel "wechat-api/ent/agent" "wechat-api/hook/fastgpt" "wechat-api/internal/utils/dberrorhandler" "wechat-api/internal/svc" "wechat-api/internal/types" "github.com/zeromicro/go-zero/core/logx" ) type UploadAgentDataLogic struct { logx.Logger ctx context.Context svcCtx *svc.ServiceContext } func NewUploadAgentDataLogic(ctx context.Context, svcCtx *svc.ServiceContext) *UploadAgentDataLogic { return &UploadAgentDataLogic{ Logger: logx.WithContext(ctx), ctx: ctx, svcCtx: svcCtx} } func (l *UploadAgentDataLogic) UploadAgentData(req *types.UploadDataReq, file multipart.File, agentId uint64) (*types.BaseDataInfo, error) { var count uint64 reader := csv.NewReader(file) records, err := reader.ReadAll() if err != nil { return nil, err } agent, err := l.svcCtx.DB.Agent.Query().Where(agentModel.ID(agentId)).Only(l.ctx) if err != nil { return nil, dberrorhandler.DefaultEntError(l.Logger, err, req) } //TODO token := agent.CollectionID var params fastgpt.CreateBulkDataReq params.CollectionID = agent.CollectionID params.TrainingMode = "chunk" qas := make([]fastgpt.DataQuestion, 0, 100) for idx, record := range records { // 第一行标题内容过滤 //fmt.Printf("idx=%v, q=%v, a=%v \n", idx, record[0], record[1]) //fmt.Println(idx == 0, record[0] == "问题", record[1] == "答案") //fmt.Printf("record=%+v\n", record) if idx == 0 && record[1] == "答案" { continue } // 空内容过滤 if record[0] == "" || record[1] == "" { continue } qas = append(qas, fastgpt.DataQuestion{ Q: record[0], A: record[1], }) if idx%100 == 0 && len(qas) > 0 { params.Data = qas response, err := fastgpt.CreateBulkData(¶ms) if err != nil { l.Logger.Errorf("batch insert data to fastgpt failed. collection=%s error=%s", agent.CollectionID, err.Error()) return nil, err } count += response.Data.InsertLen qas = make([]fastgpt.DataQuestion, 0, 100) } } if len(qas) > 0 { params.Data = qas response, err := fastgpt.DiyCreateBulkData(token, ¶ms) if err != nil { l.Logger.Errorf("batch insert data to fastgpt failed. collection=%s error=%s", agent.CollectionID, err.Error()) return nil, err } count += response.Data.InsertLen } resp := &types.BaseDataInfo{} resp.Code = 0 resp.Msg = errormsg.Success resp.Data = fmt.Sprintf("upload %d rows", count) return resp, nil }