|
|
@@ -1,10 +1,12 @@
|
|
|
package agent
|
|
|
|
|
|
import (
|
|
|
+ "bytes"
|
|
|
"context"
|
|
|
"encoding/csv"
|
|
|
"fmt"
|
|
|
"github.com/suyuan32/simple-admin-common/msg/errormsg"
|
|
|
+ "io"
|
|
|
"mime/multipart"
|
|
|
agentModel "wechat-api/ent/agent"
|
|
|
"wechat-api/hook/fastgpt"
|
|
|
@@ -14,6 +16,9 @@ import (
|
|
|
"wechat-api/internal/types"
|
|
|
|
|
|
"github.com/zeromicro/go-zero/core/logx"
|
|
|
+
|
|
|
+ "golang.org/x/text/encoding/simplifiedchinese"
|
|
|
+ "golang.org/x/text/transform"
|
|
|
)
|
|
|
|
|
|
type UploadAgentDataLogic struct {
|
|
|
@@ -31,14 +36,12 @@ func NewUploadAgentDataLogic(ctx context.Context, svcCtx *svc.ServiceContext) *U
|
|
|
|
|
|
func (l *UploadAgentDataLogic) UploadAgentData(req *types.UploadDataReq, file multipart.File, agentId uint64) (*types.BaseDataInfo, error) {
|
|
|
var count uint64 = 0
|
|
|
- //fmt.Printf("count=%d \n", count)
|
|
|
|
|
|
reader := csv.NewReader(file)
|
|
|
records, err := reader.ReadAll()
|
|
|
if err != nil {
|
|
|
return nil, err
|
|
|
}
|
|
|
- //fmt.Printf("records=%v \n", records)
|
|
|
|
|
|
agent, err := l.svcCtx.DB.Agent.Query().Where(agentModel.ID(agentId)).Only(l.ctx)
|
|
|
if err != nil {
|
|
|
@@ -51,10 +54,6 @@ func (l *UploadAgentDataLogic) UploadAgentData(req *types.UploadDataReq, file mu
|
|
|
|
|
|
qas := make([]fastgpt.DataQuestion, 0, 100)
|
|
|
for idx, record := range records {
|
|
|
- // 第一行标题内容过滤
|
|
|
- //fmt.Printf("idx=%v, record=%v \n", idx, record)
|
|
|
- //fmt.Println(idx == 0, record[0] == "问题", record[1] == "答案")
|
|
|
- //fmt.Printf("record=%+v\n", record)
|
|
|
if idx == 0 && record[1] == "答案" {
|
|
|
continue
|
|
|
}
|
|
|
@@ -63,23 +62,32 @@ func (l *UploadAgentDataLogic) UploadAgentData(req *types.UploadDataReq, file mu
|
|
|
continue
|
|
|
}
|
|
|
|
|
|
+ fmt.Printf("转换前：question=%s, answer=%s \n", record[0], record[1])
|
|
|
+
|
|
|
+ var question, answer []byte
|
|
|
+ reader0 := transform.NewReader(bytes.NewReader([]byte(record[0])), simplifiedchinese.GBK.NewDecoder())
|
|
|
+ question, _ = io.ReadAll(reader0)
|
|
|
+
|
|
|
+ reader1 := transform.NewReader(bytes.NewReader([]byte(record[1])), simplifiedchinese.GBK.NewDecoder())
|
|
|
+ answer, _ = io.ReadAll(reader1)
|
|
|
+
|
|
|
+ fmt.Printf("转换后：question=%s, answer=%s \n", question, answer)
|
|
|
+
|
|
|
qas = append(qas, fastgpt.DataQuestion{
|
|
|
- Q: record[0],
|
|
|
- A: record[1],
|
|
|
+ Q: string(question),
|
|
|
+ A: string(answer),
|
|
|
})
|
|
|
|
|
|
length := len(qas)
|
|
|
if length > 0 && length%100 == 0 {
|
|
|
params.Data = qas
|
|
|
- //fmt.Printf("11 qas=%+v \n", qas)
|
|
|
+ //fmt.Printf("params=%+v\n", params)
|
|
|
response, err := fastgpt.CreateBulkData(¶ms)
|
|
|
if err != nil {
|
|
|
l.Logger.Errorf("batch insert data to fastgpt failed. collection=%s error=%s", agent.CollectionID, err.Error())
|
|
|
return nil, err
|
|
|
}
|
|
|
- //fmt.Printf("response=%+v \n", response)
|
|
|
count += response.Data.InsertLen
|
|
|
- //fmt.Printf("count=%d insert=%d \n", count, response.Data.InsertLen)
|
|
|
|
|
|
qas = make([]fastgpt.DataQuestion, 0, 100)
|
|
|
}
|
|
|
@@ -87,16 +95,13 @@ func (l *UploadAgentDataLogic) UploadAgentData(req *types.UploadDataReq, file mu
|
|
|
|
|
|
if len(qas) > 0 {
|
|
|
params.Data = qas
|
|
|
- //fmt.Printf("22 qas=%+v \n", qas)
|
|
|
response, err := fastgpt.CreateBulkData(¶ms)
|
|
|
if err != nil {
|
|
|
l.Logger.Errorf("batch insert data to fastgpt failed. collection=%s error=%s", agent.CollectionID, err.Error())
|
|
|
return nil, err
|
|
|
}
|
|
|
- //fmt.Printf("response=%+v \n", response)
|
|
|
- //fmt.Printf("count=%d insert=%d \n", count, response.Data.InsertLen)
|
|
|
count += response.Data.InsertLen
|
|
|
- //fmt.Printf("count=%d insert=%d \n", count, response.Data.InsertLen)
|
|
|
+ qas = make([]fastgpt.DataQuestion, 0, 100)
|
|
|
}
|
|
|
|
|
|
resp := &types.BaseDataInfo{}
|
