295 lines
8.3 KiB
Go
295 lines
8.3 KiB
Go
package log_reader
|
||
|
||
import (
|
||
"bytes"
|
||
"encoding/csv"
|
||
"encoding/json"
|
||
"errors"
|
||
"gitee.com/golang-module/carbon/v2"
|
||
"github.com/extrame/xls"
|
||
"github.com/xuri/excelize/v2"
|
||
"golang.org/x/text/encoding/simplifiedchinese"
|
||
"golang.org/x/text/transform"
|
||
"gorm.io/gorm"
|
||
"io/ioutil"
|
||
"log"
|
||
"os"
|
||
"path"
|
||
"path/filepath"
|
||
"strings"
|
||
"testData/global"
|
||
"testData/model"
|
||
"testData/utils"
|
||
)
|
||
|
||
func RuiSiLogReader(fileText *model.FileText) {
|
||
rows := make([][]string, 0)
|
||
if strings.ToLower(path.Ext(fileText.Name)) == ".csv" {
|
||
csvBytes, err := ioutil.ReadFile(fileText.Path)
|
||
if err != nil {
|
||
log.Println("open err:", err)
|
||
return
|
||
}
|
||
//解决读取csv中文乱码的问题
|
||
reader := csv.NewReader(transform.NewReader(bytes.NewReader(csvBytes), simplifiedchinese.GBK.NewDecoder()))
|
||
//reader := csv.NewReader(file)
|
||
reader.FieldsPerRecord = -1
|
||
csvRows, err := reader.ReadAll()
|
||
if err != nil {
|
||
log.Println("open err:", err)
|
||
return
|
||
}
|
||
rows = csvRows
|
||
} else if strings.ToLower(path.Ext(fileText.Name)) == ".xls" {
|
||
if xlFile, err := xls.Open(fileText.Path, "utf-8"); err == nil {
|
||
//第一个sheet
|
||
sheet := xlFile.GetSheet(0)
|
||
if sheet.MaxRow != 0 {
|
||
temp := make([][]string, 0)
|
||
for i := 0; i < int(sheet.MaxRow); i++ {
|
||
row := sheet.Row(i)
|
||
if row == nil {
|
||
continue
|
||
}
|
||
data := make([]string, 0)
|
||
if row.LastCol() > 0 {
|
||
for j := 0; j < row.LastCol(); j++ {
|
||
col := row.Col(j)
|
||
data = append(data, col)
|
||
}
|
||
temp = append(temp, data)
|
||
}
|
||
}
|
||
rows = append(rows, temp...)
|
||
} else {
|
||
log.Println("open err:", err)
|
||
return
|
||
}
|
||
}
|
||
} else if strings.ToLower(path.Ext(fileText.Name)) == ".xlsx" {
|
||
xlsxFile, err := excelize.OpenFile(fileText.Path)
|
||
if err != nil {
|
||
log.Println("open err:", err)
|
||
return
|
||
}
|
||
sheetName := xlsxFile.GetSheetName(0)
|
||
rows, err = xlsxFile.GetRows(sheetName)
|
||
if err != nil {
|
||
log.Println("xlsx读取数据失败:", err)
|
||
return
|
||
}
|
||
}
|
||
|
||
details := make(map[string]string)
|
||
newExcel := make([][]string, 0)
|
||
var title []string
|
||
titleInfoMap := make(map[string]model.DataInfo)
|
||
sBinMap := make(map[string]model.BinInfo)
|
||
sbinStartIndex, sbinEndIndex := -1, -1
|
||
titleIndex, unitIndex, limitLIndex, limitUIndex, dataIndex := -1, -1, -1, -1, -1
|
||
for i := 1; i < len(rows); i++ {
|
||
if rows[i][0] == "" {
|
||
continue
|
||
}
|
||
if i == len(rows)-1 && unitIndex == -1 {
|
||
log.Println("特殊文件格式,文件路径为:", fileText.Path)
|
||
return
|
||
}
|
||
|
||
// SBin HBin standard and relation
|
||
if sbinEndIndex < 0 {
|
||
if strings.Contains(rows[i][0], "SBin") {
|
||
if sbinStartIndex < 0 {
|
||
sbinStartIndex = i
|
||
}
|
||
s := rows[i][0]
|
||
if strings.Index(s, "]") != -1 {
|
||
sbinNumStartIndex := strings.Index(s, "[")
|
||
sbinNumEndIndex := strings.Index(s, "]")
|
||
var startIndex, endIndex int
|
||
for index := sbinNumEndIndex + 1; index < len(s); index++ {
|
||
if startIndex == 0 {
|
||
if s[index] != ' ' {
|
||
startIndex = index
|
||
}
|
||
} else {
|
||
if s[index] == ' ' || s[index] == '_' {
|
||
endIndex = index
|
||
break
|
||
}
|
||
}
|
||
}
|
||
sBinMap[s[sbinNumStartIndex+1:sbinNumEndIndex]] = model.BinInfo{
|
||
Name: s[startIndex:endIndex],
|
||
HBin: s[strings.LastIndex(s, " ")+1:],
|
||
}
|
||
}
|
||
} else {
|
||
if sbinStartIndex > 0 {
|
||
sbinEndIndex = i
|
||
continue
|
||
}
|
||
}
|
||
}
|
||
//if len(sBinMap) == 0 {
|
||
//}
|
||
if titleIndex < 0 {
|
||
// Log File Details
|
||
rows[i][0] = strings.ReplaceAll(rows[i][0], "：", ":")
|
||
splitIndex := strings.Index(rows[i][0], ":")
|
||
if splitIndex != -1 {
|
||
details[rows[i][0][:splitIndex]] = rows[i][0][splitIndex+1:]
|
||
}
|
||
// Data Title
|
||
if strings.Contains(rows[i][0], "SITE_NUM") {
|
||
titleIndex = i
|
||
title = rows[i]
|
||
continue
|
||
}
|
||
} else {
|
||
// Data Title's Unit or Limit
|
||
if dataIndex < 0 {
|
||
if unitIndex < 0 {
|
||
if strings.Contains(rows[i][0], "Unit") {
|
||
rows[i] = append(rows[i], utils.FillData(len(rows[titleIndex])-len(rows[i]))...)
|
||
unitIndex = i
|
||
continue
|
||
}
|
||
}
|
||
if limitLIndex < 0 {
|
||
if strings.Contains(rows[i][0], "LimitL") {
|
||
rows[i] = append(rows[i], utils.FillData(len(rows[titleIndex])-len(rows[i]))...)
|
||
limitLIndex = i
|
||
continue
|
||
}
|
||
}
|
||
if limitUIndex < 0 {
|
||
if strings.Contains(rows[i][0], "LimitU") {
|
||
rows[i] = append(rows[i], utils.FillData(len(rows[titleIndex])-len(rows[i]))...)
|
||
limitUIndex = i
|
||
continue
|
||
}
|
||
}
|
||
// Get Data Index
|
||
if unitIndex > 0 && limitLIndex > 0 && limitUIndex > 0 {
|
||
dataIndex = i
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if titleIndex != -1 {
|
||
// Save Title Info
|
||
for k, v := range rows[titleIndex] {
|
||
titleInfoMap[v] = model.DataInfo{
|
||
Unit: rows[unitIndex][k],
|
||
LimitL: rows[limitLIndex][k],
|
||
LimitU: rows[limitUIndex][k],
|
||
}
|
||
}
|
||
// Complete and Save Data
|
||
for i := dataIndex; i < len(rows); i++ {
|
||
if len(rows[i]) < len(rows[titleIndex]) {
|
||
rows[i] = append(rows[i], utils.FillData(len(rows[titleIndex])-len(rows[i]))...)
|
||
}
|
||
newExcel = append(newExcel, rows[i])
|
||
}
|
||
} else {
|
||
log.Println("特殊文件格式,文件路径为:", fileText.Path)
|
||
return
|
||
}
|
||
|
||
//if details["Test_Code"] == "" {
|
||
// details["Test_Code"] = step
|
||
//} else {
|
||
// details["Test_Code"] = details["Test_Code"][:2]
|
||
//}
|
||
if fileText.Factory == "ruisi" {
|
||
s := strings.Split(fileText.Name, "_")
|
||
details["PBI"] = s[0]
|
||
details["Device_Name"] = s[1]
|
||
details["Lot"] = s[2]
|
||
if strings.Contains(s[len(s)-1], "RT") {
|
||
details["Test_Code"] = "RT"
|
||
} else if strings.Contains(s[len(s)-1], "FT") {
|
||
details["Test_Code"] = "FT"
|
||
}
|
||
}
|
||
//strReader := transform.NewReader(bytes.NewReader([]byte(details["Device_Name"])), simplifiedchinese.GBK.NewDecoder())
|
||
//product, _ := ioutil.ReadAll(strReader)
|
||
//if string(product) == "" {
|
||
// product = []byte(fileText.ProductName)
|
||
//}
|
||
details["Program"] = details["Program"][strings.LastIndex(details["Program"], "\\")+1:]
|
||
sbinHbin, _ := json.Marshal(&sBinMap)
|
||
titleInfo, _ := json.Marshal(&titleInfoMap)
|
||
fileName := details["Device_Name"] + "_" + details["Lot"] + "_" + details["Sub_LotNo"] + "_" + details["Test_Code"] + ".csv"
|
||
var waferID string
|
||
//strings.ReplaceAll(fileText.Lot, ".", "-")
|
||
dirPath := filepath.Join("/testData/test/" + fileText.Factory)
|
||
utils.MakeDir(dirPath)
|
||
filePath := filepath.Join(dirPath, fileName)
|
||
var fileHandled *model.FileHandled
|
||
if _, err := os.Stat(filePath); err != nil {
|
||
if os.IsNotExist(err) {
|
||
newExcel = append([][]string{title}, newExcel...)
|
||
_, err = os.Create(filePath)
|
||
if err != nil {
|
||
log.Println("创建文件失败:", err)
|
||
return
|
||
}
|
||
if errors.Is(global.PostGreSQL.Where("name = ?", fileName).First(&fileHandled).Error, gorm.ErrRecordNotFound) {
|
||
global.PostGreSQL.Create(&model.FileHandled{
|
||
Name: fileName,
|
||
Path: filePath,
|
||
Size: "",
|
||
Product: details["Device_Name"],
|
||
Factory: fileText.Factory,
|
||
PBI: details["PBI"],
|
||
Step: details["Test_Code"],
|
||
Lot: details["Lot"],
|
||
SubBatch: details["Sub_LotNo"],
|
||
TestMachineModel: "STS8200",
|
||
TestMachine: details["Tester ID"],
|
||
TestProgram: details["Program"],
|
||
BeginningTime: details["Beginning Time"],
|
||
EndingTime: details["Ending Time"],
|
||
SbinHbin: string(sbinHbin),
|
||
TitleInfo: string(titleInfo),
|
||
WaferID: waferID,
|
||
})
|
||
}
|
||
}
|
||
}
|
||
newCsv, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
|
||
if err != nil {
|
||
log.Println("打开新Excel错误:", fileText.Path)
|
||
return
|
||
}
|
||
defer newCsv.Close()
|
||
writer := csv.NewWriter(newCsv)
|
||
defer writer.Flush()
|
||
err = writer.WriteAll(newExcel)
|
||
if err != nil {
|
||
log.Println("写入新Excel错误:", fileText.Path)
|
||
return
|
||
}
|
||
newFile, err := os.Stat(filePath)
|
||
if err != nil {
|
||
log.Println("获取新Excel信息:", fileText.Path)
|
||
return
|
||
}
|
||
if !carbon.Parse(fileHandled.BeginningTime).Gt(carbon.Parse(details["Beginning Time"])) {
|
||
details["Beginning Time"] = fileHandled.BeginningTime
|
||
}
|
||
if !carbon.Parse(details["Ending Time"]).Gt(carbon.Parse(fileHandled.EndingTime)) {
|
||
details["Ending Time"] = fileHandled.EndingTime
|
||
}
|
||
global.PostGreSQL.Model(&fileHandled).Where("name = ?", fileName).Updates(map[string]interface{}{
|
||
"beginning_time": details["Beginning Time"],
|
||
"ending_time": details["Ending Time"],
|
||
"size": utils.FormatFileSize(float64(newFile.Size())),
|
||
})
|
||
}
|