mirror of
https://github.com/hanzoai/team.git
synced 2026-09-30 01:57:35 +00:00
zeekay
8feb24ea1f
The server module named github.com/hanzoai/team/server/v8 — a path that resolves to the retired workspace's repository, not this one — while server/public still published github.com/mattermost/mattermost/server/public. Both are github.com/hanzoteam/server/... now, which is where this code actually lives. That second half is what made the binary unbootable. Upstream's public path stayed claimed by upstream, mattermost-plugin-ai imported it, and the build linked public/model twice; public/plugin's gob.Register then ran twice under one name and panicked at init, before main. Reverting the rename traded the brand for a working binary. Rebranding the plugin alongside it needs no such trade: the agents fork is github.com/hanzoteam/agents/v2 against our public module, so no module named mattermost/mattermost is in the graph at all. Measured: `go list -deps ./cmd/mattermost` names exactly one public/model, and it is ours.
163 lines
4.4 KiB
Go
163 lines
4.4 KiB
Go
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
// See LICENSE.txt for license information.
|
|
|
|
package extract_content
|
|
|
|
import (
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/hanzoteam/server/server/public/model"
|
|
"github.com/hanzoteam/server/server/public/shared/mlog"
|
|
"github.com/hanzoteam/server/server/public/shared/request"
|
|
"github.com/hanzoteam/server/server/v8/channels/jobs"
|
|
"github.com/hanzoteam/server/server/v8/channels/store"
|
|
)
|
|
|
|
const (
|
|
catchupJobDataKey = "catchup"
|
|
catchupLookback = 24 * time.Hour
|
|
catchupBatchSize = 1000
|
|
)
|
|
|
|
var ignoredFiles = map[string]bool{
|
|
"png": true, "jpg": true, "jpeg": true, "gif": true, "wmv": true,
|
|
"mpg": true, "mpeg": true, "mp3": true, "mp4": true, "ogg": true,
|
|
"ogv": true, "mov": true, "apk": true, "svg": true, "webm": true,
|
|
"mkv": true,
|
|
}
|
|
|
|
type AppIface interface {
|
|
ExtractContentFromFileInfo(rctx request.CTX, fileInfo *model.FileInfo) error
|
|
}
|
|
|
|
func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store) *jobs.SimpleWorker {
|
|
const workerName = "ExtractContent"
|
|
|
|
isEnabled := func(cfg *model.Config) bool {
|
|
return true
|
|
}
|
|
execute := func(logger mlog.LoggerIFace, job *model.Job) error {
|
|
jobServer.HandleJobPanic(logger, job)
|
|
|
|
if job.Data[catchupJobDataKey] == "true" {
|
|
return runCatchupExtraction(logger, job, jobServer, app, store)
|
|
}
|
|
return runRangeExtraction(logger, job, jobServer, app, store)
|
|
}
|
|
worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
|
|
return worker
|
|
}
|
|
|
|
func runCatchupExtraction(logger mlog.LoggerIFace, job *model.Job, jobServer *jobs.JobServer, app AppIface, store store.Store) error {
|
|
cursor := model.GetMillis() - catchupLookback.Milliseconds()
|
|
|
|
var nFiles int
|
|
var nErrs int
|
|
for {
|
|
opts := model.GetFileInfosOptions{
|
|
Since: cursor,
|
|
SortBy: model.FileinfoSortByCreated,
|
|
IncludeDeleted: false,
|
|
OnlyEmptyContent: true,
|
|
}
|
|
fileInfos, err := store.FileInfo().GetWithOptions(0, catchupBatchSize, &opts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(fileInfos) == 0 {
|
|
break
|
|
}
|
|
|
|
for _, fileInfo := range fileInfos {
|
|
if ignoredFiles[fileInfo.Extension] {
|
|
continue
|
|
}
|
|
|
|
logger.Debug("Extracting file", mlog.String("filename", fileInfo.Name), mlog.String("filepath", fileInfo.Path))
|
|
|
|
err = app.ExtractContentFromFileInfo(request.EmptyContext(logger), fileInfo)
|
|
if err != nil {
|
|
logger.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id))
|
|
nErrs++
|
|
}
|
|
nFiles++
|
|
}
|
|
|
|
lastFileInfo := fileInfos[len(fileInfos)-1]
|
|
cursor = lastFileInfo.CreateAt + 1
|
|
|
|
if len(fileInfos) < catchupBatchSize {
|
|
break
|
|
}
|
|
}
|
|
|
|
job.Data["errors"] = strconv.Itoa(nErrs)
|
|
job.Data["processed"] = strconv.Itoa(nFiles)
|
|
|
|
if err := jobServer.UpdateInProgressJobData(job); err != nil {
|
|
logger.Error("Worker: Failed to update job data", mlog.Err(err))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func runRangeExtraction(logger mlog.LoggerIFace, job *model.Job, jobServer *jobs.JobServer, app AppIface, store store.Store) error {
|
|
var err error
|
|
var fromTS int64
|
|
var toTS int64 = model.GetMillis()
|
|
if fromStr, ok := job.Data["from"]; ok {
|
|
if fromTS, err = strconv.ParseInt(fromStr, 10, 64); err != nil {
|
|
return err
|
|
}
|
|
fromTS *= 1000
|
|
}
|
|
if toStr, ok := job.Data["to"]; ok {
|
|
if toTS, err = strconv.ParseInt(toStr, 10, 64); err != nil {
|
|
return err
|
|
}
|
|
toTS *= 1000
|
|
}
|
|
|
|
var nFiles int
|
|
var nErrs int
|
|
for {
|
|
opts := model.GetFileInfosOptions{
|
|
Since: fromTS,
|
|
SortBy: model.FileinfoSortByCreated,
|
|
IncludeDeleted: false,
|
|
}
|
|
fileInfos, err := store.FileInfo().GetWithOptions(0, catchupBatchSize, &opts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(fileInfos) == 0 {
|
|
break
|
|
}
|
|
for _, fileInfo := range fileInfos {
|
|
if !ignoredFiles[fileInfo.Extension] {
|
|
logger.Debug("Extracting file", mlog.String("filename", fileInfo.Name), mlog.String("filepath", fileInfo.Path))
|
|
|
|
err = app.ExtractContentFromFileInfo(request.EmptyContext(logger), fileInfo)
|
|
if err != nil {
|
|
logger.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id))
|
|
nErrs++
|
|
}
|
|
nFiles++
|
|
}
|
|
}
|
|
lastFileInfo := fileInfos[len(fileInfos)-1]
|
|
if lastFileInfo.CreateAt > toTS {
|
|
break
|
|
}
|
|
fromTS = lastFileInfo.CreateAt + 1
|
|
}
|
|
|
|
job.Data["errors"] = strconv.Itoa(nErrs)
|
|
job.Data["processed"] = strconv.Itoa(nFiles)
|
|
|
|
if err := jobServer.UpdateInProgressJobData(job); err != nil {
|
|
logger.Error("Worker: Failed to update job data", mlog.Err(err))
|
|
}
|
|
return nil
|
|
}
|