mirror of
https://github.com/hanzoai/team.git
synced 2026-09-30 01:57:35 +00:00
e3271eec4b
* Propagate context through Extractor.Extract Adds ctx context.Context to the Extractor interface and all implementations so callers can cancel in-flight extractions. pdfExtractor passes the context down to GetPlainText(ctx), which checks ctx.Done() at each page boundary. When extractWithTimeout fires (deadline exceeded or parent cancel), it cancels the derived context via defer cancel(), and the PDF goroutine stops at the next page rather than running to completion, eliminating the heap growth that caused OOM on complex documents. For non-context-aware extractors (docconv, plain, archive) the context is accepted but not yet acted on; they continue to run to completion on a detached goroutine after the timeout, as before. The call site in channels/app/file.go passes rctx.Context() so request cancellation (e.g. HTTP disconnect) also propagates. * Use new mattermost/pdf fork * Update notice reflecting the new fork * Fail archiveExtractor.Extract early on ctx errors * Bump mattermost/pdf to cd8a834041c4 --------- Co-authored-by: Mattermost Build <[email protected]>
57 lines
1.1 KiB
Go
57 lines
1.1 KiB
Go
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
// See LICENSE.txt for license information.
|
|
|
|
package docextractor
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
type plainExtractor struct{}
|
|
|
|
func (pe *plainExtractor) Name() string {
|
|
return "plainExtractor"
|
|
}
|
|
|
|
func (pe *plainExtractor) Match(filename string) bool {
|
|
return true
|
|
}
|
|
|
|
func (pe *plainExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
|
|
// This detects any visible character plus any whitespace
|
|
validRanges := append(unicode.GraphicRanges, unicode.White_Space)
|
|
|
|
runes := make([]byte, 1024)
|
|
total, err := r.Read(runes)
|
|
if err != nil && err != io.EOF {
|
|
return "", err
|
|
}
|
|
|
|
if total == 0 {
|
|
return "", nil
|
|
}
|
|
|
|
count := 0
|
|
for {
|
|
c, size := utf8.DecodeRune(runes[count:])
|
|
if !unicode.In(c, validRanges...) {
|
|
return "", nil
|
|
}
|
|
if size == 0 {
|
|
break
|
|
}
|
|
count += size
|
|
|
|
// subtract the max rune size to prevent accidentally splitted runes at the end of first 1024 bytes
|
|
if count > total-utf8.UTFMax {
|
|
break
|
|
}
|
|
}
|
|
|
|
text, _ := io.ReadAll(r)
|
|
return string(runes[0:total]) + string(text), nil
|
|
}
|