mirror of
https://github.com/hanzoai/team.git
synced 2026-09-30 01:57:35 +00:00
e3271eec4b
* Propagate context through Extractor.Extract Adds ctx context.Context to the Extractor interface and all implementations so callers can cancel in-flight extractions. pdfExtractor passes the context down to GetPlainText(ctx), which checks ctx.Done() at each page boundary. When extractWithTimeout fires (deadline exceeded or parent cancel), it cancels the derived context via defer cancel(), and the PDF goroutine stops at the next page rather than running to completion, eliminating the heap growth that caused OOM on complex documents. For non-context-aware extractors (docconv, plain, archive) the context is accepted but not yet acted on; they continue to run to completion on a detached goroutine after the timeout, as before. The call site in channels/app/file.go passes rctx.Context() so request cancellation (e.g. HTTP disconnect) also propagates. * Use new mattermost/pdf fork * Update notice reflecting the new fork * Fail archiveExtractor.Extract early on ctx errors * Bump mattermost/pdf to cd8a834041c4 --------- Co-authored-by: Mattermost Build <[email protected]>
91 lines
2.6 KiB
Go
91 lines
2.6 KiB
Go
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
// See LICENSE.txt for license information.
|
|
|
|
package docextractor
|
|
|
|
// MMPreview is a micro-service to convert from any libreoffice supported
|
|
// format into a PDF file, and then we use the regular pdf extractor to convert
|
|
// it into plain text.
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"path"
|
|
"strings"
|
|
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
type mmPreviewExtractor struct {
|
|
url string
|
|
secret string
|
|
pdfExtractor pdfExtractor
|
|
}
|
|
|
|
var mmpreviewSupportedExtensions = map[string]bool{
|
|
"ppt": true,
|
|
"odp": true,
|
|
"xls": true,
|
|
"xlsx": true,
|
|
"ods": true,
|
|
}
|
|
|
|
func newMMPreviewExtractor(url string, secret string, pdfExtractor pdfExtractor) *mmPreviewExtractor {
|
|
return &mmPreviewExtractor{url: url, secret: secret, pdfExtractor: pdfExtractor}
|
|
}
|
|
|
|
func (mpe *mmPreviewExtractor) Name() string {
|
|
return "mmPreviewExtractor"
|
|
}
|
|
|
|
func (mpe *mmPreviewExtractor) Match(filename string) bool {
|
|
extension := strings.TrimPrefix(path.Ext(filename), ".")
|
|
return mmpreviewSupportedExtensions[extension]
|
|
}
|
|
|
|
func (mpe *mmPreviewExtractor) Extract(ctx context.Context, filename string, file io.ReadSeeker, maxFileSize int64) (string, error) {
|
|
b, w, err := createMultipartFormData("file", filename, file)
|
|
if err != nil {
|
|
return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, "POST", mpe.url+"/toPDF", &b)
|
|
if err != nil {
|
|
return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
|
|
}
|
|
req.Header.Set("Content-Type", w.FormDataContentType())
|
|
if mpe.secret != "" {
|
|
req.Header.Add("Authentication", mpe.secret)
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != 200 {
|
|
return "", errors.New("Unable to generate file preview using mmpreview (The server has replied with an error)")
|
|
}
|
|
data, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return "", errors.Wrap(err, "unable to read the response from mmpreview")
|
|
}
|
|
return mpe.pdfExtractor.Extract(ctx, filename, bytes.NewReader(data), maxFileSize)
|
|
}
|
|
|
|
func createMultipartFormData(fieldName, fileName string, fileData io.ReadSeeker) (bytes.Buffer, *multipart.Writer, error) {
|
|
var b bytes.Buffer
|
|
var err error
|
|
w := multipart.NewWriter(&b)
|
|
var fw io.Writer
|
|
if fw, err = w.CreateFormFile(fieldName, fileName); err != nil {
|
|
return b, nil, err
|
|
}
|
|
if _, err = io.Copy(fw, fileData); err != nil {
|
|
return b, nil, err
|
|
}
|
|
w.Close()
|
|
return b, w, nil
|
|
}
|