Files
ai/txt/markitdown.go
zeekay c1c8c6f530 chore: OSS attribution — restore casibase/OpenAgent per-file copyright (Apache-2.0 §4c)
Casibase's upstream '// Copyright <yr> The OpenAgent Authors. All Rights
Reserved.' (and legacy 'The Casibase Authors' on older files) was stripped
from every derived source file and replaced with a Hanzo-only line. This
re-adds it as a '// Portions Copyright ...' line beneath the Hanzo line on
the 573 casibase-derived files, using each file's real upstream year/holder.
Net-new Hanzo files are untouched. Nothing deleted. LICENSE + NOTICE were
already correct.
2026-07-02 13:02:21 -07:00

88 lines
2.2 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
// Portions Copyright 2025 The OpenAgent Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package txt
import (
"bytes"
"fmt"
"io"
"os"
"os/exec"
"unicode/utf8"
"github.com/hanzoai/ai/i18n"
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/transform"
)
var markitdownExists bool
func init() {
if _, err := exec.LookPath("markitdown"); err == nil {
markitdownExists = true
} else {
markitdownExists = false
}
}
func gbkToUtf8(gbkData []byte) (string, error) {
reader := transform.NewReader(bytes.NewReader(gbkData), simplifiedchinese.GBK.NewDecoder())
data, err := io.ReadAll(reader)
if err != nil {
return "", err
}
return string(data), nil
}
func GetTextFromMarkitdown(path string, lang string) (string, error) {
if !markitdownExists {
return "", fmt.Errorf("%s", i18n.Translate(lang, "txt:GetTextFromMarkitdown() error, markitdown does not exist"))
}
// Open the input file
file, err := os.Open(path)
if err != nil {
return "", fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "txt:GetTextFromMarkitdown() error, failed to open file: %v"), err))
}
defer file.Close()
cmd := exec.Command("markitdown")
cmd.Stdin = file
var out, stderr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &stderr
err = cmd.Run()
if err != nil {
return "", fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "txt:GetTextFromMarkitdown() error, markitdown failed to run: %v: %s"), err, stderr.String()))
}
outputBytes := out.Bytes()
if utf8.Valid(outputBytes) {
return string(outputBytes), nil
}
utf8Output, err := gbkToUtf8(outputBytes)
if err != nil {
return "", err
}
return utf8Output, nil
}