mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
Casibase's upstream '// Copyright <yr> The OpenAgent Authors. All Rights Reserved.' (and legacy 'The Casibase Authors' on older files) was stripped from every derived source file and replaced with a Hanzo-only line. This re-adds it as a '// Portions Copyright ...' line beneath the Hanzo line on the 573 casibase-derived files, using each file's real upstream year/holder. Net-new Hanzo files are untouched. Nothing deleted. LICENSE + NOTICE were already correct.
88 lines
2.2 KiB
Go
88 lines
2.2 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
// Portions Copyright 2025 The OpenAgent Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package txt
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"unicode/utf8"
|
|
|
|
"github.com/hanzoai/ai/i18n"
|
|
"golang.org/x/text/encoding/simplifiedchinese"
|
|
"golang.org/x/text/transform"
|
|
)
|
|
|
|
var markitdownExists bool
|
|
|
|
func init() {
|
|
if _, err := exec.LookPath("markitdown"); err == nil {
|
|
markitdownExists = true
|
|
} else {
|
|
markitdownExists = false
|
|
}
|
|
}
|
|
|
|
func gbkToUtf8(gbkData []byte) (string, error) {
|
|
reader := transform.NewReader(bytes.NewReader(gbkData), simplifiedchinese.GBK.NewDecoder())
|
|
data, err := io.ReadAll(reader)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return string(data), nil
|
|
}
|
|
|
|
func GetTextFromMarkitdown(path string, lang string) (string, error) {
|
|
if !markitdownExists {
|
|
return "", fmt.Errorf("%s", i18n.Translate(lang, "txt:GetTextFromMarkitdown() error, markitdown does not exist"))
|
|
}
|
|
|
|
// Open the input file
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return "", fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "txt:GetTextFromMarkitdown() error, failed to open file: %v"), err))
|
|
}
|
|
defer file.Close()
|
|
|
|
cmd := exec.Command("markitdown")
|
|
|
|
cmd.Stdin = file
|
|
|
|
var out, stderr bytes.Buffer
|
|
cmd.Stdout = &out
|
|
cmd.Stderr = &stderr
|
|
|
|
err = cmd.Run()
|
|
if err != nil {
|
|
return "", fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "txt:GetTextFromMarkitdown() error, markitdown failed to run: %v: %s"), err, stderr.String()))
|
|
}
|
|
|
|
outputBytes := out.Bytes()
|
|
if utf8.Valid(outputBytes) {
|
|
return string(outputBytes), nil
|
|
}
|
|
|
|
utf8Output, err := gbkToUtf8(outputBytes)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return utf8Output, nil
|
|
}
|