mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
Every address a request names now reaches the network through one client, github.com/hanzoai/outbound (proxy.Public), which judges the address at the dial, on the IP the socket connects to, and re-dials and re-judges every redirect. What it replaces failed three ways. A message href was checked by util.Fetchable and then fetched by a different client: the name was resolved twice, so a rebinding answer passed the check, and a redirect to 169.254.169.254 was followed unchecked. The check itself was net.IP's predicates, which admit NAT64 64:ff9b::a9fe:a9fe, 6to4, Teredo, 198.18/15, 240/4 and the TEST-NETs; proxy/public.go's routable admitted the same. And /v1/ai/rag/embed's url and /v1/ai/rag/ingest's files[].url had no check at all: a loopback or metadata URL was fetched, and a scheme-less string was read off this machine's disk. A tenant document now goes through txt.GetParsedTextFromAddress (http and https only, over proxy.Public); txt.GetParsedTextFromUrl is for addresses this system stored. The Go crawler fetches every page and robots.txt through proxy.Public, and the browser engine is handed only addresses outbound.Host finds public.
84 lines
2.7 KiB
Go
84 lines
2.7 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
// Portions Copyright 2023 The OpenAgent Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package txt
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"github.com/hanzoai/ai/proxy"
|
|
"github.com/hanzoai/ai/util"
|
|
)
|
|
|
|
// getTempFilePathFromUrl downloads a document this system stored to a temp file
|
|
// and returns its path. The caller removes the file; this closes the handle.
|
|
//
|
|
// It used to do neither. The handle stayed open for the life of the process, so
|
|
// every remote document parsed leaked a descriptor — and on Unix the caller's
|
|
// os.Remove does not reclaim the bytes while a descriptor is still open, so the
|
|
// disk went with it. A service parsing documents runs out of descriptors and then
|
|
// fails at everything that needs one, including accepting a connection.
|
|
//
|
|
// A copy that fails takes the file with it. Its path is never returned, so the
|
|
// caller has nothing to remove and would leave it behind.
|
|
func getTempFilePathFromUrl(url string) (string, error) {
|
|
buffer, err := util.DownloadFile(url)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
file, err := os.CreateTemp("", filepath.Base(url))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file.Close()
|
|
|
|
if _, err = io.Copy(file, buffer); err != nil {
|
|
_ = os.Remove(file.Name())
|
|
return "", err
|
|
}
|
|
|
|
return file.Name(), nil
|
|
}
|
|
|
|
// getTempFilePathFromAddress downloads the document at an address a request named
|
|
// to a temp file and returns its path, over proxy.Public — outbound's client — so
|
|
// an address on this network is refused at the dial and so is every redirect to
|
|
// one. The caller removes the file.
|
|
func getTempFilePathFromAddress(address string) (string, error) {
|
|
resp, err := proxy.Public.Get(address)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", fmt.Errorf("%s answered %s", address, resp.Status)
|
|
}
|
|
file, err := os.CreateTemp("", "address-*"+filepath.Ext(resp.Request.URL.Path))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file.Close()
|
|
if _, err = io.Copy(file, resp.Body); err != nil {
|
|
_ = os.Remove(file.Name())
|
|
return "", err
|
|
}
|
|
return file.Name(), nil
|
|
}
|