mirror of
https://github.com/hanzoai/forge.git
synced 2026-10-30 15:00:39 +00:00
zeekayandHanzo Dev
aa87442fdd
Source files keep their SPDX-License-Identifier and Hanzo copyright line; the Gitea, Gogs and LFS Test Server copyrights they used to repeat now live once in NOTICE, alongside LICENSE. The header lint accepts any copyright line above the SPDX line, so it no longer demands an upstream one. Co-authored-by: Hanzo Dev <[email protected]>
248 lines
9.4 KiB
Go
248 lines
9.4 KiB
Go
// Copyright 2026 Hanzo AI, Inc. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package charset
|
|
|
|
import (
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/hanzoai/git/modules/setting"
|
|
"github.com/hanzoai/git/modules/test"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestMain(m *testing.M) {
|
|
setting.Repository.DetectedCharsetScore = map[string]int{}
|
|
for i, charset := range setting.Repository.DetectedCharsetsOrder {
|
|
setting.Repository.DetectedCharsetScore[strings.ToLower(charset)] = i
|
|
}
|
|
os.Exit(m.Run())
|
|
}
|
|
|
|
func TestMaybeRemoveBOM(t *testing.T) {
|
|
res := maybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
|
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
|
|
|
|
res = maybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
|
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
|
|
}
|
|
|
|
func TestToUTF8(t *testing.T) {
|
|
// Note: golang compiler seems so behave differently depending on the current
|
|
// locale, so some conversions might behave differently. For that reason, we don't
|
|
// depend on particular conversions but in expected behaviors.
|
|
|
|
res := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
|
|
assert.Equal(t, "ABC", string(res))
|
|
|
|
// "áéíóú"
|
|
res = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
|
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
|
|
|
|
// "áéíóú"
|
|
res = ToUTF8([]byte{
|
|
0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
|
|
0xc3, 0xba,
|
|
}, ConvertOpts{})
|
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
|
|
|
|
res = ToUTF8([]byte{
|
|
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
|
|
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
|
|
}, ConvertOpts{})
|
|
stringMustStartWith(t, "Hola,", res)
|
|
stringMustEndWith(t, "AAA.", res)
|
|
|
|
res = ToUTF8([]byte{
|
|
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
|
|
0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
|
|
}, ConvertOpts{})
|
|
stringMustStartWith(t, "Hola,", res)
|
|
stringMustEndWith(t, "AAA.", res)
|
|
|
|
res = ToUTF8([]byte{
|
|
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
|
|
0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
|
|
}, ConvertOpts{})
|
|
stringMustStartWith(t, "Hola,", res)
|
|
stringMustEndWith(t, "AAA.", res)
|
|
|
|
// Japanese (Shift-JIS)
|
|
// 日属秘ぞしちゅ。
|
|
res = ToUTF8([]byte{
|
|
0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
|
|
0xBF, 0x82, 0xE3, 0x81, 0x42,
|
|
}, ConvertOpts{})
|
|
assert.Equal(t, []byte{
|
|
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
|
|
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
|
|
}, res)
|
|
|
|
res = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
|
|
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
|
|
}
|
|
|
|
func TestToUTF8WithFallback(t *testing.T) {
|
|
// "ABC"
|
|
res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
|
|
assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
|
|
|
|
// "áéíóú"
|
|
res = ToUTF8WithFallback([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
|
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
|
|
|
|
// UTF8 BOM + "áéíóú"
|
|
res = ToUTF8WithFallback([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
|
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
|
|
|
|
// "Hola, así cómo ños"
|
|
res = ToUTF8WithFallback([]byte{
|
|
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
|
|
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73,
|
|
}, ConvertOpts{})
|
|
assert.Equal(t, []byte{
|
|
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63,
|
|
0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73,
|
|
}, res)
|
|
|
|
// "Hola, así cómo "
|
|
minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
|
|
|
|
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{})
|
|
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
|
|
assert.Equal(t, minmatch, res[0:len(minmatch)])
|
|
|
|
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{})
|
|
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
|
|
assert.Equal(t, minmatch, res[0:len(minmatch)])
|
|
|
|
// Japanese (Shift-JIS)
|
|
// "日属秘ぞしちゅ。"
|
|
res = ToUTF8WithFallback([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{})
|
|
assert.Equal(t, []byte{
|
|
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
|
|
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
|
|
}, res)
|
|
|
|
res = ToUTF8WithFallback([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
|
|
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
|
|
}
|
|
|
|
func TestToUTF8DropErrors(t *testing.T) {
|
|
// "ABC"
|
|
res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43})
|
|
assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
|
|
|
|
// "áéíóú"
|
|
res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
|
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
|
|
|
|
// UTF8 BOM + "áéíóú"
|
|
res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
|
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
|
|
|
|
// "Hola, así cómo ños"
|
|
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
|
|
assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73}, res[:8])
|
|
assert.Equal(t, []byte{0x73}, res[len(res)-1:])
|
|
|
|
// "Hola, así cómo "
|
|
minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
|
|
|
|
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
|
|
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
|
|
assert.Equal(t, minmatch, res[0:len(minmatch)])
|
|
|
|
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
|
|
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
|
|
assert.Equal(t, minmatch, res[0:len(minmatch)])
|
|
|
|
// Japanese (Shift-JIS)
|
|
// "日属秘ぞしちゅ。"
|
|
res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
|
|
assert.Equal(t, []byte{
|
|
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
|
|
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
|
|
}, res)
|
|
|
|
res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00})
|
|
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
|
|
}
|
|
|
|
func TestDetectEncoding(t *testing.T) {
|
|
testSuccess := func(b []byte, expected string) {
|
|
encoding, err := DetectEncoding(b)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, expected, encoding)
|
|
}
|
|
|
|
// invalid bytes
|
|
encoding, err := DetectEncoding([]byte{0xfa})
|
|
assert.Error(t, err)
|
|
assert.Equal(t, "UTF-8", encoding)
|
|
|
|
// utf-8
|
|
b := []byte("just some ascii")
|
|
testSuccess(b, "UTF-8")
|
|
|
|
// utf-8-sig: "hey" (with BOM)
|
|
b = []byte{0xef, 0xbb, 0xbf, 0x68, 0x65, 0x79}
|
|
testSuccess(b, "UTF-8")
|
|
|
|
// utf-16: "hey<accented G>"
|
|
b = []byte{0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x79, 0x00, 0xf4, 0x01}
|
|
testSuccess(b, "UTF-16LE")
|
|
|
|
// iso-8859-1: d<accented e>cor<newline>
|
|
b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a}
|
|
encoding, err = DetectEncoding(b)
|
|
assert.NoError(t, err)
|
|
assert.Contains(t, encoding, "ISO-8859-1")
|
|
|
|
defer test.MockVariableValue(&setting.Repository.AnsiCharset, "MyEncoding")()
|
|
testSuccess(b, "MyEncoding")
|
|
}
|
|
|
|
func stringMustStartWith(t *testing.T, expected string, value []byte) {
|
|
assert.Equal(t, expected, string(value[:len(expected)]))
|
|
}
|
|
|
|
func stringMustEndWith(t *testing.T, expected string, value []byte) {
|
|
assert.Equal(t, expected, string(value[len(value)-len(expected):]))
|
|
}
|
|
|
|
func TestToUTF8WithFallbackReader(t *testing.T) {
|
|
test.MockVariableValue(&ToUTF8WithFallbackReaderPrefetchSize)
|
|
|
|
block := "aá啊🤔"
|
|
runes := []rune(block)
|
|
assert.Len(t, string(runes[0]), 1)
|
|
assert.Len(t, string(runes[1]), 2)
|
|
assert.Len(t, string(runes[2]), 3)
|
|
assert.Len(t, string(runes[3]), 4)
|
|
|
|
content := strings.Repeat(block, 2)
|
|
for i := 1; i < len(content); i++ {
|
|
encoding, err := DetectEncoding([]byte(content[:i]))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "UTF-8", encoding)
|
|
|
|
ToUTF8WithFallbackReaderPrefetchSize = i
|
|
rd := ToUTF8WithFallbackReader(strings.NewReader(content), ConvertOpts{})
|
|
r, _ := io.ReadAll(rd)
|
|
assert.Equal(t, content, string(r))
|
|
}
|
|
for _, r := range runes {
|
|
content = "abc abc " + string(r) + string(r) + string(r)
|
|
for i := 0; i < len(content); i++ {
|
|
encoding, err := DetectEncoding([]byte(content[:i]))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "UTF-8", encoding)
|
|
}
|
|
}
|
|
}
|