mirror of
https://github.com/hanzoai/metrics.git
synced 2026-10-30 20:03:39 +00:00
c6843a834d
Previously, influx parser allocated a new slice byte for
unescape of Row fields. It adds extra pressure at GC and increases CPU
usage.
This commit changes escape to in-place updates for provided []byte.
Since request for parsing is actually a []byte converted into the
string, it's safe to update it in-place. To be able to interact with
[]byte directly, this commit changes parser API and accepts []byte
instead of string.
Benchstat:
```
│ before │ after │
│ sec/op │ sec/op vs base │
RowsUnmarshalUnescape-10 74.68n ± 4% 54.23n ± 5% -27.38% (p=0.000 n=10)
RowsUnmarshalUnescapeNoEscape-10 40.41n ± 2% 42.59n ± 1% +5.39% (p=0.000 n=10)
geomean 54.93n 48.06n -12.51%
│ before │ after │
│ B/s │ B/s vs base │
RowsUnmarshalUnescape-10 1.035Gi ± 4% 1.425Gi ± 5% +37.72% (p=0.000 n=10)
RowsUnmarshalUnescapeNoEscape-10 1.613Gi ± 2% 1.531Gi ± 1% -5.11% (p=0.000 n=10)
geomean 1.292Gi 1.477Gi +14.32%
│ before │ after │
│ B/op │ B/op vs base │
RowsUnmarshalUnescape-10 149.00 ± 0% 96.00 ± 0% -35.57% (p=0.000 n=10)
RowsUnmarshalUnescapeNoEscape-10 80.00 ± 0% 80.00 ± 0% ~ (p=1.000 n=10) ¹
geomean 109.2 87.64 -19.73%
¹ all samples are equal
│ before │ after │
│ allocs/op │ allocs/op vs base │
RowsUnmarshalUnescape-10 5.000 ± 0% 1.000 ± 0% -80.00% (p=0.000 n=10)
RowsUnmarshalUnescapeNoEscape-10 1.000 ± 0% 1.000 ± 0% ~ (p=1.000 n=10) ¹
geomean 2.236 1.000 -55.28%
```
Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10053
### Describe Your Changes
Please provide a brief description of the changes you made. Be as
specific as possible to help others understand the purpose and impact of
your modifications.
### Checklist
The following checks are **mandatory**:
- [ ] My change adheres to [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
- [ ] My change adheres to [VictoriaMetrics development
goals](https://docs.victoriametrics.com/victoriametrics/goals/).
---------
Co-authored-by: Hanzo Dev <[email protected]>
Co-authored-by: Aliaksandr Valialkin <[email protected]>
554 lines
10 KiB
Go
554 lines
10 KiB
Go
package influx
|
|
|
|
import (
|
|
"reflect"
|
|
"testing"
|
|
)
|
|
|
|
func TestNextUnquotedChar(t *testing.T) {
|
|
f := func(s string, ch byte, hasEscapeChars bool, nExpected int) {
|
|
t.Helper()
|
|
|
|
uc := &unmarshalContext{
|
|
hasEscapeChars: hasEscapeChars,
|
|
hasQuotedFields: true,
|
|
}
|
|
n := nextUnquotedChar(s, ch, uc)
|
|
if n != nExpected {
|
|
t.Fatalf("unexpected n for nextUnquotedChar(%q, '%c', %v); got %d; want %d", s, ch, hasEscapeChars, n, nExpected)
|
|
}
|
|
}
|
|
|
|
f(``, ' ', true, -1)
|
|
f(``, ' ', false, -1)
|
|
f(`""`, ' ', true, -1)
|
|
f(`""`, ' ', false, -1)
|
|
f(`"foo bar\" " baz`, ' ', true, 12)
|
|
f(`"foo bar\" " baz`, ' ', false, 10)
|
|
}
|
|
|
|
func TestNextUnescapedChar(t *testing.T) {
|
|
f := func(s string, ch byte, hasEscapeChars bool, nExpected int) {
|
|
t.Helper()
|
|
|
|
uc := &unmarshalContext{
|
|
hasEscapeChars: hasEscapeChars,
|
|
}
|
|
n := nextUnescapedChar(s, ch, uc)
|
|
if n != nExpected {
|
|
t.Fatalf("unexpected n for nextUnescapedChar(%q, '%c', %v); got %d; want %d", s, ch, hasEscapeChars, n, nExpected)
|
|
}
|
|
}
|
|
|
|
f("", ' ', false, -1)
|
|
f("", ' ', true, -1)
|
|
f(" ", ' ', false, 0)
|
|
f(" ", ' ', true, 0)
|
|
f("x y", ' ', false, 1)
|
|
f("x y", ' ', true, 1)
|
|
f(`x\ y`, ' ', false, 2)
|
|
f(`x\ y`, ' ', true, 3)
|
|
f(`\\,`, ',', false, 2)
|
|
f(`\\,`, ',', true, 2)
|
|
f(`\\\=`, '=', false, 3)
|
|
f(`\\\=`, '=', true, -1)
|
|
f(`\\\=aa`, '=', false, 3)
|
|
f(`\\\=aa`, '=', true, -1)
|
|
f(`\\\=a=a`, '=', false, 3)
|
|
f(`\\\=a=a`, '=', true, 5)
|
|
f(`a\`, ' ', false, -1)
|
|
f(`a\`, ' ', true, -1)
|
|
}
|
|
|
|
func TestUnescapeTagValue(t *testing.T) {
|
|
f := func(s, sExpected string) {
|
|
t.Helper()
|
|
|
|
uc := &unmarshalContext{
|
|
hasEscapeChars: true,
|
|
}
|
|
ss := unescapeTagValue(s, uc)
|
|
if ss != sExpected {
|
|
t.Fatalf("unexpected value for %q; got %q; want %q", s, ss, sExpected)
|
|
}
|
|
}
|
|
|
|
f("", "")
|
|
f("x", "x")
|
|
f("foobar", "foobar")
|
|
f("привет", "привет")
|
|
f(`\a\b\cd`, `\a\b\cd`)
|
|
f(`\`, `\`)
|
|
f(`foo\`, `foo\`)
|
|
f(`\,foo\\\=\ bar`, `,foo\= bar`)
|
|
}
|
|
|
|
func TestRowsUnmarshalFailure(t *testing.T) {
|
|
f := func(s string) {
|
|
t.Helper()
|
|
|
|
var rows Rows
|
|
if err := rows.Unmarshal(s, false); err == nil {
|
|
t.Fatal("expecting non-nil error")
|
|
}
|
|
if len(rows.Rows) != 0 {
|
|
t.Fatalf("expecting zero rows; got %d rows", len(rows.Rows))
|
|
}
|
|
|
|
// Try again
|
|
if err := rows.Unmarshal(s, false); err == nil {
|
|
t.Fatalf("expecting non-nil error on the second attempt")
|
|
}
|
|
if len(rows.Rows) != 0 {
|
|
t.Fatalf("expecting zero rows; got %d rows", len(rows.Rows))
|
|
}
|
|
}
|
|
|
|
// No fields
|
|
f("foo")
|
|
f("foo,bar=baz 1234")
|
|
|
|
// Missing tag value
|
|
f("foo,bar")
|
|
f("foo,bar baz")
|
|
f("foo,bar=123, 123")
|
|
|
|
// Missing field value
|
|
f("foo bar")
|
|
f("foo bar=")
|
|
f("foo bar=,baz=23 123")
|
|
f("foo bar=1, 123")
|
|
f(`foo bar=" 123`)
|
|
f(`foo bar="123`)
|
|
f(`foo bar=",123`)
|
|
f(`foo bar=a"", 123`)
|
|
|
|
// Missing field name
|
|
f("foo =123")
|
|
f("foo =123\nbar")
|
|
|
|
// Invalid timestamp
|
|
f("foo bar=123 baz")
|
|
|
|
// Invalid field value
|
|
f("foo bar=1abci")
|
|
f("foo bar=-2abci")
|
|
f("foo bar=3abcu")
|
|
|
|
// HTTP request line
|
|
f("GET /foo HTTP/1.1")
|
|
f("GET /foo?bar=baz HTTP/1.0")
|
|
}
|
|
|
|
func TestRowsUnmarshalSuccess(t *testing.T) {
|
|
f := func(s string, rowsExpected *Rows) {
|
|
t.Helper()
|
|
|
|
var rows Rows
|
|
if err := rows.Unmarshal(s, true); err != nil {
|
|
t.Fatalf("unexpected error: %s", err)
|
|
}
|
|
if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
|
|
t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
|
|
}
|
|
|
|
// Try unmarshaling again
|
|
if err := rows.Unmarshal(s, true); err != nil {
|
|
t.Fatalf("unexpected error on the second unmarshal attempt: %s", err)
|
|
}
|
|
if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
|
|
t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
|
|
}
|
|
|
|
rows.Reset()
|
|
if len(rows.Rows) != 0 {
|
|
t.Fatalf("non-empty rows after reset: %+v", rows.Rows)
|
|
}
|
|
}
|
|
|
|
// Empty line
|
|
f("", &Rows{})
|
|
f("\n\n", &Rows{})
|
|
f("\n\r\n", &Rows{})
|
|
|
|
// Comment
|
|
f("\n# foobar\n", &Rows{})
|
|
f("#foobar baz", &Rows{})
|
|
f("#foobar baz\n#sss", &Rows{})
|
|
|
|
// Missing measurement
|
|
f(" baz=123", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "",
|
|
Fields: []Field{{
|
|
Key: "baz",
|
|
Value: 123,
|
|
}},
|
|
}},
|
|
})
|
|
f(",foo=bar baz=123", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "",
|
|
Tags: []Tag{{
|
|
Key: "foo",
|
|
Value: "bar",
|
|
}},
|
|
Fields: []Field{{
|
|
Key: "baz",
|
|
Value: 123,
|
|
}},
|
|
}},
|
|
})
|
|
|
|
// Minimal line without tags and timestamp
|
|
f("foo bar=123", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "foo",
|
|
Fields: []Field{{
|
|
Key: "bar",
|
|
Value: 123,
|
|
}},
|
|
}},
|
|
})
|
|
// Excess whitespace after final field. Issue #10049
|
|
f("foo bar=123 ", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "foo",
|
|
Fields: []Field{{
|
|
Key: "bar",
|
|
Value: 123,
|
|
}},
|
|
}},
|
|
})
|
|
f("# comment\nfoo bar=123\r\n#comment2 sdsf dsf", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "foo",
|
|
Fields: []Field{{
|
|
Key: "bar",
|
|
Value: 123,
|
|
}},
|
|
}},
|
|
})
|
|
f("foo bar=123\n", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "foo",
|
|
Fields: []Field{{
|
|
Key: "bar",
|
|
Value: 123,
|
|
}},
|
|
}},
|
|
})
|
|
|
|
// Line without tags and with a timestamp.
|
|
f("foo bar=123.45 -345", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "foo",
|
|
Fields: []Field{{
|
|
Key: "bar",
|
|
Value: 123.45,
|
|
}},
|
|
Timestamp: -345,
|
|
}},
|
|
})
|
|
|
|
// Line with a single tag
|
|
f("foo,tag1=xyz bar=123", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "foo",
|
|
Tags: []Tag{{
|
|
Key: "tag1",
|
|
Value: "xyz",
|
|
}},
|
|
Fields: []Field{{
|
|
Key: "bar",
|
|
Value: 123,
|
|
}},
|
|
}},
|
|
})
|
|
|
|
// Line with multiple tags
|
|
f("foo,tag1=xyz,tag2=43as bar=123", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "foo",
|
|
Tags: []Tag{
|
|
{
|
|
Key: "tag1",
|
|
Value: "xyz",
|
|
},
|
|
{
|
|
Key: "tag2",
|
|
Value: "43as",
|
|
},
|
|
},
|
|
Fields: []Field{{
|
|
Key: "bar",
|
|
Value: 123,
|
|
}},
|
|
}},
|
|
})
|
|
|
|
// Line with empty tag values
|
|
f("foo,tag1=xyz,tagN=,tag2=43as,=xxx bar=123", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "foo",
|
|
Tags: []Tag{
|
|
{
|
|
Key: "tag1",
|
|
Value: "xyz",
|
|
},
|
|
{
|
|
Key: "tag2",
|
|
Value: "43as",
|
|
},
|
|
},
|
|
Fields: []Field{{
|
|
Key: "bar",
|
|
Value: 123,
|
|
}},
|
|
}},
|
|
})
|
|
|
|
// Line with multiple tags, multiple fields and timestamp
|
|
f(`system,host=ip-172-16-10-144 uptime_format="3 days, 21:01",quoted_float="-1.23",quoted_int="123" 1557761040000000000`, &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "system",
|
|
Tags: []Tag{{
|
|
Key: "host",
|
|
Value: "ip-172-16-10-144",
|
|
}},
|
|
Fields: []Field{
|
|
{
|
|
Key: "uptime_format",
|
|
Value: 0,
|
|
},
|
|
{
|
|
Key: "quoted_float",
|
|
Value: -1.23,
|
|
},
|
|
{
|
|
Key: "quoted_int",
|
|
Value: 123,
|
|
},
|
|
},
|
|
Timestamp: 1557761040000000000,
|
|
}},
|
|
})
|
|
f(`foo,tag1=xyz,tag2=43as bar=-123e4,x=True,y=-45i,z=f,aa="f,= \"a",bb=23u 48934`, &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "foo",
|
|
Tags: []Tag{
|
|
{
|
|
Key: "tag1",
|
|
Value: "xyz",
|
|
},
|
|
{
|
|
Key: "tag2",
|
|
Value: "43as",
|
|
},
|
|
},
|
|
Fields: []Field{
|
|
{
|
|
Key: "bar",
|
|
Value: -123e4,
|
|
},
|
|
{
|
|
Key: "x",
|
|
Value: 1,
|
|
},
|
|
{
|
|
Key: "y",
|
|
Value: -45,
|
|
},
|
|
{
|
|
Key: "z",
|
|
Value: 0,
|
|
},
|
|
{
|
|
Key: "aa",
|
|
Value: 0,
|
|
},
|
|
{
|
|
Key: "bb",
|
|
Value: 23,
|
|
},
|
|
},
|
|
Timestamp: 48934,
|
|
}},
|
|
})
|
|
|
|
// Escape chars
|
|
f(`fo\,bar\=b\ az,x\=\ b=\\a\,\=\q\ \\\a\ b\=\,=4.34`, &Rows{
|
|
Rows: []Row{{
|
|
Measurement: `fo,bar=b az`,
|
|
Tags: []Tag{{
|
|
Key: `x= b`,
|
|
Value: `\a,=\q `,
|
|
}},
|
|
Fields: []Field{{
|
|
Key: `\\a b=,`,
|
|
Value: 4.34,
|
|
}},
|
|
}},
|
|
})
|
|
// Test case from https://community.librenms.org/t/integration-with-victoriametrics/9689
|
|
f("ports,foo=a,bar=et\\ +\\ V,baz=ype INDISCARDS=245333676,OUTDISCARDS=1798680", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "ports",
|
|
Tags: []Tag{
|
|
{
|
|
Key: "foo",
|
|
Value: "a",
|
|
},
|
|
{
|
|
Key: "bar",
|
|
Value: "et + V",
|
|
},
|
|
{
|
|
Key: "baz",
|
|
Value: "ype",
|
|
},
|
|
},
|
|
Fields: []Field{
|
|
{
|
|
Key: "INDISCARDS",
|
|
Value: 245333676,
|
|
},
|
|
{
|
|
Key: "OUTDISCARDS",
|
|
Value: 1798680,
|
|
},
|
|
},
|
|
}},
|
|
})
|
|
|
|
// Multiple lines
|
|
f("foo,tag=xyz field=1.23 48934\n"+
|
|
"bar x=-1i\n\n", &Rows{
|
|
Rows: []Row{
|
|
{
|
|
Measurement: "foo",
|
|
Tags: []Tag{{
|
|
Key: "tag",
|
|
Value: "xyz",
|
|
}},
|
|
Fields: []Field{{
|
|
Key: "field",
|
|
Value: 1.23,
|
|
}},
|
|
Timestamp: 48934,
|
|
},
|
|
{
|
|
Measurement: "bar",
|
|
Fields: []Field{{
|
|
Key: "x",
|
|
Value: -1,
|
|
}},
|
|
},
|
|
},
|
|
})
|
|
|
|
// Multiple lines with invalid line in the middle.
|
|
f("foo,tag=xyz field=1.23 48934\n"+
|
|
"invalid line\n"+
|
|
"bar x=-1i\n\n", &Rows{
|
|
Rows: []Row{
|
|
{
|
|
Measurement: "foo",
|
|
Tags: []Tag{{
|
|
Key: "tag",
|
|
Value: "xyz",
|
|
}},
|
|
Fields: []Field{{
|
|
Key: "field",
|
|
Value: 1.23,
|
|
}},
|
|
Timestamp: 48934,
|
|
},
|
|
{
|
|
Measurement: "bar",
|
|
Fields: []Field{{
|
|
Key: "x",
|
|
Value: -1,
|
|
}},
|
|
},
|
|
},
|
|
})
|
|
|
|
// No newline after the second line.
|
|
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/82
|
|
f("foo,tag=xyz field=1.23 48934\n"+
|
|
"bar x=-1i", &Rows{
|
|
Rows: []Row{
|
|
{
|
|
Measurement: "foo",
|
|
Tags: []Tag{{
|
|
Key: "tag",
|
|
Value: "xyz",
|
|
}},
|
|
Fields: []Field{{
|
|
Key: "field",
|
|
Value: 1.23,
|
|
}},
|
|
Timestamp: 48934,
|
|
},
|
|
{
|
|
Measurement: "bar",
|
|
Fields: []Field{{
|
|
Key: "x",
|
|
Value: -1,
|
|
}},
|
|
},
|
|
},
|
|
})
|
|
|
|
// Superfluous whitespace between tags, fields and timestamps.
|
|
f(`cpu_utilization,host=mnsbook-pro.local value=119.8 1607222595591`, &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "cpu_utilization",
|
|
Tags: []Tag{{
|
|
Key: "host",
|
|
Value: "mnsbook-pro.local",
|
|
}},
|
|
Fields: []Field{{
|
|
Key: "value",
|
|
Value: 119.8,
|
|
}},
|
|
Timestamp: 1607222595591,
|
|
}},
|
|
})
|
|
f(`cpu_utilization,host=mnsbook-pro.local value=119.8 1607222595591`, &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "cpu_utilization",
|
|
Tags: []Tag{{
|
|
Key: "host",
|
|
Value: "mnsbook-pro.local",
|
|
}},
|
|
Fields: []Field{{
|
|
Key: "value",
|
|
Value: 119.8,
|
|
}},
|
|
Timestamp: 1607222595591,
|
|
}},
|
|
})
|
|
|
|
f("x,y=z,g=p:\\ \\ 5432\\,\\ gp\\ mon\\ [lol]\\ con10\\ cmd5\\ SELECT f=1", &Rows{
|
|
Rows: []Row{{
|
|
Measurement: "x",
|
|
Tags: []Tag{
|
|
{
|
|
Key: "y",
|
|
Value: "z",
|
|
},
|
|
{
|
|
Key: "g",
|
|
Value: "p: 5432, gp mon [lol] con10 cmd5 SELECT",
|
|
},
|
|
},
|
|
Fields: []Field{{
|
|
Key: "f",
|
|
Value: 1,
|
|
}},
|
|
}},
|
|
})
|
|
}
|