mirror of
https://github.com/hanzoai/traces.git
synced 2026-10-30 18:20:41 +00:00
cc86e98b04
* Add operators IN, NOT IN, MATCH ANY, and MATCH NONE
* Implement Static.append()
* Print string arrays with '`' instead of '"'
Example ["aa", "bb"] -> [`aa`, `bb`]
* Rewrite binary operations to array equivalent
The rewriter handles the following cases:
- { .a = "a" || .a = "b" } => { .a IN ["a", "b"] }
- { .a != "a" && .a != "b" } => { .a NOT IN ["a", "b"] }
- { .a =~ "a" || .a =~ "b" } => { .a MATCH ANY ["a", "b"] }
- { .a !~ "a" && .a !~ "b" } => { .a MATCH NONE ["a", "b"] }
* Add test cases with arrays to TestBinOp
* traceql.Parse() applies AST optimizations by default
* Rename predicate_test.go to predicates_test.go
* Add array predicates to parquetquery
* BREAKING CHANGE: more precise regex validation
BinaryOperation.validate() considered everythin a valid regex where
FieldExpression.EncodeToString(false) happened to return a valid regex.
With this change the RHS of a regex operation is required to be a string
or string array
* vp4: handle predicates with arrays correctly
* vp5: handle predicates with arrays correctly
* vp3: handle predicates with arrays correctly
* Add test cases for optimized queries
* Update query test examples with optimizable queries
* vp3: fix handling of array types
* CHANGELOG.md
* Pass array operators down to fetch layer
* Read to first virtual row after seeking on a low definition level
This helps avoid additional spans/op in queries with only
resource attributes
---------
Co-authored-by: Martin Disibio <[email protected]>
373 lines
8.2 KiB
Go
373 lines
8.2 KiB
Go
package main
|
|
|
|
import (
|
|
"os"
|
|
"strings"
|
|
"text/template"
|
|
)
|
|
|
|
func StringContains(str, substr string) bool {
|
|
return strings.Contains(str, substr)
|
|
}
|
|
|
|
func StringLength(str string) int {
|
|
return len(str)
|
|
}
|
|
|
|
func main() {
|
|
tmplString := `// Code generated by make gen-parquet-query. DO NOT EDIT.
|
|
// source: pkg/parquetquerygen/predicates.go
|
|
|
|
package parquetquery
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
|
|
pq "github.com/parquet-go/parquet-go"
|
|
)
|
|
{{- range .}}
|
|
{{- $pred := . }}
|
|
{{- range .Ops}}
|
|
{{- $structName := printf "%s%sPredicate" $pred.Name .Op }}
|
|
{{- $minInRange := (contains .RangeCond "min") }}
|
|
{{- $maxInRange := (contains .RangeCond "max") }}
|
|
|
|
var _ Predicate = (*{{ $structName }})(nil)
|
|
|
|
type {{ $structName }} struct {
|
|
value {{$pred.Type}}
|
|
}
|
|
|
|
func New{{ $structName }}(val {{$pred.Type}}) {{ $structName }} {
|
|
return {{ $structName }}{value: val}
|
|
}
|
|
|
|
func (p {{ $structName }}) String() string {
|
|
return fmt.Sprintf("{{ $structName }}{{"{"}}{{ $pred.FormatModifier }}{{"}"}}", p.value)
|
|
}
|
|
|
|
func (p {{ $structName }}) KeepColumnChunk(c *ColumnChunkHelper) bool {
|
|
if d := c.Dictionary(); d != nil {
|
|
return keepDictionary(d, p.KeepValue)
|
|
}
|
|
|
|
{{- if gt (.RangeCond | strlen) 0 }}
|
|
ci, err := c.ColumnIndex()
|
|
if err == nil && ci != nil {
|
|
for i := 0; i < ci.NumPages(); i++ {
|
|
{{ if $minInRange }}min := ci.MinValue(i).{{ $pred.ParquetFunc }}{{end}}
|
|
{{ if $maxInRange }}max := ci.MaxValue(i).{{ $pred.ParquetFunc }}{{end}}
|
|
|
|
if {{ .RangeCond }} {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
{{- end }}
|
|
|
|
return true
|
|
}
|
|
|
|
func (p {{ $structName }}) KeepPage(page pq.Page) bool {
|
|
{{- if gt (.RangeCond | strlen) 0 }}
|
|
{{ if $minInRange }}minV{{else}}_{{end}}, {{ if $maxInRange }}maxV{{else}}_{{end}}, ok := page.Bounds()
|
|
if ok {
|
|
{{ if $minInRange }}min := minV.{{ $pred.ParquetFunc }}{{end}}
|
|
{{ if $maxInRange }}max := maxV.{{ $pred.ParquetFunc }}{{end}}
|
|
|
|
return {{ .RangeCond }}
|
|
}
|
|
{{- end }}
|
|
|
|
return true
|
|
}
|
|
|
|
func (p {{ $structName }}) KeepValue(v pq.Value) bool {
|
|
vv := v.{{ $pred.ParquetFunc }}
|
|
return {{ .CompareCond }}
|
|
}
|
|
{{- end }}
|
|
{{- end }}
|
|
|
|
{{- /* Generate In/NotIn for Int, Float, Bool */}}
|
|
{{- range .}}
|
|
{{- $pred := . }}
|
|
{{- if or (eq $pred.Name "Int") (eq $pred.Name "Float") (eq $pred.Name "Bool") }}
|
|
{{- $structName := printf "%sInPredicate" $pred.Name }}
|
|
var _ Predicate = (*{{ $structName }})(nil)
|
|
|
|
type {{ $structName }} struct {
|
|
values []{{$pred.Type}}
|
|
}
|
|
|
|
func New{{ $structName }}(vals []{{$pred.Type}}) {{ $structName }} {
|
|
return {{ $structName }}{values: vals}
|
|
}
|
|
|
|
func (p {{ $structName }}) String() string { return fmt.Sprintf("{{ $structName }}{%v}", p.values) }
|
|
|
|
func (p {{ $structName }}) KeepColumnChunk(c *ColumnChunkHelper) bool {
|
|
if d := c.Dictionary(); d != nil {
|
|
return keepDictionary(d, p.KeepValue)
|
|
}
|
|
{{- if or (eq $pred.Name "Int") (eq $pred.Name "Float") }}
|
|
ci, err := c.ColumnIndex()
|
|
if err == nil && ci != nil {
|
|
for i := 0; i < ci.NumPages(); i++ {
|
|
min := ci.MinValue(i).{{ $pred.ParquetFunc }}
|
|
max := ci.MaxValue(i).{{ $pred.ParquetFunc }}
|
|
for _, v := range p.values {
|
|
if min <= v && v <= max {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
{{- end }}
|
|
return true
|
|
}
|
|
|
|
func (p {{ $structName }}) KeepPage(page pq.Page) bool {
|
|
{{- if or (eq $pred.Name "Int") (eq $pred.Name "Float") }}
|
|
minV, maxV, ok := page.Bounds()
|
|
if ok {
|
|
min := minV.{{ $pred.ParquetFunc }}
|
|
max := maxV.{{ $pred.ParquetFunc }}
|
|
for _, v := range p.values {
|
|
if min <= v && v <= max {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
{{- end }}
|
|
return true
|
|
}
|
|
|
|
func (p {{ $structName }}) KeepValue(v pq.Value) bool {
|
|
vv := v.{{ $pred.ParquetFunc }}
|
|
for _, x := range p.values {
|
|
if vv == x {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
{{- $structName := printf "%sNotInPredicate" $pred.Name }}
|
|
var _ Predicate = (*{{ $structName }})(nil)
|
|
|
|
type {{ $structName }} struct {
|
|
values []{{$pred.Type}}
|
|
}
|
|
|
|
func New{{ $structName }}(vals []{{$pred.Type}}) {{ $structName }} {
|
|
return {{ $structName }}{values: vals}
|
|
}
|
|
|
|
func (p {{ $structName }}) String() string {
|
|
return fmt.Sprintf("{{ $structName }}{%v}", p.values)
|
|
}
|
|
|
|
func (p {{ $structName }}) KeepColumnChunk(c *ColumnChunkHelper) bool {
|
|
if d := c.Dictionary(); d != nil {
|
|
return keepDictionary(d, p.KeepValue)
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (p {{ $structName }}) KeepPage(page pq.Page) bool {
|
|
return true
|
|
}
|
|
|
|
func (p {{ $structName }}) KeepValue(v pq.Value) bool {
|
|
vv := v.{{ $pred.ParquetFunc }}
|
|
for _, x := range p.values {
|
|
if vv == x {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
{{- end }}
|
|
{{- end }}
|
|
`
|
|
|
|
type op struct {
|
|
Op string
|
|
CompareCond string
|
|
RangeCond string
|
|
}
|
|
|
|
preds := []struct {
|
|
Name string
|
|
Type string
|
|
FormatModifier string
|
|
ParquetFunc string
|
|
Ops []op
|
|
}{
|
|
// int64!
|
|
{
|
|
Name: "Int",
|
|
Type: "int64",
|
|
ParquetFunc: "Int64()",
|
|
FormatModifier: "%d",
|
|
Ops: []op{
|
|
{
|
|
Op: "Equal",
|
|
CompareCond: "vv == p.value",
|
|
RangeCond: "min <= p.value && p.value <= max",
|
|
},
|
|
{
|
|
Op: "NotEqual",
|
|
CompareCond: "vv != p.value",
|
|
RangeCond: "min != p.value || p.value != max",
|
|
},
|
|
{
|
|
Op: "Greater",
|
|
CompareCond: "vv > p.value",
|
|
RangeCond: "max > p.value",
|
|
},
|
|
{
|
|
Op: "GreaterEqual",
|
|
CompareCond: "vv >= p.value",
|
|
RangeCond: "max >= p.value",
|
|
},
|
|
{
|
|
Op: "Less",
|
|
CompareCond: "vv < p.value",
|
|
RangeCond: "min < p.value",
|
|
},
|
|
{
|
|
Op: "LessEqual",
|
|
CompareCond: "vv <= p.value",
|
|
RangeCond: "min <= p.value",
|
|
},
|
|
},
|
|
},
|
|
// float64!
|
|
{
|
|
Name: "Float",
|
|
Type: "float64",
|
|
ParquetFunc: "Double()",
|
|
FormatModifier: "%f",
|
|
Ops: []op{
|
|
{
|
|
Op: "Equal",
|
|
CompareCond: "vv == p.value",
|
|
RangeCond: "min <= p.value && p.value <= max",
|
|
},
|
|
{
|
|
Op: "NotEqual",
|
|
CompareCond: "vv != p.value",
|
|
RangeCond: "min != p.value || p.value != max",
|
|
},
|
|
{
|
|
Op: "Greater",
|
|
CompareCond: "vv > p.value",
|
|
RangeCond: "max > p.value",
|
|
},
|
|
{
|
|
Op: "GreaterEqual",
|
|
CompareCond: "vv >= p.value",
|
|
RangeCond: "max >= p.value",
|
|
},
|
|
{
|
|
Op: "Less",
|
|
CompareCond: "vv < p.value",
|
|
RangeCond: "min < p.value",
|
|
},
|
|
{
|
|
Op: "LessEqual",
|
|
CompareCond: "vv <= p.value",
|
|
RangeCond: "min <= p.value",
|
|
},
|
|
},
|
|
},
|
|
// bool!
|
|
{
|
|
Name: "Bool",
|
|
Type: "bool",
|
|
ParquetFunc: "Boolean()",
|
|
FormatModifier: "%t",
|
|
Ops: []op{
|
|
{
|
|
Op: "Equal",
|
|
CompareCond: "vv == p.value",
|
|
RangeCond: "",
|
|
},
|
|
{
|
|
Op: "NotEqual",
|
|
CompareCond: "vv != p.value",
|
|
RangeCond: "",
|
|
},
|
|
},
|
|
},
|
|
// string!
|
|
// =~ covered by RegexInPredicate
|
|
// !~ covered by RegexNotInPredicate
|
|
{
|
|
Name: "String",
|
|
Type: "[]byte",
|
|
ParquetFunc: "ByteArray()",
|
|
FormatModifier: "%s",
|
|
Ops: []op{
|
|
{
|
|
Op: "Greater",
|
|
CompareCond: "bytes.Compare(vv, p.value) > 0",
|
|
RangeCond: "bytes.Compare(max, p.value) > 0",
|
|
},
|
|
{
|
|
Op: "GreaterEqual",
|
|
CompareCond: "bytes.Compare(vv, p.value) >= 0",
|
|
RangeCond: "bytes.Compare(max, p.value) >= 0",
|
|
},
|
|
{
|
|
Op: "Less",
|
|
CompareCond: "bytes.Compare(vv, p.value) < 0",
|
|
RangeCond: "bytes.Compare(min, p.value) < 0",
|
|
},
|
|
{
|
|
Op: "LessEqual",
|
|
CompareCond: "bytes.Compare(vv, p.value) <= 0",
|
|
RangeCond: "bytes.Compare(min, p.value) <= 0",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "Byte",
|
|
Type: "[]byte",
|
|
ParquetFunc: "ByteArray()",
|
|
FormatModifier: "%s",
|
|
Ops: []op{
|
|
{
|
|
Op: "Equal",
|
|
CompareCond: `bytes.Equal(vv, p.value)`,
|
|
RangeCond: "bytes.Compare(p.value, min) >= 0 && bytes.Compare(p.value, max) <= 0",
|
|
},
|
|
{
|
|
Op: "NotEqual",
|
|
CompareCond: `!bytes.Equal(vv, p.value)`,
|
|
RangeCond: "!bytes.Equal(min, p.value) || !bytes.Equal(p.value, max)",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
funcMap := template.FuncMap{
|
|
"contains": StringContains,
|
|
"strlen": StringLength,
|
|
}
|
|
tmpl, err := template.New("predicates").Funcs(funcMap).Parse(tmplString)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
err = tmpl.Execute(os.Stdout, preds)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|