From 2ea5c6629991bd9764368a83804f0fc463de7b96 Mon Sep 17 00:00:00 2001 From: Noah Treuhaft Date: Fri, 4 Sep 2026 14:35:20 -0400 Subject: [PATCH] runtime/sam: remove dead code --- runtime/sam/expr/agg.go | 43 -- runtime/sam/expr/agg/any.go | 5 - runtime/sam/expr/boolean.go | 284 ---------- runtime/sam/expr/coerce/coerce.go | 39 -- runtime/sam/expr/dot.go | 36 -- runtime/sam/expr/dropper.go | 113 ---- runtime/sam/expr/eval.go | 724 ------------------------ runtime/sam/expr/filter.go | 175 ------ runtime/sam/expr/function/bytes.go | 54 -- runtime/sam/expr/function/coalesce.go | 15 - runtime/sam/expr/function/fields.go | 63 --- runtime/sam/expr/function/function.go | 1 - runtime/sam/expr/function/fusion.go | 37 -- runtime/sam/expr/function/grep.go | 29 - runtime/sam/expr/function/grok.go | 103 ---- runtime/sam/expr/function/has.go | 40 -- runtime/sam/expr/function/ip.go | 101 ---- runtime/sam/expr/function/ksuid.go | 35 -- runtime/sam/expr/function/len.go | 37 -- runtime/sam/expr/function/math.go | 134 ----- runtime/sam/expr/function/nestdotted.go | 82 --- runtime/sam/expr/function/nullif.go | 29 - runtime/sam/expr/function/parse.go | 32 -- runtime/sam/expr/function/regexp.go | 84 --- runtime/sam/expr/function/string.go | 215 ------- runtime/sam/expr/function/time.go | 74 --- runtime/sam/expr/function/types.go | 98 ---- runtime/sam/expr/function/unblend.go | 138 ----- runtime/sam/expr/literal.go | 17 - runtime/sam/expr/map.go | 72 --- runtime/sam/expr/putter.go | 26 - runtime/sam/expr/slice.go | 98 ---- runtime/sam/expr/udf.go | 40 -- runtime/sam/expr/values.go | 423 -------------- runtime/sam/op/meta/lister.go | 8 - runtime/sam/op/spill/file.go | 8 - runtime/sam/op/spill/merge.go | 4 - 37 files changed, 3516 deletions(-) delete mode 100644 runtime/sam/expr/agg.go delete mode 100644 runtime/sam/expr/dropper.go delete mode 100644 runtime/sam/expr/function/bytes.go delete mode 100644 runtime/sam/expr/function/coalesce.go delete mode 100644 runtime/sam/expr/function/fields.go delete mode 100644 runtime/sam/expr/function/fusion.go delete mode 100644 runtime/sam/expr/function/grep.go delete mode 100644 runtime/sam/expr/function/grok.go delete mode 100644 runtime/sam/expr/function/has.go delete mode 100644 runtime/sam/expr/function/ip.go delete mode 100644 runtime/sam/expr/function/ksuid.go delete mode 100644 runtime/sam/expr/function/nestdotted.go delete mode 100644 runtime/sam/expr/function/nullif.go delete mode 100644 runtime/sam/expr/function/regexp.go delete mode 100644 runtime/sam/expr/function/string.go delete mode 100644 runtime/sam/expr/function/time.go delete mode 100644 runtime/sam/expr/function/unblend.go delete mode 100644 runtime/sam/expr/literal.go delete mode 100644 runtime/sam/expr/map.go delete mode 100644 runtime/sam/expr/putter.go delete mode 100644 runtime/sam/expr/values.go diff --git a/runtime/sam/expr/agg.go b/runtime/sam/expr/agg.go deleted file mode 100644 index b53e8a30d5..0000000000 --- a/runtime/sam/expr/agg.go +++ /dev/null @@ -1,43 +0,0 @@ -package expr - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/runtime/sam/expr/agg" -) - -type Aggregator struct { - pattern agg.Pattern - expr Evaluator - where Evaluator -} - -func NewAggregator(op string, distinct bool, expr Evaluator, where Evaluator) (*Aggregator, error) { - pattern, err := agg.NewPattern(op, distinct, expr != nil) - if err != nil { - return nil, err - } - if expr == nil { - // Count is the only that has no argument so we just return - // true so it counts each value encountered. - expr = &Literal{super.True} - } - return &Aggregator{ - pattern: pattern, - expr: expr, - where: where, - }, nil -} - -func (a *Aggregator) NewFunction() agg.Function { - return a.pattern() -} - -func (a *Aggregator) Apply(sctx *super.Context, f agg.Function, this super.Value) { - if a.where != nil { - if val := EvalBool(sctx, this, a.where); !val.AsBool() { - // XXX Issue #3401: do something with "where" errors. - return - } - } - f.Consume(a.expr.Eval(this)) -} diff --git a/runtime/sam/expr/agg/any.go b/runtime/sam/expr/agg/any.go index 4940fb81cc..9e8e62bb2b 100644 --- a/runtime/sam/expr/agg/any.go +++ b/runtime/sam/expr/agg/any.go @@ -8,11 +8,6 @@ type Any super.Value var _ Function = (*Any)(nil) -func NewAny() *Any { - a := (Any)(super.Null) - return &a -} - func (a *Any) Consume(val super.Value) { if val.DeunionIntoNameds().IsNull() { return diff --git a/runtime/sam/expr/boolean.go b/runtime/sam/expr/boolean.go index 5c02fa6957..ffd73c3d40 100644 --- a/runtime/sam/expr/boolean.go +++ b/runtime/sam/expr/boolean.go @@ -1,233 +1,16 @@ package expr import ( - "bytes" - "fmt" - "math" - "net/netip" "regexp" "regexp/syntax" "github.com/brimdata/super" - "github.com/brimdata/super/pkg/byteconv" - "github.com/brimdata/super/scode" ) // Boolean is a function that takes a Value and returns a boolean result // based on the typed value. type Boolean func(super.Value) super.Value -var compareBool = map[string]func(bool, bool) bool{ - "==": func(a, b bool) bool { return a == b }, - "!=": func(a, b bool) bool { return a != b }, - ">": func(a, b bool) bool { return a && !b }, - ">=": func(a, b bool) bool { return a || !b }, - "<": func(a, b bool) bool { return !a && b }, - "<=": func(a, b bool) bool { return !a || b }, -} - -// CompareBool returns a Predicate that compares super.Values to a boolean literal -// that must be a boolean or coercible to an integer. In the later case, the integer -// is converted to a boolean. -func CompareBool(op string, pattern bool) (Boolean, error) { - compare, ok := compareBool[op] - if !ok { - return nil, fmt.Errorf("unknown bool comparator: %s", op) - } - return func(val super.Value) super.Value { - val = val.Under() - if val.IsNull() { - return super.Null - } - if val.IsError() { - return val - } - if val.Type().ID() != super.IDBool { - return super.False - } - b := val.Bool() - return super.NewBool(compare(b, pattern)) - }, nil -} - -var compareInt = map[string]func(int64, int64) bool{ - "==": func(a, b int64) bool { return a == b }, - "!=": func(a, b int64) bool { return a != b }, - ">": func(a, b int64) bool { return a > b }, - ">=": func(a, b int64) bool { return a >= b }, - "<": func(a, b int64) bool { return a < b }, - "<=": func(a, b int64) bool { return a <= b }} - -var compareFloat = map[string]func(float64, float64) bool{ - "==": func(a, b float64) bool { return a == b }, - "!=": func(a, b float64) bool { return a != b }, - ">": func(a, b float64) bool { return a > b }, - ">=": func(a, b float64) bool { return a >= b }, - "<": func(a, b float64) bool { return a < b }, - "<=": func(a, b float64) bool { return a <= b }} - -// Return a predicate for comparing this value to one more typed -// byte slices by calling the predicate function with a Value. -// Operand is one of "==", "!=", "<", "<=", ">", ">=". -func CompareInt64(op string, pattern int64) (Boolean, error) { - CompareInt, ok1 := compareInt[op] - CompareFloat, ok2 := compareFloat[op] - if !ok1 || !ok2 { - return nil, fmt.Errorf("unknown int comparator: %s", op) - } - // many different data types can be compared with integers - return func(val super.Value) super.Value { - val = val.Under() - if val.IsNull() { - return super.Null - } - if val.IsError() { - return val - } - switch val.Type().ID() { - case super.IDUint8, super.IDUint16, super.IDUint32, super.IDUint64: - if v := val.Uint(); v <= math.MaxInt64 { - return super.NewBool(CompareInt(int64(v), pattern)) - } - case super.IDInt8, super.IDInt16, super.IDInt32, super.IDInt64, super.IDTime, super.IDDuration: - return super.NewBool(CompareInt(val.Int(), pattern)) - case super.IDFloat16, super.IDFloat32, super.IDFloat64: - return super.NewBool(CompareFloat(val.Float(), float64(pattern))) - } - return super.False - }, nil -} - -// XXX should just do equality and we should compare in the encoded domain -// and not make copies and have separate cases for len 4 and len 16 -var compareAddr = map[string]func(netip.Addr, netip.Addr) bool{ - "==": func(a, b netip.Addr) bool { return a.Compare(b) == 0 }, - "!=": func(a, b netip.Addr) bool { return a.Compare(b) != 0 }, - ">": func(a, b netip.Addr) bool { return a.Compare(b) > 0 }, - ">=": func(a, b netip.Addr) bool { return a.Compare(b) >= 0 }, - "<": func(a, b netip.Addr) bool { return a.Compare(b) < 0 }, - "<=": func(a, b netip.Addr) bool { return a.Compare(b) <= 0 }, -} - -// Comparison returns a Predicate that compares typed byte slices that must -// be TypeAddr with the value's address using a comparison based on op. -// Only equality operands are allowed. -func CompareIP(op string, pattern netip.Addr) (Boolean, error) { - compare, ok := compareAddr[op] - if !ok { - return nil, fmt.Errorf("unknown addr comparator: %s", op) - } - return func(val super.Value) super.Value { - val = val.Under() - if val.IsNull() { - return super.Null - } - if val.IsError() { - return val - } - if val.Type().ID() != super.IDIP { - return super.False - } - return super.NewBool(compare(super.DecodeIP(val.Bytes()), pattern)) - }, nil -} - -// CompareFloat64 returns a Predicate that compares typed byte slices that must -// be coercible to an double with the value's double value using a comparison -// based on op. Int, count, port, and double types can -// all be converted to the integer value. XXX there are some overflow issues here. -func CompareFloat64(op string, pattern float64) (Boolean, error) { - compare, ok := compareFloat[op] - if !ok { - return nil, fmt.Errorf("unknown double comparator: %s", op) - } - return func(val super.Value) super.Value { - val = val.Under() - if val.IsNull() { - return super.Null - } - if val.IsError() { - return val - } - switch val.Type().ID() { - // We allow comparison of float constant with integer-y - // fields and just use typeDouble to parse since it will do - // the right thing for integers. XXX do we want to allow - // integers that cause float64 overflow? user can always - // use an integer constant instead of a float constant to - // compare with the integer-y field. - case super.IDUint8, super.IDUint16, super.IDUint32, super.IDUint64: - return super.NewBool(compare(float64(val.Uint()), pattern)) - case super.IDInt8, super.IDInt16, super.IDInt32, super.IDInt64, super.IDTime, super.IDDuration: - return super.NewBool(compare(float64(val.Int()), pattern)) - case super.IDFloat16, super.IDFloat32, super.IDFloat64: - return super.NewBool(compare(val.Float(), pattern)) - } - return super.False - }, nil -} - -var compareString = map[string]func(string, string) bool{ - "==": func(a, b string) bool { return a == b }, - "!=": func(a, b string) bool { return a != b }, - ">": func(a, b string) bool { return a > b }, - ">=": func(a, b string) bool { return a >= b }, - "<": func(a, b string) bool { return a < b }, - "<=": func(a, b string) bool { return a <= b }, -} - -func CompareString(op string, pattern []byte) (Boolean, error) { - compare, ok := compareString[op] - if !ok { - return nil, fmt.Errorf("unknown string comparator: %s", op) - } - s := string(pattern) - return func(val super.Value) super.Value { - val = val.Under() - if val.IsNull() { - return super.Null - } - if val.IsError() { - return val - } - if val.Type().ID() == super.IDString { - return super.NewBool(compare(byteconv.UnsafeString(val.Bytes()), s)) - } - return super.False - }, nil -} - -var compareBytes = map[string]func([]byte, []byte) bool{ - "==": func(a, b []byte) bool { return bytes.Equal(a, b) }, - "!=": func(a, b []byte) bool { return !bytes.Equal(a, b) }, - ">": func(a, b []byte) bool { return bytes.Compare(a, b) > 0 }, - ">=": func(a, b []byte) bool { return bytes.Compare(a, b) >= 0 }, - "<": func(a, b []byte) bool { return bytes.Compare(a, b) < 0 }, - "<=": func(a, b []byte) bool { return bytes.Compare(a, b) <= 0 }, -} - -func CompareBytes(op string, pattern []byte) (Boolean, error) { - compare, ok := compareBytes[op] - if !ok { - return nil, fmt.Errorf("unknown bytes comparator: %s", op) - } - return func(val super.Value) super.Value { - val = val.Under() - if val.IsNull() { - return super.Null - } - if val.IsError() { - return val - } - v := false - switch val.Type().ID() { - case super.IDBytes, super.IDType: - v = compare(val.Bytes(), pattern) - } - return super.NewBool(v) - }, nil -} - func CompileRegexp(pattern string) (*regexp.Regexp, error) { re, err := regexp.Compile(pattern) if err != nil { @@ -238,70 +21,3 @@ func CompileRegexp(pattern string) (*regexp.Regexp, error) { } return re, err } - -// NewRegexpBoolean returns a Boolean that compares values that must -// be a stringy the given regexp. -func NewRegexpBoolean(re *regexp.Regexp) Boolean { - return func(val super.Value) super.Value { - if val.IsNull() { - return super.Null - } - v := false - if val.IsString() { - v = re.Match(val.Bytes()) - } - return super.NewBool(v) - } -} - -func CompareNull(op string) (Boolean, error) { - return func(val super.Value) super.Value { - if val.IsError() { - return val - } - return super.Null - }, nil -} - -// Given a predicate for comparing individual elements, produce a new -// predicate that implements the "in" comparison. -func Contains(compare Boolean) Boolean { - return func(val super.Value) super.Value { - err := val.Walk(func(typ super.Type, body scode.Bytes) error { - if compare(super.NewValue(typ, body)) == super.True { - return errMatch - } - return nil - }) - return super.NewBool(err == errMatch) - } -} - -// Comparison returns a Predicate for comparing this value to other values. -// The op argument is one of "==", "!=", "<", "<=", ">", ">=". -// See the comments of the various type implementations -// of this method as some types limit the operand to equality and -// the various types handle coercion in different ways. -func Comparison(op string, val super.Value) (Boolean, error) { - if val.IsNull() { - return CompareNull(op) - } - switch super.TypeUnder(val.Type()).(type) { - case *super.TypeOfNull: - return CompareNull(op) - case *super.TypeOfIP: - return CompareIP(op, super.DecodeIP(val.Bytes())) - case *super.TypeOfBool: - return CompareBool(op, val.Bool()) - case *super.TypeOfFloat64: - return CompareFloat64(op, val.Float()) - case *super.TypeOfString: - return CompareString(op, val.Bytes()) - case *super.TypeOfBytes, *super.TypeOfType: - return CompareBytes(op, val.Bytes()) - case *super.TypeOfInt64, *super.TypeOfTime, *super.TypeOfDuration: - return CompareInt64(op, super.DecodeInt(val.Bytes())) - default: - return nil, fmt.Errorf("literal comparison of type %q unsupported", val.Type()) - } -} diff --git a/runtime/sam/expr/coerce/coerce.go b/runtime/sam/expr/coerce/coerce.go index c1767c13e2..71e2ae6fd8 100644 --- a/runtime/sam/expr/coerce/coerce.go +++ b/runtime/sam/expr/coerce/coerce.go @@ -1,50 +1,11 @@ package coerce import ( - "bytes" - "github.com/brimdata/super" "github.com/brimdata/super/sup" "golang.org/x/exp/constraints" ) -func Equal(a, b super.Value) bool { - if a.IsNull() { - return b.IsNull() - } else if b.IsNull() { - // We know a isn't null. - return false - } - switch aid, bid := a.Type().ID(), b.Type().ID(); { - case !super.IsNumber(aid) || !super.IsNumber(bid): - if aid != bid { - return false - } - if aid == super.IDNet { - return super.DecodeNet(a.Bytes()) == super.DecodeNet(b.Bytes()) - } - return bytes.Equal(a.Bytes(), b.Bytes()) - case super.IsFloat(aid): - return a.Float() == ToNumeric[float64](b) - case super.IsFloat(bid): - return b.Float() == ToNumeric[float64](a) - case super.IsSigned(aid): - av := a.Int() - if super.IsUnsigned(bid) { - return uint64(av) == b.Uint() && av >= 0 - } - return av == b.Int() - case super.IsSigned(bid): - bv := b.Int() - if super.IsUnsigned(aid) { - return uint64(bv) == a.Uint() && bv >= 0 - } - return bv == a.Int() - default: - return a.Uint() == b.Uint() - } -} - func ToNumeric[T constraints.Integer | constraints.Float](val super.Value) T { if val.IsNull() { return 0 diff --git a/runtime/sam/expr/dot.go b/runtime/sam/expr/dot.go index 2dc2ecd363..83087e5de7 100644 --- a/runtime/sam/expr/dot.go +++ b/runtime/sam/expr/dot.go @@ -1,7 +1,6 @@ package expr import ( - "errors" "slices" "github.com/brimdata/super" @@ -88,38 +87,3 @@ func (d *DotExpr) evalTypeOfType(b scode.Bytes) super.Value { } return d.sctx.Missing() } - -// DotExprToString returns text for the Evaluator assuming it's a field expr. -func DotExprToString(e Evaluator) (string, error) { - f, err := DotExprToField(e) - if err != nil { - return "", err - } - return f.String(), nil -} - -func DotExprToField(e Evaluator) (field.Path, error) { - switch e := e.(type) { - case *This: - return field.Path{}, nil - case *DotExpr: - lhs, err := DotExprToField(e.record) - if err != nil { - return nil, err - } - return append(lhs, e.field), nil - case *Literal: - return field.Path{e.val.String()}, nil - case *Index: - lhs, err := DotExprToField(e.container) - if err != nil { - return nil, err - } - rhs, err := DotExprToField(e.index) - if err != nil { - return nil, err - } - return append(lhs, rhs...), nil - } - return nil, errors.New("not a field") -} diff --git a/runtime/sam/expr/dropper.go b/runtime/sam/expr/dropper.go deleted file mode 100644 index f6f4d47167..0000000000 --- a/runtime/sam/expr/dropper.go +++ /dev/null @@ -1,113 +0,0 @@ -package expr - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/pkg/field" - "github.com/brimdata/super/scode" -) - -type Dropper struct { - sctx *super.Context - dropMap fieldsMap - dropType map[super.Type]super.Type - emptyType super.Type - builder scode.Builder -} - -func NewDropper(sctx *super.Context, fields field.List) *Dropper { - dropMap := fieldsMap{} - for _, f := range fields { - dropMap.Add(f) - } - return &Dropper{ - sctx: sctx, - dropMap: dropMap, - dropType: make(map[super.Type]super.Type), - emptyType: sctx.MustLookupTypeRecord(nil), - } -} - -func (d *Dropper) recode(b *scode.Builder, typ super.Type, bytes scode.Bytes, outType super.Type, dropMap fieldsMap) { - recType := super.TypeRecordOf(typ) - if recType == nil { - b.Append(bytes) - return - } - outRecType := super.TypeUnder(outType).(*super.TypeRecord) - b.BeginContainer() - it := bytes.Iter() - for k, f := range recType.Fields { - elem := it.Next() - dropMapChild, ok := dropMap[f.Name] - if ok { - if dropMapChild == nil { - continue - } - d.recode(b, f.Type, elem, outRecType.Fields[k].Type, dropMapChild) - } else { - b.Append(elem) - } - } - b.EndContainer() -} - -func (d *Dropper) Eval(in super.Value) super.Value { - typ := in.Type() - dropType, ok := d.dropType[typ] - if !ok { - dropType = d.dropMap.dropType(d.sctx, typ) - d.dropType[typ] = dropType - } - if dropType == typ { - return in - } - b := &d.builder - b.Reset() - d.recode(b, typ, in.Bytes(), dropType, d.dropMap) - return super.NewValue(dropType, b.Bytes().Body()) -} - -type fieldsMap map[string]fieldsMap - -func (f fieldsMap) Add(path field.Path) { - if len(path) == 1 { - f[path[0]] = nil - } else if len(path) > 1 { - ff, ok := f[path[0]] - if ff == nil { - if ok { - return - } - ff = fieldsMap{} - f[path[0]] = ff - } - ff.Add(path[1:]) - } -} - -func (f fieldsMap) dropType(sctx *super.Context, typ super.Type) super.Type { - if named, ok := typ.(*super.TypeNamed); ok { - inner := f.dropType(sctx, named.Type) - if inner == named.Type { - return typ - } - return inner - } - recType := super.TypeRecordOf(typ) - if recType == nil { - return typ - } - var out []super.Field - for _, field := range recType.Fields { - typ := field.Type - ff, ok := f[field.Name] - if ok { - if ff == nil { - continue - } - typ = ff.dropType(sctx, typ) - } - out = append(out, super.NewField(field.Name, typ)) - } - return sctx.MustLookupTypeRecord(out) -} diff --git a/runtime/sam/expr/eval.go b/runtime/sam/expr/eval.go index 489f1a7245..6def47f662 100644 --- a/runtime/sam/expr/eval.go +++ b/runtime/sam/expr/eval.go @@ -3,15 +3,10 @@ package expr import ( "bytes" "cmp" - "errors" - "fmt" - "math" - "regexp" "github.com/brimdata/super" "github.com/brimdata/super/runtime/sam/expr/coerce" "github.com/brimdata/super/scode" - "github.com/brimdata/super/sup" ) type Evaluator interface { @@ -22,45 +17,6 @@ type Function interface { Call([]super.Value) super.Value } -type Not struct { - sctx *super.Context - expr Evaluator -} - -var _ Evaluator = (*Not)(nil) - -func NewLogicalNot(sctx *super.Context, e Evaluator) *Not { - return &Not{sctx, e} -} - -func (n *Not) Eval(this super.Value) super.Value { - val := EvalBool(n.sctx, this, n.expr) - if val.IsError() || val.IsNull() { - return val - } - return super.NewBool(!val.Bool()) -} - -type And struct { - sctx *super.Context - lhs Evaluator - rhs Evaluator -} - -func NewLogicalAnd(sctx *super.Context, lhs, rhs Evaluator) *And { - return &And{sctx, lhs, rhs} -} - -type Or struct { - sctx *super.Context - lhs Evaluator - rhs Evaluator -} - -func NewLogicalOr(sctx *super.Context, lhs, rhs Evaluator) *Or { - return &Or{sctx, lhs, rhs} -} - // EvalBool evaluates e with this and returns the result if it is a bool or error. // Otherwise, EvalBool returns an error. func EvalBool(sctx *super.Context, this super.Value, e Evaluator) super.Value { @@ -71,276 +27,6 @@ func EvalBool(sctx *super.Context, this super.Value, e Evaluator) super.Value { return sctx.WrapError("not type bool", val) } -func (a *And) Eval(this super.Value) super.Value { - lhs := EvalBool(a.sctx, this, a.lhs) - rhs := EvalBool(a.sctx, this, a.rhs) - if isfalse(lhs) || isfalse(rhs) { - // anything AND FALSE = FALSE - return super.False - } - // ERROR AND NULL = ERROR - // ERROR AND TRUE = ERROR - if lhs.IsError() { - return lhs - } - if rhs.IsError() { - return rhs - } - if lhs.IsNull() || rhs.IsNull() { - // NULL AND TRUE = NULL - return super.Null - } - return super.True -} - -func isfalse(val super.Value) bool { - return val.Type().ID() == super.IDBool && !val.IsNull() && !val.Bool() -} - -func (o *Or) Eval(this super.Value) super.Value { - lhs := EvalBool(o.sctx, this, o.lhs) - rhs := EvalBool(o.sctx, this, o.rhs) - if lhs.AsBool() || rhs.AsBool() { - // anything OR TRUE = TRUE - return super.True - } - if lhs.IsNull() || rhs.IsNull() { - // NULL OR FALSE = NULL - // NULL OR ERROR = NULL - return super.Null - } - // ERROR OR FALSE = ERROR - if lhs.IsError() { - return lhs - } - if rhs.IsError() { - return rhs - } - return super.False -} - -type In struct { - sctx *super.Context - elem Evaluator - container Evaluator -} - -func NewIn(sctx *super.Context, elem, container Evaluator) *In { - return &In{ - sctx: sctx, - elem: elem, - container: container, - } -} - -func (i *In) Eval(this super.Value) super.Value { - elem := i.elem.Eval(this).DeunionIntoNameds() - if elem.IsError() { - return elem - } - if elem.IsNull() { - return super.Null - } - container := i.container.Eval(this) - if container.IsError() { - return container - } - var hasnull bool - err := container.Walk(func(typ super.Type, body scode.Bytes) error { - if super.TypeUnder(typ) == super.TypeNull { - hasnull = true - } - if coerce.Equal(elem, super.NewValue(typ, body)) { - return errMatch - } - return nil - }) - switch err { - case errMatch: - return super.True - case nil: - if hasnull { - return super.Null - } - return super.False - default: - return i.sctx.NewError(err) - } -} - -type Equal struct { - numeric - equality bool -} - -func NewCompareEquality(sctx *super.Context, lhs, rhs Evaluator, operator string) (*Equal, error) { - e := &Equal{numeric: newNumeric(sctx, lhs, rhs)} //XXX - switch operator { - case "==": - e.equality = true - case "!=": - default: - return nil, fmt.Errorf("unknown equality operator: %s", operator) - } - return e, nil -} - -func (e *Equal) Eval(this super.Value) super.Value { - lhsVal, rhsVal, errVal := e.numeric.eval(this) - if errVal != nil { - return *errVal - } - if lhsVal.IsNull() || rhsVal.IsNull() { - return super.Null - } - result := coerce.Equal(lhsVal, rhsVal) - if !e.equality { - result = !result - } - if result { - return super.True - } - return super.False -} - -type RegexpMatch struct { - re *regexp.Regexp - expr Evaluator -} - -func NewRegexpMatch(re *regexp.Regexp, e Evaluator) *RegexpMatch { - return &RegexpMatch{re, e} -} - -func (r *RegexpMatch) Eval(this super.Value) super.Value { - val := r.expr.Eval(this).Under() - switch id := val.Type().ID(); id { - case super.IDString: - if val.IsNull() { - return super.Null - } - if r.re.Match(val.Bytes()) { - return super.True - } - case super.IDNull: - return super.Null - } - return super.False -} - -type numeric struct { - sctx *super.Context - lhs Evaluator - rhs Evaluator -} - -func newNumeric(sctx *super.Context, lhs, rhs Evaluator) numeric { - return numeric{ - sctx: sctx, - lhs: lhs, - rhs: rhs, - } -} - -func (n *numeric) evalAndPromote(this super.Value) (super.Value, super.Value, super.Type, *super.Value) { - lhsVal, rhsVal, errVal := n.eval(this) - if errVal != nil { - return super.Null, super.Null, nil, errVal - } - id, err := coerce.Promote(lhsVal, rhsVal) - if err != nil { - return super.Null, super.Null, nil, n.sctx.NewError(err).Ptr() - } - typ, err := n.sctx.LookupType(id) - if err != nil { - return super.Null, super.Null, nil, n.sctx.NewError(err).Ptr() - } - return lhsVal, rhsVal, typ, nil -} - -func (n *numeric) eval(this super.Value) (super.Value, super.Value, *super.Value) { - lhs := n.lhs.Eval(this).Under() - if lhs.IsNull() { - return super.Null, super.Null, &super.Null - } - rhs := n.rhs.Eval(this).Under() - if lhs.IsNull() || rhs.IsNull() { - return super.Null, super.Null, &super.Null - } - if lhs.IsError() { - return super.Null, super.Null, &lhs - } - if rhs.IsError() { - return super.Null, super.Null, &rhs - } - return lhs, rhs, nil -} - -type Compare struct { - sctx *super.Context - numeric - operator string - convert func(int) bool -} - -func NewCompareRelative(sctx *super.Context, lhs, rhs Evaluator, operator string) (*Compare, error) { - c := &Compare{sctx: sctx, numeric: newNumeric(sctx, lhs, rhs), operator: operator} - switch operator { - case "<": - c.convert = func(v int) bool { return v < 0 } - case "<=": - c.convert = func(v int) bool { return v <= 0 } - case ">": - c.convert = func(v int) bool { return v > 0 } - case ">=": - c.convert = func(v int) bool { return v >= 0 } - default: - return nil, fmt.Errorf("unknown comparison operator: %s", operator) - } - return c, nil -} - -func (c *Compare) result(result int) super.Value { - return super.NewBool(c.convert(result)) -} - -func (c *Compare) Eval(this super.Value) super.Value { - lhs := c.lhs.Eval(this) - if lhs.IsError() { - return lhs - } - rhs := c.rhs.Eval(this) - if rhs.IsError() { - return rhs - } - lhs, rhs = lhs.Under(), rhs.Under() - if lhs.IsNull() || rhs.IsNull() { - return super.Null - } - switch lid, rid := lhs.Type().ID(), rhs.Type().ID(); { - case super.IsNumber(lid) && super.IsNumber(rid): - return c.result(compareNumbers(lhs, rhs, lid, rid)) - case lid != rid: - return super.False - case lid == super.IDBool: - v := lhs.Bool() == rhs.Bool() && (c.operator == "<=" || c.operator == ">=") - return super.NewBool(v) - case lid == super.IDBytes: - return c.result(bytes.Compare(super.DecodeBytes(lhs.Bytes()), super.DecodeBytes(rhs.Bytes()))) - case lid == super.IDString: - return c.result(cmp.Compare(super.DecodeString(lhs.Bytes()), super.DecodeString(rhs.Bytes()))) - case lid == super.IDIP: - return c.result(super.DecodeIP(lhs.Bytes()).Compare(super.DecodeIP(rhs.Bytes()))) - case lid == super.IDNet: - return c.sctx.NewErrorf("type net incompatible with '%s' operator", c.operator) - default: - if bytes.Equal(lhs.Bytes(), rhs.Bytes()) { - return c.result(0) - } - } - return super.False -} - func compareNumbers(a, b super.Value, aid, bid int) int { switch { case super.IsFloat(aid): @@ -369,262 +55,7 @@ func compareNumbers(a, b super.Value, aid, bid int) int { return cmp.Compare(a.Uint(), b.Uint()) } -type isNullExpr struct { - eval Evaluator -} - -func NewIsNullExpr(e Evaluator) Evaluator { - return &isNullExpr{e} -} - -func (i *isNullExpr) Eval(this super.Value) super.Value { - val := i.eval.Eval(this).Under() - if val.IsError() { - return val - } - return super.NewBool(val.IsNull()) - -} - func toFloat(val super.Value) float64 { return coerce.ToNumeric[float64](val) } -func toInt(val super.Value) int64 { return coerce.ToNumeric[int64](val) } -func toUint(val super.Value) uint64 { return coerce.ToNumeric[uint64](val) } - -type Add struct { - sctx *super.Context - operands numeric -} - -type Subtract struct { - sctx *super.Context - operands numeric -} - -type Multiply struct { - sctx *super.Context - operands numeric -} - -type Divide struct { - sctx *super.Context - operands numeric -} - -type Modulo struct { - sctx *super.Context - operands numeric -} - -var DivideByZero = errors.New("divide by zero") - -// NewArithmetic compiles an expression of the form "expr1 op expr2" -// for the arithmetic operators +, -, *, / -func NewArithmetic(sctx *super.Context, op string, lhs, rhs Evaluator) (Evaluator, error) { - n := newNumeric(sctx, lhs, rhs) - switch op { - case "+": - return &Add{sctx: sctx, operands: n}, nil - case "-": - return &Subtract{sctx: sctx, operands: n}, nil - case "*": - return &Multiply{sctx: sctx, operands: n}, nil - case "/": - return &Divide{sctx: sctx, operands: n}, nil - case "%": - return &Modulo{sctx: sctx, operands: n}, nil - } - return nil, fmt.Errorf("unknown arithmetic operator: %s", op) -} - -func (a *Add) Eval(this super.Value) super.Value { - lhsVal, rhsVal, typ, errVal := a.operands.evalAndPromote(this) - if errVal != nil { - return *errVal - } - switch id := typ.ID(); { - case super.IsUnsigned(id): - return super.NewUint(typ, toUint(lhsVal)+toUint(rhsVal)) - case super.IsSigned(id): - return super.NewInt(typ, toInt(lhsVal)+toInt(rhsVal)) - case super.IsFloat(id): - return super.NewFloat(typ, toFloat(lhsVal)+toFloat(rhsVal)) - } - return a.sctx.NewErrorf("type %s incompatible with '+' operator", sup.FormatType(typ)) -} - -func (s *Subtract) Eval(this super.Value) super.Value { - lhsVal, rhsVal, typ, errVal := s.operands.evalAndPromote(this) - if errVal != nil { - return *errVal - } - switch id := typ.ID(); { - case super.IsUnsigned(id): - return super.NewUint(typ, toUint(lhsVal)-toUint(rhsVal)) - case super.IsSigned(id): - if lhsVal.Type().ID() == super.IDTime && rhsVal.Type().ID() == super.IDDuration { - // Time minus duration should be time - typ = super.TypeTime - } else if id == super.IDTime { - // Return the difference of two times as a duration. - typ = super.TypeDuration - } - return super.NewInt(typ, toInt(lhsVal)-toInt(rhsVal)) - case super.IsFloat(id): - return super.NewFloat(typ, toFloat(lhsVal)-toFloat(rhsVal)) - } - return s.sctx.NewErrorf("type %s incompatible with '-' operator", sup.FormatType(typ)) -} - -func (m *Multiply) Eval(this super.Value) super.Value { - lhsVal, rhsVal, typ, errVal := m.operands.evalAndPromote(this) - if errVal != nil { - return *errVal - } - switch id := typ.ID(); { - case super.IsUnsigned(id): - return super.NewUint(typ, toUint(lhsVal)*toUint(rhsVal)) - case super.IsSigned(id): - return super.NewInt(typ, toInt(lhsVal)*toInt(rhsVal)) - case super.IsFloat(id): - return super.NewFloat(typ, toFloat(lhsVal)*toFloat(rhsVal)) - } - return m.sctx.NewErrorf("type %s incompatible with '*' operator", sup.FormatType(typ)) -} - -func (d *Divide) Eval(this super.Value) super.Value { - lhsVal, rhsVal, typ, errVal := d.operands.evalAndPromote(this) - if errVal != nil { - return *errVal - } - switch id := typ.ID(); { - case super.IsUnsigned(id): - v := toUint(rhsVal) - if v == 0 { - return d.sctx.NewError(DivideByZero) - } - return super.NewUint(typ, toUint(lhsVal)/v) - case super.IsSigned(id): - v := toInt(rhsVal) - if v == 0 { - return d.sctx.NewError(DivideByZero) - } - return super.NewInt(typ, toInt(lhsVal)/v) - case super.IsFloat(id): - return super.NewFloat(typ, toFloat(lhsVal)/toFloat(rhsVal)) - } - return d.sctx.NewErrorf("type %s incompatible with '/' operator", sup.FormatType(typ)) -} - -func (m *Modulo) Eval(this super.Value) super.Value { - lhsVal, rhsVal, typ, errVal := m.operands.evalAndPromote(this) - if errVal != nil { - return *errVal - } - switch id := typ.ID(); { - case super.IsUnsigned(id): - v := toUint(rhsVal) - if v == 0 { - return m.sctx.NewError(DivideByZero) - } - return super.NewUint(typ, lhsVal.Uint()%v) - case super.IsSigned(id): - v := toInt(rhsVal) - if v == 0 { - return m.sctx.NewError(DivideByZero) - } - return super.NewInt(typ, toInt(lhsVal)%v) - } - return m.sctx.NewErrorf("type %s incompatible with '%%' operator", sup.FormatType(typ)) -} - -type UnaryMinus struct { - sctx *super.Context - expr Evaluator -} - -func NewUnaryMinus(sctx *super.Context, e Evaluator) *UnaryMinus { - return &UnaryMinus{ - sctx: sctx, - expr: e, - } -} - -func (u *UnaryMinus) Eval(this super.Value) super.Value { - val := u.expr.Eval(this).Under() - if val.IsNull() || val.IsError() { - return val - } - typ := val.Type() - if super.IsUnsigned(typ.ID()) { - switch typ.ID() { - case super.IDUint8: - typ = super.TypeInt8 - case super.IDUint16: - typ = super.TypeInt16 - case super.IDUint32: - typ = super.TypeInt32 - default: - typ = super.TypeInt64 - } - v, ok := coerce.ToInt(val, typ) - if !ok { - return u.sctx.WrapError("cannot cast to "+sup.FormatType(typ), val) - } - if val.IsNull() { - return super.Null - } - val = super.NewInt(typ, v) - } - switch typ.ID() { - case super.IDFloat16, super.IDFloat32, super.IDFloat64: - return super.NewFloat(typ, -val.Float()) - case super.IDInt8: - v := val.Int() - if v == math.MinInt8 { - return u.sctx.WrapError("unary '-' underflow", val) - } - return super.NewInt8(int8(-v)) - case super.IDInt16: - v := val.Int() - if v == math.MinInt16 { - return u.sctx.WrapError("unary '-' underflow", val) - } - return super.NewInt16(int16(-v)) - case super.IDInt32: - v := val.Int() - if v == math.MinInt32 { - return u.sctx.WrapError("unary '-' underflow", val) - } - return super.NewInt32(int32(-v)) - case super.IDInt64: - v := val.Int() - if v == math.MinInt64 { - return u.sctx.WrapError("unary '-' underflow", val) - } - return super.NewInt64(-v) - } - return u.sctx.WrapError("type incompatible with unary '-' operator", val) -} - -func getNthFromContainer(container scode.Bytes, idx int) (scode.Bytes, int) { - if idx < 0 { - var length int - for it := container.Iter(); !it.Done(); it.Next() { - length++ - } - idx = length + idx - if idx < 0 || idx >= length { - return nil, -1 - } - } - for i, it := 0, container.Iter(); !it.Done(); i++ { - zv := it.Next() - if i == idx { - return zv, idx - } - } - return nil, -1 -} func getNthFromRecord(typ *super.TypeRecord, container scode.Bytes, idx int) (scode.Bytes, int) { if idx < 0 { @@ -654,103 +85,6 @@ func lookupKey(mapBytes, target scode.Bytes) (scode.Bytes, bool) { return nil, false } -// Index represents an index operator "container[index]" where container is -// either an array (with index type integer) or a record (with index type string). -type Index struct { - sctx *super.Context - container Evaluator - index Evaluator - base1 bool -} - -func NewIndexExpr(sctx *super.Context, container, index Evaluator, base1 bool) Evaluator { - return &Index{sctx, container, index, base1} -} - -func (i *Index) Eval(this super.Value) super.Value { - container := i.container.Eval(this) - index := i.index.Eval(this) - switch typ := super.TypeUnder(container.Type()).(type) { - case *super.TypeArray, *super.TypeSet: - idx, ok, err := getNumericIndex(i.sctx, index, i.base1) - if err != nil { - return *err - } - if !ok { - return i.sctx.WrapError("index is not an integer", index) - } - return indexArrayOrSet(i.sctx, super.InnerType(typ), container.Bytes(), idx) - case *super.TypeRecord: - idx, ok, err := getNumericIndex(i.sctx, index, i.base1) - if err != nil { - return *err - } - if ok { - return indexRecordByIndex(i.sctx, typ, container.Bytes(), idx) - } - return indexRecordByName(i.sctx, typ, container.Bytes(), index) - case *super.TypeMap: - return indexMap(i.sctx, typ, container.Bytes(), index) - default: - return i.sctx.Missing() - } -} - -func getNumericIndex(sctx *super.Context, index super.Value, base1 bool) (int, bool, *super.Value) { - id := index.Type().ID() - if super.IsUnsigned(id) { - index = LookupPrimitiveCaster(sctx, super.TypeInt64).Eval(index) - id = index.Type().ID() - } - if !super.IsInteger(index.Type().ID()) { - if index.IsError() { - return 0, false, index.Ptr() - } - return 0, false, nil - } - if index.IsNull() { - return 0, false, sctx.Missing().Ptr() - } - idx := int(index.AsInt()) - if base1 { - if idx == 0 { - return -1, false, sctx.Missing().Ptr() - } - if idx > 0 { - idx-- - } - } - return idx, true, nil -} - -func indexArrayOrSet(sctx *super.Context, inner super.Type, vector scode.Bytes, idx int) super.Value { - bytes, idx := getNthFromContainer(vector, idx) - if idx < 0 { - return sctx.Missing() - } - return deunion(inner, bytes) -} - -func indexRecordByIndex(sctx *super.Context, typ *super.TypeRecord, record scode.Bytes, idx int) super.Value { - bytes, idx := getNthFromRecord(typ, record, idx) - if idx < 0 { - return sctx.Missing() - } - return super.NewValue(typ.Fields[idx].Type, bytes) -} - -func indexRecordByName(sctx *super.Context, typ *super.TypeRecord, record scode.Bytes, index super.Value) super.Value { - if index.Type().ID() != super.IDString { - return sctx.WrapError("invalid value for record index", index) - } - field := super.DecodeString(index.Bytes()) - val := super.NewValue(typ, record).Ptr().Deref(field) - if val.IsMissing() { - return sctx.Missing() - } - return *val -} - func indexMap(sctx *super.Context, typ *super.TypeMap, mapBytes scode.Bytes, key super.Value) super.Value { if key.IsMissing() { return sctx.Missing() @@ -779,61 +113,3 @@ func deunion(typ super.Type, b scode.Bytes) super.Value { } return super.NewValue(typ, b) } - -type Conditional struct { - sctx *super.Context - predicate Evaluator - thenExpr Evaluator - elseExpr Evaluator -} - -func NewConditional(sctx *super.Context, predicate, thenExpr, elseExpr Evaluator) *Conditional { - return &Conditional{ - sctx: sctx, - predicate: predicate, - thenExpr: thenExpr, - elseExpr: elseExpr, - } -} - -func (c *Conditional) Eval(this super.Value) super.Value { - val := c.predicate.Eval(this).DeunionIntoNameds() - switch typ := val.Type(); { - case typ.ID() == super.IDBool: - if val.Bool() { - return c.thenExpr.Eval(this) - } - return c.elseExpr.Eval(this) - case typ.ID() == super.IDNull: - return c.elseExpr.Eval(this) - case typ.Kind() == super.ErrorKind: - return val - } - return c.sctx.WrapError("?-operator: bool predicate required", val) -} - -type Call struct { - fn Function - exprs []Evaluator - args []super.Value -} - -func NewCall(fn Function, exprs []Evaluator) *Call { - return &Call{ - fn: fn, - exprs: exprs, - args: make([]super.Value, len(exprs)), - } -} - -func (c *Call) Eval(this super.Value) super.Value { - for k, e := range c.exprs { - c.args[k] = e.Eval(this) - } - return c.fn.Call(c.args) -} - -type Assignment struct { - LHS *Lval - RHS Evaluator -} diff --git a/runtime/sam/expr/filter.go b/runtime/sam/expr/filter.go index daa03fcf45..90134765ee 100644 --- a/runtime/sam/expr/filter.go +++ b/runtime/sam/expr/filter.go @@ -1,54 +1,11 @@ package expr import ( - "bytes" - "errors" - "net/netip" "strings" "github.com/brimdata/super" - "github.com/brimdata/super/pkg/byteconv" - "github.com/brimdata/super/scode" ) -type searchByPred struct { - pred Boolean - expr Evaluator - fnm *FieldNameMatcher -} - -func SearchByPredicate(pred Boolean, e Evaluator) Evaluator { - return &searchByPred{ - pred: pred, - expr: e, - fnm: NewFieldNameMatcher(func(b []byte) bool { - val := super.NewValue(super.TypeString, b) - return pred(val).Ptr().AsBool() - }), - } -} - -func (s *searchByPred) Eval(val super.Value) super.Value { - if s.expr != nil { - val = s.expr.Eval(val) - if val.IsError() { - return super.False - } - } - if s.fnm.Match(val.Type()) { - return super.True - } - if errMatch == val.Walk(func(typ super.Type, body scode.Bytes) error { - if s.pred(super.NewValue(typ, body)).Ptr().AsBool() { - return errMatch - } - return nil - }) { - return super.True - } - return super.False -} - // StringContainsFold is like strings.Contains but with case-insensitive // comparison. func StringContainsFold(a, b string) bool { @@ -70,138 +27,6 @@ func StringContainsFold(a, b string) bool { return false } -var errMatch = errors.New("match") - -type search struct { - text string - compare Boolean - expr Evaluator -} - -// NewSearch creates a filter that searches records for the -// given value, which must be of a type other than string. The filter -// matches a record that contains this value either as the value of any -// field or inside any set or array. It also matches a record if the string -// representaton of the search value appears inside inside any string-valued -// field (or inside any element of a set or array of strings). -func NewSearch(searchtext string, searchval super.Value, expr Evaluator) (Evaluator, error) { - if super.TypeUnder(searchval.Type()) == super.TypeNet { - return &searchCIDR{ - net: super.DecodeNet(searchval.Bytes()), - bytes: searchval.Bytes(), - }, nil - } - typedCompare, err := Comparison("==", searchval) - if err != nil { - return nil, err - } - return &search{searchtext, typedCompare, expr}, nil -} - -func (s *search) Eval(val super.Value) super.Value { - if s.expr != nil { - val = s.expr.Eval(val) - if val.IsError() { - return super.False - } - } - if errMatch == val.Walk(func(typ super.Type, body scode.Bytes) error { - if typ.ID() == super.IDString { - if StringContainsFold(byteconv.UnsafeString(body), s.text) { - return errMatch - } - return nil - } - if s.compare(super.NewValue(typ, body)).Ptr().AsBool() { - return errMatch - } - return nil - }) { - return super.True - } - return super.False -} - -type searchCIDR struct { - net netip.Prefix - bytes scode.Bytes -} - -func (s *searchCIDR) Eval(val super.Value) super.Value { - if errMatch == val.Walk(func(typ super.Type, body scode.Bytes) error { - switch typ.ID() { - case super.IDNet: - if bytes.Equal(body, s.bytes) { - return errMatch - } - case super.IDIP: - if s.net.Contains(super.DecodeIP(body)) { - return errMatch - } - } - return nil - }) { - return super.True - } - return super.False -} - -type searchString struct { - term string - expr Evaluator - fnm *FieldNameMatcher -} - -// NewSearchString is like NewSeach but handles the special case of matching -// field names in addition to string values. -func NewSearchString(term string, expr Evaluator) Evaluator { - return &searchString{ - term: term, - expr: expr, - fnm: NewFieldNameMatcher(func(b []byte) bool { - return StringContainsFold(byteconv.UnsafeString(b), term) - }), - } -} - -func (s *searchString) Eval(val super.Value) super.Value { - if s.expr != nil { - val = s.expr.Eval(val) - if val.IsError() { - return super.False - } - } - if s.fnm.Match(val.Type()) { - return super.True - } - if errMatch == val.Walk(func(typ super.Type, body scode.Bytes) error { - if typ.ID() == super.IDString && - StringContainsFold(byteconv.UnsafeString(body), s.term) { - return errMatch - } - return nil - }) { - return super.True - } - return super.False -} - -type filterApplier struct { - sctx *super.Context - expr Evaluator -} - -func NewFilterApplier(sctx *super.Context, e Evaluator) Evaluator { - return &filterApplier{sctx, e} -} - func IsTrue(val super.Value) bool { return val.Deunion().Ptr().AsBool() } - -func (f *filterApplier) Eval(this super.Value) super.Value { - if !IsTrue(f.expr.Eval(this)) { - return f.sctx.Missing() - } - return this -} diff --git a/runtime/sam/expr/function/bytes.go b/runtime/sam/expr/function/bytes.go deleted file mode 100644 index dace783329..0000000000 --- a/runtime/sam/expr/function/bytes.go +++ /dev/null @@ -1,54 +0,0 @@ -package function - -import ( - "encoding/base64" - "encoding/hex" - - "github.com/brimdata/super" -) - -type Base64 struct { - sctx *super.Context -} - -func (b *Base64) Call(args []super.Value) super.Value { - val := args[0].Under() - if val.IsNull() { - return super.Null - } - switch val.Type().ID() { - case super.IDBytes: - return super.NewString(base64.StdEncoding.EncodeToString(val.Bytes())) - case super.IDString: - bytes, err := base64.StdEncoding.DecodeString(super.DecodeString(val.Bytes())) - if err != nil { - return b.sctx.WrapError("base64: string argument is not base64", val) - } - return super.NewBytes(bytes) - default: - return b.sctx.WrapError("base64: argument must a bytes or string type", val) - } -} - -type Hex struct { - sctx *super.Context -} - -func (h *Hex) Call(args []super.Value) super.Value { - val := args[0].Under() - if val.IsNull() { - return super.Null - } - switch val.Type().ID() { - case super.IDBytes: - return super.NewString(hex.EncodeToString(val.Bytes())) - case super.IDString: - b, err := hex.DecodeString(super.DecodeString(val.Bytes())) - if err != nil { - return h.sctx.WrapError("hex: string argument is not hexadecimal", val) - } - return super.NewBytes(b) - default: - return h.sctx.WrapError("base64: argument must a bytes or string type", val) - } -} diff --git a/runtime/sam/expr/function/coalesce.go b/runtime/sam/expr/function/coalesce.go deleted file mode 100644 index f3e4fa21d0..0000000000 --- a/runtime/sam/expr/function/coalesce.go +++ /dev/null @@ -1,15 +0,0 @@ -package function - -import "github.com/brimdata/super" - -type Coalesce struct{} - -func (c *Coalesce) Call(args []super.Value) super.Value { - for i := range args { - val := args[i].Under() - if !val.IsNull() && !val.IsError() { - return args[i] - } - } - return super.Null -} diff --git a/runtime/sam/expr/function/fields.go b/runtime/sam/expr/function/fields.go deleted file mode 100644 index fd8af765de..0000000000 --- a/runtime/sam/expr/function/fields.go +++ /dev/null @@ -1,63 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/scode" -) - -type Fields struct { - sctx *super.Context - typ super.Type -} - -func NewFields(sctx *super.Context) *Fields { - return &Fields{ - sctx: sctx, - typ: sctx.LookupTypeArray(sctx.LookupTypeArray(super.TypeString)), - } -} - -func buildPath(typ *super.TypeRecord, b *scode.Builder, prefix []string) { - for _, f := range typ.Fields { - if typ, ok := super.TypeUnder(f.Type).(*super.TypeRecord); ok { - buildPath(typ, b, append(prefix, f.Name)) - } else { - b.BeginContainer() - for _, s := range prefix { - b.Append([]byte(s)) - } - b.Append([]byte(f.Name)) - b.EndContainer() - } - } -} - -func (f *Fields) Call(args []super.Value) super.Value { - subjectVal := args[0].Under() - if subjectVal.IsNull() { - return super.Null - } - typ := f.recordType(subjectVal) - if typ == nil { - return f.sctx.Missing() - } - var b scode.Builder - buildPath(typ, &b, nil) - return super.NewValue(f.typ, b.Bytes()) -} - -func (f *Fields) recordType(val super.Value) *super.TypeRecord { - if typ, ok := super.TypeUnder(val.Type()).(*super.TypeRecord); ok { - return typ - } - if val.Type() == super.TypeType { - typ, err := f.sctx.LookupByValue(val.Bytes()) - if err != nil { - return nil - } - if typ, ok := super.TypeUnder(typ).(*super.TypeRecord); ok { - return typ - } - } - return nil -} diff --git a/runtime/sam/expr/function/function.go b/runtime/sam/expr/function/function.go index fe15a38117..cfcfc81ea1 100644 --- a/runtime/sam/expr/function/function.go +++ b/runtime/sam/expr/function/function.go @@ -7,7 +7,6 @@ import ( ) var ( - ErrBadArgument = errors.New("bad argument") ErrNoSuchFunction = errors.New("no such function") ErrTooFewArgs = errors.New("too few arguments") ErrTooManyArgs = errors.New("too many arguments") diff --git a/runtime/sam/expr/function/fusion.go b/runtime/sam/expr/function/fusion.go deleted file mode 100644 index d7191d241a..0000000000 --- a/runtime/sam/expr/function/fusion.go +++ /dev/null @@ -1,37 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/scode" - "github.com/brimdata/super/sup" -) - -type fusion struct { - sctx *super.Context - downcast Caster -} - -func newFusion(sctx *super.Context) *fusion { - return &fusion{ - sctx: sctx, - downcast: newDowncast(sctx), - } -} - -func (f *fusion) Call(args []super.Value) super.Value { - superVal, subTypeVal := args[0], args[1] - if _, ok := super.TypeUnder(subTypeVal.Type()).(*super.TypeOfType); !ok { - return f.sctx.WrapError("fusion: super type argument not a type", subTypeVal) - } - subType, err := f.sctx.LookupByValue(subTypeVal.Bytes()) - if err != nil { - panic(err) - } - if _, ok := f.downcast.Cast(superVal, subType); !ok { - return f.sctx.WrapError("fusion: value not a supertype of subtype arg: "+sup.FormatType(subType), superVal) - } - typ := f.sctx.LookupTypeFusion(superVal.Type()) - var b scode.Builder - super.BuildFusion(&b, superVal.Bytes(), subTypeVal.Bytes()) - return super.NewValue(typ, b.Bytes().Body()) -} diff --git a/runtime/sam/expr/function/grep.go b/runtime/sam/expr/function/grep.go deleted file mode 100644 index 988c140b19..0000000000 --- a/runtime/sam/expr/function/grep.go +++ /dev/null @@ -1,29 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/runtime/sam/expr" - "golang.org/x/text/unicode/norm" -) - -type Grep struct { - grep expr.Evaluator - pattern string - sctx *super.Context -} - -func (g *Grep) Call(vals []super.Value) super.Value { - patternVal, inputVal := vals[0], vals[1] - if super.TypeUnder(patternVal.Type()) != super.TypeString { - return g.sctx.WrapError("grep: pattern argument must be a string", patternVal) - } - if patternVal.IsNull() { - return super.Null - } - if p := patternVal.AsString(); g.grep == nil || g.pattern != p { - g.pattern = p - term := norm.NFC.Bytes(patternVal.Bytes()) - g.grep = expr.NewSearchString(string(term), nil) - } - return g.grep.Eval(inputVal) -} diff --git a/runtime/sam/expr/function/grok.go b/runtime/sam/expr/function/grok.go deleted file mode 100644 index 5ada7e1ea6..0000000000 --- a/runtime/sam/expr/function/grok.go +++ /dev/null @@ -1,103 +0,0 @@ -package function - -import ( - "strings" - - "github.com/brimdata/super" - "github.com/brimdata/super/pkg/grok" - "github.com/brimdata/super/scode" -) - -type Grok struct { - sctx *super.Context - builder scode.Builder - hosts map[string]*host - // fields is used as a scratch space to avoid allocating a new slice. - fields []super.Field -} - -func newGrok(sctx *super.Context) *Grok { - return &Grok{ - sctx: sctx, - hosts: make(map[string]*host), - } -} - -func (g *Grok) Call(args []super.Value) super.Value { - patternArg, inputArg, defArg := args[0].Under(), args[1].Under(), super.NewString("") - if len(args) == 3 { - defArg = args[2].Under() - } - if patternArg.IsNull() || inputArg.IsNull() || defArg.IsNull() { - return super.Null - } - switch { - case super.TypeUnder(defArg.Type()) != super.TypeString: - return g.error("definitions argument must be a string", defArg) - case super.TypeUnder(patternArg.Type()) != super.TypeString: - return g.error("pattern argument must be a string", patternArg) - case super.TypeUnder(inputArg.Type()) != super.TypeString: - return g.error("input argument must be a string", inputArg) - } - h, err := g.getHost(defArg.AsString()) - if err != nil { - return g.error(err.Error(), defArg) - } - p, err := h.getPattern(patternArg.AsString()) - if err != nil { - return g.error(err.Error(), patternArg) - } - keys, vals, match := p.ParseKeyValues(inputArg.AsString()) - if !match { - return g.error("value does not match pattern", inputArg) - } - g.fields = g.fields[:0] - for _, key := range keys { - g.fields = append(g.fields, super.NewField(key, super.TypeString)) - } - typ := g.sctx.MustLookupTypeRecord(g.fields) - g.builder.Reset() - if len(vals) == 0 { - // If we have a match but no key/vals return empty record. - g.builder.Append(nil) - } else { - for _, s := range vals { - g.builder.Append([]byte(s)) - } - } - return super.NewValue(typ, g.builder.Bytes()) -} - -func (g *Grok) error(msg string, val super.Value) super.Value { - return g.sctx.WrapError("grok: "+msg, val) -} - -func (g *Grok) getHost(defs string) (*host, error) { - h, ok := g.hosts[defs] - if !ok { - h = &host{Host: grok.NewBase(), patterns: make(map[string]*grok.Pattern)} - if err := h.AddFromReader(strings.NewReader(defs)); err != nil { - return nil, err - } - g.hosts[defs] = h - } - return h, nil -} - -type host struct { - grok.Host - patterns map[string]*grok.Pattern -} - -func (h *host) getPattern(patternArg string) (*grok.Pattern, error) { - p, ok := h.patterns[patternArg] - if !ok { - var err error - p, err = h.Host.Compile(patternArg) - if err != nil { - return nil, err - } - h.patterns[patternArg] = p - } - return p, nil -} diff --git a/runtime/sam/expr/function/has.go b/runtime/sam/expr/function/has.go deleted file mode 100644 index 7dc854b185..0000000000 --- a/runtime/sam/expr/function/has.go +++ /dev/null @@ -1,40 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" -) - -type Has struct{} - -func (h *Has) Call(args []super.Value) super.Value { - args = underAll(args) - for _, val := range args { - if val.IsNull() { - return super.Null - } - } - for _, val := range args { - if val.IsError() { - if val.IsMissing() { - return super.False - } - return val - } - if val.IsNone() { - return super.False - } - } - return super.True -} - -type Missing struct { - has Has -} - -func (m *Missing) Call(args []super.Value) super.Value { - val := m.has.Call(args) - if val.Type() == super.TypeBool && !val.IsNull() { - return super.NewBool(!val.Bool()) - } - return val -} diff --git a/runtime/sam/expr/function/ip.go b/runtime/sam/expr/function/ip.go deleted file mode 100644 index 14ee7ba2ee..0000000000 --- a/runtime/sam/expr/function/ip.go +++ /dev/null @@ -1,101 +0,0 @@ -package function - -import ( - "errors" - "net/netip" - - "github.com/brimdata/super" - "github.com/brimdata/super/scode" -) - -type NetworkOf struct { - sctx *super.Context -} - -func (n *NetworkOf) Call(args []super.Value) super.Value { - ipVal := args[0].Under() - if ipVal.Type().ID() != super.IDIP { - return n.sctx.WrapError("network_of: not an IP", ipVal) - } - ip := super.DecodeIP(ipVal.Bytes()) - var bits int - if len(args) == 1 { - switch { - case !ip.Is4(): - return n.sctx.WrapError("network_of: not an IPv4 address", ipVal) - case ip.As4()[0] < 0x80: - bits = 8 - case ip.As4()[0] < 0xc0: - bits = 16 - default: - bits = 24 - } - } else { - // two args - maskVal := args[1].Under() - switch id := maskVal.Type().ID(); { - case id == super.IDIP: - mask := super.DecodeIP(maskVal.Bytes()) - if mask.BitLen() != ip.BitLen() { - val := addressAndMask(n.sctx, ipVal, maskVal) - return n.sctx.WrapError("network_of: address and mask have different lengths", val) - } - bits = super.LeadingOnes(mask.AsSlice()) - if netip.PrefixFrom(mask, bits).Masked().Addr() != mask { - return n.sctx.WrapError("network_of: mask is non-contiguous", maskVal) - } - case super.IsInteger(id): - if super.IsSigned(id) { - bits = int(maskVal.Int()) - } else { - bits = int(maskVal.Uint()) - } - if bits > 128 || bits > 32 && ip.Is4() { - val := addressAndMask(n.sctx, ipVal, maskVal) - return n.sctx.WrapError("network_of: CIDR bit count out of range", val) - } - default: - return n.sctx.WrapError("network_of: bad arg for CIDR mask", args[1]) - } - } - // Mask for canonical form. - prefix := netip.PrefixFrom(ip, bits).Masked() - return super.NewNet(prefix) -} - -func addressAndMask(sctx *super.Context, address, mask super.Value) super.Value { - typ := sctx.MustLookupTypeRecord([]super.Field{ - super.NewField("address", address.Type()), - super.NewField("mask", mask.Type()), - }) - var b scode.Builder - b.Append(address.Bytes()) - b.Append(mask.Bytes()) - return super.NewValue(typ, b.Bytes()) -} - -type CIDRMatch struct { - sctx *super.Context -} - -var errMatch = errors.New("match") - -func (c *CIDRMatch) Call(args []super.Value) super.Value { - maskVal := args[0].Under() - if maskVal.IsNull() || args[1].IsNull() { - return super.Null - } - if maskVal.Type() != super.TypeNet { - return c.sctx.WrapError("cidr_match: not a net", maskVal) - } - prefix := super.DecodeNet(maskVal.Bytes()) - err := args[1].Walk(func(typ super.Type, body scode.Bytes) error { - if typ.ID() == super.IDIP { - if prefix.Contains(super.DecodeIP(body)) { - return errMatch - } - } - return nil - }) - return super.NewBool(err == errMatch) -} diff --git a/runtime/sam/expr/function/ksuid.go b/runtime/sam/expr/function/ksuid.go deleted file mode 100644 index 02744c77d8..0000000000 --- a/runtime/sam/expr/function/ksuid.go +++ /dev/null @@ -1,35 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" - "github.com/segmentio/ksuid" -) - -type KSUIDToString struct { - sctx *super.Context -} - -func (k *KSUIDToString) Call(args []super.Value) super.Value { - if len(args) == 0 { - return super.NewBytes(ksuid.New().Bytes()) - } - val := args[0] - switch val.Type().ID() { - case super.IDBytes: - id, err := ksuid.FromBytes(val.Bytes()) - if err != nil { - return k.sctx.WrapError("ksuid: invalid ksuid value", val) - } - return super.NewString(id.String()) - case super.IDString: - id, err := ksuid.Parse(string(val.Bytes())) - if err != nil { - return k.sctx.WrapError("ksuid: invalid ksuid value", val) - } - return super.NewBytes(id.Bytes()) - case super.IDNull: - return super.Null - default: - return k.sctx.WrapError("ksuid: argument must a bytes or string type", val) - } -} diff --git a/runtime/sam/expr/function/len.go b/runtime/sam/expr/function/len.go index 653b8c28ea..2429079f37 100644 --- a/runtime/sam/expr/function/len.go +++ b/runtime/sam/expr/function/len.go @@ -1,46 +1,9 @@ package function import ( - "unicode/utf8" - "github.com/brimdata/super" ) -type LenFn struct { - sctx *super.Context -} - -func (l *LenFn) Call(args []super.Value) super.Value { - val := args[0].Under() - var length int - switch typ := super.TypeUnder(val.Type()).(type) { - case *super.TypeOfNull: - case *super.TypeRecord: - length = len(typ.Fields) - case *super.TypeArray, *super.TypeSet, *super.TypeMap: - n, ok := val.ContainerLength() - if !ok { - panic(val.Type()) - } - length = n - case *super.TypeOfString: - length = utf8.RuneCount(val.Bytes()) - case *super.TypeOfBytes, *super.TypeOfIP, *super.TypeOfNet: - length = len(val.Bytes()) - case *super.TypeError: - return l.sctx.WrapError("len()", val) - case *super.TypeOfType: - t, err := l.sctx.LookupByValue(val.Bytes()) - if err != nil { - return l.sctx.NewError(err) - } - length = TypeLength(t) - default: - return l.sctx.WrapError("len: bad type", val) - } - return super.NewInt64(int64(length)) -} - func TypeLength(typ super.Type) int { switch typ := typ.(type) { case *super.TypeNamed: diff --git a/runtime/sam/expr/function/math.go b/runtime/sam/expr/function/math.go index 9ffecdd93c..2d523740ea 100644 --- a/runtime/sam/expr/function/math.go +++ b/runtime/sam/expr/function/math.go @@ -1,87 +1,12 @@ package function import ( - "math" - "github.com/brimdata/super" "github.com/brimdata/super/pkg/anymath" "github.com/brimdata/super/runtime/sam/expr" "github.com/brimdata/super/runtime/sam/expr/coerce" ) -type Abs struct { - sctx *super.Context -} - -func (a *Abs) Call(args []super.Value) super.Value { - val := args[0].Under() - if val.IsNull() { - return val - } - switch id := val.Type().ID(); { - case super.IsUnsigned(id): - return val - case super.IsSigned(id): - x := val.Int() - if x < 0 { - x = -x - } - return super.NewInt(val.Type(), x) - case super.IsFloat(id): - return super.NewFloat(val.Type(), math.Abs(val.Float())) - } - return a.sctx.WrapError("abs: not a number", val) -} - -type Ceil struct { - sctx *super.Context -} - -func (c *Ceil) Call(args []super.Value) super.Value { - val := args[0].Under() - switch id := val.Type().ID(); { - case super.IsUnsigned(id) || super.IsSigned(id): - return val - case super.IsFloat(id): - return super.NewFloat(val.Type(), math.Ceil(val.Float())) - } - return c.sctx.WrapError("ceil: not a number", val) -} - -type Floor struct { - sctx *super.Context -} - -func (f *Floor) Call(args []super.Value) super.Value { - val := args[0].Under() - switch id := val.Type().ID(); { - case super.IsUnsigned(id) || super.IsSigned(id): - return val - case super.IsFloat(id): - return super.NewFloat(val.Type(), math.Floor(val.Float())) - } - return f.sctx.WrapError("floor: not a number", val) -} - -type Log struct { - sctx *super.Context -} - -func (l *Log) Call(args []super.Value) super.Value { - val := args[0].Under() - if val.IsNull() || val.IsError() { - return val - } - x, ok := coerce.ToFloat(val, super.TypeFloat64) - if !ok { - return l.sctx.WrapError("log: not a number", args[0]) - } - if x <= 0 { - return l.sctx.WrapError("log: illegal argument", args[0]) - } - return super.NewFloat64(math.Log(x)) -} - type reducer struct { sctx *super.Context name string @@ -152,62 +77,3 @@ func (r *reducer) Call(args []super.Value) super.Value { func (r *reducer) errNotNumber(val super.Value) super.Value { return r.sctx.WrapError(r.name+": not a number", val) } - -type Round struct { - sctx *super.Context -} - -func (r *Round) Call(args []super.Value) super.Value { - val := args[0].Under() - switch id := val.Type().ID(); { - case id == super.IDNull: - return val - case super.IsUnsigned(id) || super.IsSigned(id): - return val - case super.IsFloat(id): - if val.IsNull() { - return val - } - return super.NewFloat(val.Type(), math.Round(val.Float())) - } - return r.sctx.WrapError("round: not a number", val) -} - -type Pow struct { - sctx *super.Context -} - -func (p *Pow) Call(args []super.Value) super.Value { - a, b := args[0].Under(), args[1].Under() - if a.IsNull() || b.IsNull() { - return super.Null - } - if !super.IsNumber(a.Type().ID()) { - return p.sctx.WrapError("pow: not a number", args[0]) - } - if !super.IsNumber(b.Type().ID()) { - return p.sctx.WrapError("pow: not a number", args[1]) - } - x, _ := coerce.ToFloat(a, super.TypeFloat64) - y, _ := coerce.ToFloat(b, super.TypeFloat64) - return super.NewFloat64(math.Pow(x, y)) -} - -type Sqrt struct { - sctx *super.Context -} - -func (s *Sqrt) Call(args []super.Value) super.Value { - val := args[0].Under() - if val.IsNull() { - return super.Null - } - if !super.IsNumber(val.Type().ID()) { - return s.sctx.WrapError("sqrt: number argument required", val) - } - x, ok := coerce.ToFloat(val, super.TypeFloat64) - if !ok { - return s.sctx.WrapError("sqrt: not a number", val) - } - return super.NewFloat64(math.Sqrt(x)) -} diff --git a/runtime/sam/expr/function/nestdotted.go b/runtime/sam/expr/function/nestdotted.go deleted file mode 100644 index a2527257c4..0000000000 --- a/runtime/sam/expr/function/nestdotted.go +++ /dev/null @@ -1,82 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/pkg/field" -) - -type NestDotted struct { - sctx *super.Context - builders map[int]*super.RecordBuilder - recordTypes map[int]*super.TypeRecord -} - -// NewNestDotted returns a function that turns successive dotted -// field names into nested records. For example, unflattening {"a.a": -// 1, "a.b": 1} results in {a:{a:1,b:1}}. Note that while -// unflattening is applied recursively from the top-level and applies -// to arbitrary-depth dotted names, it is not applied to dotted names -// that start at lower levels (for example {a:{"a.a":1}} is -// unchanged). -func NewNestDotted(sctx *super.Context) *NestDotted { - return &NestDotted{ - sctx: sctx, - builders: make(map[int]*super.RecordBuilder), - recordTypes: make(map[int]*super.TypeRecord), - } -} - -func (n *NestDotted) lookupBuilderAndType(in *super.TypeRecord) (*super.RecordBuilder, *super.TypeRecord, error) { - if b, ok := n.builders[in.ID()]; ok { - return b, n.recordTypes[in.ID()], nil - } - var foundDotted bool - var fields field.List - var types []super.Type - for _, f := range in.Fields { - dotted := field.Dotted(f.Name) - if len(dotted) > 1 { - foundDotted = true - } - fields = append(fields, dotted) - types = append(types, f.Type) - } - if !foundDotted { - return nil, nil, nil - } - b, err := super.NewRecordBuilder(n.sctx, fields) - if err != nil { - return nil, nil, err - } - typ := b.Type(types) - n.builders[in.ID()] = b - n.recordTypes[in.ID()] = typ - return b, typ, nil -} - -func (n *NestDotted) Call(args []super.Value) super.Value { - val := args[0] - if val.IsNull() { - return val - } - rtyp := super.TypeRecordOf(val.Type()) - if rtyp == nil { - return n.sctx.WrapError("nest_dotted: non-record value", val) - } - b, typ, err := n.lookupBuilderAndType(rtyp) - if err != nil { - return n.sctx.WrapError("nest_dotted: "+err.Error(), val) - } - if b == nil { - return val - } - b.Reset() - for it := val.Bytes().Iter(); !it.Done(); { - b.Append(it.Next()) - } - zbytes, err := b.Encode() - if err != nil { - panic(err) - } - return super.NewValue(typ, zbytes) -} diff --git a/runtime/sam/expr/function/nullif.go b/runtime/sam/expr/function/nullif.go deleted file mode 100644 index 94435e2246..0000000000 --- a/runtime/sam/expr/function/nullif.go +++ /dev/null @@ -1,29 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/order" - "github.com/brimdata/super/runtime/sam/expr" -) - -type NullIf struct { - compareFn expr.CompareFn -} - -func newNullIf() *NullIf { - return &NullIf{expr.NewValueCompareFn(order.Asc, order.NullsLast)} -} - -func (n *NullIf) Call(args []super.Value) super.Value { - val0, val1 := args[0].Under(), args[1].Under() - if val0.IsNull() || val0.IsError() { - return args[0] - } - if val1.IsError() { - return args[1] - } - if n.compareFn(val0, val1) == 0 { - return super.Null - } - return args[0] -} diff --git a/runtime/sam/expr/function/parse.go b/runtime/sam/expr/function/parse.go index 3298644ef0..614d043818 100644 --- a/runtime/sam/expr/function/parse.go +++ b/runtime/sam/expr/function/parse.go @@ -4,10 +4,8 @@ import ( "fmt" "net/url" "strconv" - "strings" "github.com/brimdata/super" - "github.com/brimdata/super/sio/supio" "github.com/brimdata/super/sup" ) @@ -83,33 +81,3 @@ func (p *ParseURI) Call(args []super.Value) super.Value { } return out } - -type ParseSUP struct { - sctx *super.Context - sr *strings.Reader - zr *supio.Reader -} - -func newParseSUP(sctx *super.Context) *ParseSUP { - var sr strings.Reader - return &ParseSUP{sctx, &sr, supio.NewReader(sctx, &sr)} -} - -func (p *ParseSUP) Call(args []super.Value) super.Value { - in := args[0].Under() - if in.IsNull() { - return super.Null - } - if !in.IsString() { - return p.sctx.WrapError("parse_sup: string arg required", args[0]) - } - p.sr.Reset(super.DecodeString(in.Bytes())) - val, err := p.zr.Read() - if err != nil { - return p.sctx.WrapError("parse_sup: "+err.Error(), args[0]) - } - if val == nil { - return super.Null - } - return *val -} diff --git a/runtime/sam/expr/function/regexp.go b/runtime/sam/expr/function/regexp.go deleted file mode 100644 index c6f8541178..0000000000 --- a/runtime/sam/expr/function/regexp.go +++ /dev/null @@ -1,84 +0,0 @@ -package function - -import ( - "regexp" - "regexp/syntax" - - "github.com/brimdata/super" - "github.com/brimdata/super/scode" -) - -type Regexp struct { - builder scode.Builder - re *regexp.Regexp - restr string - typ super.Type - err error - sctx *super.Context -} - -func (r *Regexp) Call(args []super.Value) super.Value { - reVal, sVal := args[0].Under(), args[1].Under() - if reVal.IsNull() || sVal.IsNull() { - return super.Null - } - if !reVal.IsString() { - return r.sctx.WrapError("regexp: string required for first arg", args[0]) - } - s := super.DecodeString(reVal.Bytes()) - if r.restr != s { - r.restr = s - r.re, r.err = regexp.Compile(r.restr) - } - if r.err != nil { - msg := "regexp: invalid regular expression" - if syntaxErr, ok := r.err.(*syntax.Error); ok { - msg += ": " + syntaxErr.Code.String() - } - return r.sctx.WrapError(msg, args[0]) - } - if !sVal.IsString() { - return r.sctx.WrapError("regexp: string required for second arg", args[1]) - } - r.builder.Reset() - for _, b := range r.re.FindSubmatch(sVal.Bytes()) { - r.builder.Append(b) - } - if r.typ == nil { - r.typ = r.sctx.LookupTypeArray(super.TypeString) - } - return super.NewValue(r.typ, r.builder.Bytes()) -} - -type RegexpReplace struct { - sctx *super.Context - re *regexp.Regexp - restr string - err error -} - -func (r *RegexpReplace) Call(args []super.Value) super.Value { - sVal := args[0].Under() - reVal := args[1].Under() - newVal := args[2].Under() - if sVal.IsNull() || reVal.IsNull() || newVal.IsNull() { - return super.Null - } - for i, val := range []super.Value{sVal, reVal, newVal} { - if !val.IsString() { - return r.sctx.WrapError("regexp_replace: string arg required", args[i]) - } - } - if re := super.DecodeString(reVal.Bytes()); r.restr != re { - r.restr = re - r.re, r.err = regexp.Compile(re) - } - if r.err != nil { - msg := "regexp_replace: invalid regular expression" - if syntaxErr, ok := r.err.(*syntax.Error); ok { - msg += ": " + syntaxErr.Code.String() - } - return r.sctx.WrapError(msg, args[1]) - } - return super.NewString(string(r.re.ReplaceAll(sVal.Bytes(), newVal.Bytes()))) -} diff --git a/runtime/sam/expr/function/string.go b/runtime/sam/expr/function/string.go deleted file mode 100644 index 901099022f..0000000000 --- a/runtime/sam/expr/function/string.go +++ /dev/null @@ -1,215 +0,0 @@ -package function - -import ( - "strings" - - "github.com/agnivade/levenshtein" - "github.com/brimdata/super" - "github.com/brimdata/super/scode" -) - -type Concat struct { - sctx *super.Context -} - -func (c *Concat) Call(args []super.Value) super.Value { - args = underAll(args) - var b strings.Builder - for _, arg := range args { - if arg.IsError() { - return arg - } - if arg.IsNull() { - continue - } - if !arg.IsString() { - return c.sctx.WrapError("concat: string arg required", arg) - } - b.WriteString(super.DecodeString(arg.Bytes())) - } - return super.NewString(b.String()) -} - -type Position struct { - sctx *super.Context -} - -func (p *Position) Call(args []super.Value) super.Value { - val, subVal := args[0].Under(), args[1].Under() - if val.IsNull() || subVal.IsNull() { - return super.Null - } - if !val.IsString() { - return p.sctx.WrapError("position: string arguments required", val) - } - if !subVal.IsString() { - return p.sctx.WrapError("position: string arguments required", subVal) - } - i := strings.Index(val.AsString(), subVal.AsString()) - return super.NewInt64(int64(i + 1)) -} - -type Replace struct { - sctx *super.Context -} - -func (r *Replace) Call(args []super.Value) super.Value { - args = underAll(args) - sVal := args[0] - oldVal := args[1] - newVal := args[2] - if sVal.IsNull() || oldVal.IsNull() || newVal.IsNull() { - return super.Null - } - for i := range args { - if !args[i].IsString() { - return r.sctx.WrapError("replace: string arg required", args[i]) - } - } - s := super.DecodeString(sVal.Bytes()) - old := super.DecodeString(oldVal.Bytes()) - new := super.DecodeString(newVal.Bytes()) - return super.NewString(strings.ReplaceAll(s, old, new)) -} - -type ToLower struct { - sctx *super.Context -} - -func (t *ToLower) Call(args []super.Value) super.Value { - val := args[0].Under() - if val.IsNull() { - return super.Null - } - if !val.IsString() { - return t.sctx.WrapError("lower: string arg required", val) - } - s := super.DecodeString(val.Bytes()) - return super.NewString(strings.ToLower(s)) -} - -type ToUpper struct { - sctx *super.Context -} - -func (t *ToUpper) Call(args []super.Value) super.Value { - val := args[0].Under() - if val.IsNull() { - return super.Null - } - if !val.IsString() { - return t.sctx.WrapError("upper: string arg required", val) - } - s := super.DecodeString(val.Bytes()) - return super.NewString(strings.ToUpper(s)) -} - -type Trim struct { - sctx *super.Context -} - -func (t *Trim) Call(args []super.Value) super.Value { - val := args[0].Under() - if val.IsNull() { - return super.Null - } - if !val.IsString() { - return t.sctx.WrapError("trim: string arg required", val) - } - s := super.DecodeString(val.Bytes()) - return super.NewString(strings.TrimSpace(s)) -} - -type Split struct { - sctx *super.Context - typ super.Type -} - -func newSplit(sctx *super.Context) *Split { - return &Split{ - sctx: sctx, - typ: sctx.LookupTypeArray(super.TypeString), - } -} - -func (s *Split) Call(args []super.Value) super.Value { - args = underAll(args) - sVal, sepVal := args[0], args[1] - if sVal.IsNull() || sepVal.IsNull() { - return super.Null - } - if sVal.Type().Kind() == super.ErrorKind { - return sVal - } - if sepVal.Type().Kind() == super.ErrorKind { - return sVal - } - for i := range args { - if !args[i].IsString() { - return s.sctx.WrapError("split: string arg required", args[i]) - } - } - str := super.DecodeString(sVal.Bytes()) - sep := super.DecodeString(sepVal.Bytes()) - splits := strings.Split(str, sep) - var b scode.Bytes - for _, substr := range splits { - b = scode.Append(b, super.EncodeString(substr)) - } - return super.NewValue(s.typ, b) -} - -type Join struct { - sctx *super.Context -} - -func (j *Join) Call(args []super.Value) super.Value { - args = underAll(args) - for _, val := range args { - if val.IsNull() { - return super.Null - } - } - splitsVal := args[0] - typ, ok := super.TypeUnder(splitsVal.Type()).(*super.TypeArray) - if !ok || typ.Type.ID() != super.IDString { - return j.sctx.WrapError("join: array of string arg required", splitsVal) - } - var separator string - if len(args) == 2 { - sepVal := args[1] - if !sepVal.IsString() { - return j.sctx.WrapError("join: separator must be string", sepVal) - } - separator = super.DecodeString(sepVal.Bytes()) - } - var b strings.Builder - var sep string - it := splitsVal.Bytes().Iter() - for !it.Done() { - b.WriteString(sep) - b.WriteString(super.DecodeString(it.Next())) - sep = separator - } - return super.NewString(b.String()) -} - -type Levenshtein struct { - sctx *super.Context -} - -func (l *Levenshtein) Call(args []super.Value) super.Value { - args = underAll(args) - a, b := args[0], args[1] - if a.IsNull() || b.IsNull() { - return super.Null - } - if !a.IsString() { - return l.sctx.WrapError("levenshtein: string args required", a) - } - if !b.IsString() { - return l.sctx.WrapError("levenshtein: string args required", b) - } - as, bs := super.DecodeString(a.Bytes()), super.DecodeString(b.Bytes()) - return super.NewInt64(int64(levenshtein.ComputeDistance(as, bs))) -} diff --git a/runtime/sam/expr/function/time.go b/runtime/sam/expr/function/time.go deleted file mode 100644 index a3113a7e8c..0000000000 --- a/runtime/sam/expr/function/time.go +++ /dev/null @@ -1,74 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/pkg/nano" - "github.com/lestrrat-go/strftime" -) - -type Now struct{} - -func (n *Now) Call(_ []super.Value) super.Value { - return super.NewTime(nano.Now()) -} - -type Bucket struct { - name string - sctx *super.Context -} - -func (b *Bucket) Call(args []super.Value) super.Value { - args = underAll(args) - tsArg := args[0] - binArg := args[1] - if tsArg.IsNull() || binArg.IsNull() { - return super.Null - } - tsArgID := tsArg.Type().ID() - if tsArgID != super.IDDuration && tsArgID != super.IDTime { - return b.sctx.WrapError(b.name+": first argument is not a time or duration", tsArg) - } - if binArg.Type().ID() != super.IDDuration { - return b.sctx.WrapError(b.name+": second argument is not a duration", binArg) - } - bin := nano.Duration(binArg.Int()) - if tsArgID == super.IDDuration { - dur := nano.Duration(tsArg.Int()) - if bin != 0 { - dur = dur.Trunc(bin) - } - return super.NewDuration(dur) - } - ts := nano.Ts(tsArg.Int()) - if bin != 0 { - ts = ts.Trunc(bin) - } - return super.NewTime(ts) -} - -type Strftime struct { - sctx *super.Context - formatter *strftime.Strftime -} - -func (s *Strftime) Call(args []super.Value) super.Value { - formatArg, timeArg := args[0].Under(), args[1].Under() - if formatArg.IsNull() || timeArg.IsNull() { - return super.Null - } - if !formatArg.IsString() { - return s.sctx.WrapError("strftime: string value required for format arg", formatArg) - } - if super.TypeUnder(timeArg.Type()) != super.TypeTime { - return s.sctx.WrapError("strftime: time value required for time arg", args[1]) - } - format := formatArg.AsString() - if s.formatter == nil || s.formatter.Pattern() != format { - var err error - if s.formatter, err = strftime.New(format); err != nil { - return s.sctx.WrapError("strftime: "+err.Error(), formatArg) - } - } - out := s.formatter.FormatString(timeArg.AsTime().Time()) - return super.NewString(out) -} diff --git a/runtime/sam/expr/function/types.go b/runtime/sam/expr/function/types.go index 4ed00a2dcf..5033ffcb1c 100644 --- a/runtime/sam/expr/function/types.go +++ b/runtime/sam/expr/function/types.go @@ -7,85 +7,6 @@ import ( "github.com/brimdata/super/scode" ) -type TypeOf struct { - sctx *super.Context -} - -func (t *TypeOf) Call(args []super.Value) super.Value { - return t.sctx.LookupTypeValue(args[0].Type()) -} - -type NameOf struct { - sctx *super.Context -} - -func (n *NameOf) Call(args []super.Value) super.Value { - typ := args[0].Type() - if named, ok := typ.(*super.TypeNamed); ok { - return super.NewString(named.Name) - } - if typ.ID() == super.IDType { - if args[0].IsNull() { - return super.Null - } - var err error - if typ, err = n.sctx.LookupByValue(args[0].Bytes()); err != nil { - panic(err) - } - if named, ok := typ.(*super.TypeNamed); ok { - return super.NewString(named.Name) - } - } - return n.sctx.Missing() -} - -type typeName struct { - sctx *super.Context -} - -func (t *typeName) Call(args []super.Value) super.Value { - if super.TypeUnder(args[0].Type()) != super.TypeString { - return t.sctx.WrapError("typename: argument must be a string", args[0]) - } - name := string(args[0].Bytes()) - typ := t.sctx.LookupByName(name) - if typ == nil { - return t.sctx.Missing() - } - return t.sctx.LookupTypeValue(typ) -} - -type Error struct { - sctx *super.Context -} - -func (e *Error) Call(args []super.Value) super.Value { - return super.NewValue(e.sctx.LookupTypeError(args[0].Type()), args[0].Bytes()) -} - -type IsErr struct{} - -func (*IsErr) Call(args []super.Value) super.Value { - val := args[0].Under() - return super.NewBool(val.IsError() && !val.IsNull()) -} - -type Is struct { - sctx *super.Context -} - -func (i *Is) Call(args []super.Value) super.Value { - zvSubject := args[0] - zvTypeVal := args[1] - var typ super.Type - var err error - if zvTypeVal.Type().ID() != super.IDType { - return i.sctx.WrapError("is: type value argument expected", zvTypeVal) - } - typ, err = i.sctx.LookupByValue(zvTypeVal.Bytes()) - return super.NewBool(err == nil && typ == zvSubject.Type()) -} - type HasError struct{} func (h HasError) Call(args []super.Value) super.Value { @@ -122,22 +43,3 @@ func (h HasError) hasError(t super.Type, b scode.Bytes) bool { return false } } - -type Kind struct { - sctx *super.Context -} - -func (k *Kind) Call(args []super.Value) super.Value { - val := args[0] - var typ super.Type - if _, ok := super.TypeUnder(val.Type()).(*super.TypeOfType); ok { - var err error - typ, err = k.sctx.LookupByValue(val.Bytes()) - if err != nil { - panic(err) - } - } else { - typ = val.Type() - } - return super.NewString(typ.Kind().String()) -} diff --git a/runtime/sam/expr/function/unblend.go b/runtime/sam/expr/function/unblend.go deleted file mode 100644 index f4368f0764..0000000000 --- a/runtime/sam/expr/function/unblend.go +++ /dev/null @@ -1,138 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/scode" -) - -type unblend struct { - sctx *super.Context -} - -func NewUnblend(sctx *super.Context) *unblend { - return &unblend{sctx} -} - -func (u *unblend) Call(args []super.Value) super.Value { - return u.eval(args[0]) -} - -func (u *unblend) eval(in super.Value) super.Value { - switch typ := in.Type().(type) { - case *super.TypeRecord: - var fields []super.Field - var elems []super.Value - it := in.Bytes().Iter() - for _, f := range typ.Fields { - bytes := it.Next() - if super.IsNone(f.Type, bytes) { - continue - } - val := u.eval(super.NewValue(f.Type, bytes)) - elems = append(elems, val) - fields = append(fields, super.NewField(f.Name, val.Type())) - } - var b scode.Builder - for _, e := range elems { - b.Append(e.Bytes()) - } - return super.NewValue(u.sctx.MustLookupTypeRecord(fields), b.Bytes()) - case *super.TypeArray: - elems := u.arrayOrSet(typ.Type, in.Bytes()) - if len(elems) == 0 { - typ := u.sctx.LookupTypeArray(super.TypeNull) - return super.NewValue(typ, nil) - } - elemType, bytes := u.unify(elems) - return super.NewValue(u.sctx.LookupTypeArray(elemType), bytes) - case *super.TypeSet: - elems := u.arrayOrSet(typ.Type, in.Bytes()) - if len(elems) == 0 { - typ := u.sctx.LookupTypeSet(super.TypeNull) - return super.NewValue(typ, nil) - } - elemType, bytes := u.unify(elems) - return super.NewValue(u.sctx.LookupTypeSet(elemType), bytes) - case *super.TypeMap: - var keys, vals []super.Value - for it := in.Bytes().Iter(); !it.Done(); { - keys = append(keys, super.NewValue(typ.KeyType, it.Next()).DeunionIntoNameds()) - vals = append(vals, super.NewValue(typ.ValType, it.Next()).DeunionIntoNameds()) - } - keyType := u.unifyType(keys) - valType := u.unifyType(vals) - var b scode.Builder - for k, key := range keys { - if u, ok := keyType.(*super.TypeUnion); ok { - super.BuildUnion(&b, u.TagOf(key.Type()), key.Bytes()) - } else { - b.Append(key.Bytes()) - } - val := vals[k] - if u, ok := valType.(*super.TypeUnion); ok { - super.BuildUnion(&b, u.TagOf(val.Type()), val.Bytes()) - } else { - b.Append(val.Bytes()) - } - } - return super.NewValue(u.sctx.LookupTypeMap(keyType, valType), b.Bytes()) - case *super.TypeUnion: - return u.eval(in.DeunionIntoNameds()) - default: - // primitives, named types, enums - return in - } -} - -func (u *unblend) arrayOrSet(typ super.Type, bytes scode.Bytes) []super.Value { - var elems []super.Value - for it := bytes.Iter(); !it.Done(); { - elems = append(elems, u.eval(super.NewValue(typ, it.Next()))) - } - return elems -} - -func (u *unblend) unify(elems []super.Value) (super.Type, scode.Bytes) { - seen := make(map[super.Type]struct{}) - var types []super.Type - for _, e := range elems { - typ := e.Type() - if _, ok := seen[typ]; !ok { - seen[typ] = struct{}{} - types = append(types, typ) - } - } - if len(types) == 1 { - var b scode.Builder - for _, e := range elems { - b.Append(e.Bytes()) - } - return types[0], b.Bytes() - } - union := u.sctx.MustLookupTypeUnion(types) - var b scode.Builder - for _, e := range elems { - super.BuildUnion(&b, union.TagOf(e.Type()), e.Bytes()) - } - return union, b.Bytes() -} - -func (u *unblend) unifyType(vals []super.Value) super.Type { - seen := make(map[super.Type]struct{}) - var types []super.Type - for _, e := range vals { - typ := e.Type() - if _, ok := seen[typ]; !ok { - seen[typ] = struct{}{} - types = append(types, typ) - } - } - switch len(types) { - case 0: - return super.TypeNull // XXX should be TypeNone - case 1: - return types[0] - default: - return u.sctx.MustLookupTypeUnion(types) - } -} diff --git a/runtime/sam/expr/literal.go b/runtime/sam/expr/literal.go deleted file mode 100644 index e3c2214094..0000000000 --- a/runtime/sam/expr/literal.go +++ /dev/null @@ -1,17 +0,0 @@ -package expr - -import "github.com/brimdata/super" - -type Literal struct { - val super.Value -} - -var _ Evaluator = (*Literal)(nil) - -func NewLiteral(val super.Value) *Literal { - return &Literal{val: val} -} - -func (l Literal) Eval(super.Value) super.Value { - return l.val -} diff --git a/runtime/sam/expr/map.go b/runtime/sam/expr/map.go deleted file mode 100644 index 3f12b48ed7..0000000000 --- a/runtime/sam/expr/map.go +++ /dev/null @@ -1,72 +0,0 @@ -package expr - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/scode" -) - -type mapCall struct { - builder scode.Builder - eval Evaluator - lambda Evaluator - sctx *super.Context - - // vals is used to reduce allocations - vals []super.Value - // types is used to reduce allocations - types []super.Type -} - -func NewMapCall(sctx *super.Context, e, lambda Evaluator) Evaluator { - return &mapCall{eval: e, lambda: lambda, sctx: sctx} -} - -func (a *mapCall) Eval(in super.Value) super.Value { - val := a.eval.Eval(in).Under() - if val.IsNull() || val.IsError() { - return val - } - elems, ok := val.Elements() - if !ok { - return a.sctx.WrapError("map: expected array or set value", in) - } - if len(elems) == 0 { - return val - } - a.vals = a.vals[:0] - a.types = a.types[:0] - for _, elem := range elems { - val := a.lambda.Eval(elem) - a.vals = append(a.vals, val) - a.types = append(a.types, val.Type()) - } - inner := a.innerType(a.types) - bytes := a.buildVal(inner) - if _, ok := super.TypeUnder(val.Type()).(*super.TypeSet); ok { - return super.NewValue(a.sctx.LookupTypeSet(inner), super.NormalizeSet(bytes)) - } - return super.NewValue(a.sctx.LookupTypeArray(inner), bytes) -} - -func (a *mapCall) buildVal(inner super.Type) []byte { - a.builder.Reset() - if union, ok := inner.(*super.TypeUnion); ok { - for _, val := range a.vals { - val = val.Deunion() - super.BuildUnion(&a.builder, union.TagOf(val.Type()), val.Bytes()) - } - } else { - for _, val := range a.vals { - a.builder.Append(val.Bytes()) - } - } - return a.builder.Bytes() -} - -func (a *mapCall) innerType(types []super.Type) super.Type { - types = super.Flatten(super.UniqueTypes(types)) - if len(types) == 1 { - return types[0] - } - return a.sctx.MustLookupTypeUnion(types) -} diff --git a/runtime/sam/expr/putter.go b/runtime/sam/expr/putter.go deleted file mode 100644 index 916c648000..0000000000 --- a/runtime/sam/expr/putter.go +++ /dev/null @@ -1,26 +0,0 @@ -package expr - -import ( - "github.com/brimdata/super" -) - -type putter struct { - sctx *super.Context - e Evaluator -} - -// NewPutter wraps e to implement the behavior of the put operator, which emits -// an error when an input value is not a record. -func NewPutter(sctx *super.Context, e Evaluator) Evaluator { - return &putter{sctx, e} -} - -func (p *putter) Eval(val super.Value) super.Value { - if k := val.Type().Kind(); k != super.RecordKind { - if k == super.ErrorKind { - return val - } - return p.sctx.WrapError("put: not a record", val) - } - return p.e.Eval(val) -} diff --git a/runtime/sam/expr/slice.go b/runtime/sam/expr/slice.go index 4cf22f71c0..6f1d12e97b 100644 --- a/runtime/sam/expr/slice.go +++ b/runtime/sam/expr/slice.go @@ -1,107 +1,9 @@ package expr import ( - "errors" "unicode/utf8" - - "github.com/brimdata/super" - "github.com/brimdata/super/runtime/sam/expr/coerce" - "github.com/brimdata/super/scode" ) -type Slice struct { - sctx *super.Context - elem Evaluator - from Evaluator - to Evaluator - base1 bool -} - -func NewSlice(sctx *super.Context, elem, from, to Evaluator, base1 bool) *Slice { - return &Slice{ - sctx: sctx, - elem: elem, - from: from, - to: to, - base1: base1, - } -} - -func (s *Slice) Eval(this super.Value) super.Value { - elem := s.elem.Eval(this).Under() - if elem.IsNull() || elem.IsError() { - return elem - } - var length int - switch elem.Type().(type) { - case *super.TypeOfBytes: - length = len(elem.Bytes()) - case *super.TypeOfString: - length = utf8.RuneCount(elem.Bytes()) - case *super.TypeArray, *super.TypeSet: - n, ok := elem.ContainerLength() - if !ok { - panic(elem.Type()) - } - length = n - default: - return s.sctx.WrapError("sliced value is not array, set, bytes, or string", elem) - } - from, to := 0, length - if s.from != nil { - val := s.sliceIndex(s.from.Eval(this), length, s.base1) - if val.IsNull() || val.IsError() { - return val - } - from = int(val.Int()) - } - if s.to != nil { - val := s.sliceIndex(s.to.Eval(this), length, s.base1) - if val.IsNull() || val.IsError() { - return val - } - to = int(val.Int()) - } - from, to = FixSliceBounds(from, to, length) - bytes := elem.Bytes() - switch super.TypeUnder(elem.Type()).(type) { - case *super.TypeOfBytes: - bytes = bytes[from:to] - case *super.TypeOfString: - bytes = bytes[UTF8PrefixLen(bytes, from):] - bytes = bytes[:UTF8PrefixLen(bytes, to-from)] - case *super.TypeArray, *super.TypeSet: - it := bytes.Iter() - for k := 0; k < to && !it.Done(); k++ { - if k == from { - bytes = scode.Bytes(it) - } - it.Next() - } - bytes = bytes[:len(bytes)-len(it)] - default: - panic(elem.Type()) - } - return super.NewValue(elem.Type(), bytes) -} - -func (s *Slice) sliceIndex(val super.Value, length int, base1 bool) super.Value { - if val.IsNull() || val.IsError() { - return val - } - index, ok := coerce.ToInt(val, super.TypeInt64) - if !ok { - return s.sctx.NewError(errors.New("slice index is not a number")) - } - if base1 && index > 0 { - index-- - } - if index < 0 { - index += int64(length) - } - return super.NewInt64(index) -} - func FixSliceBounds(start, end, size int) (int, int) { if start > end || end < 0 { return 0, 0 diff --git a/runtime/sam/expr/udf.go b/runtime/sam/expr/udf.go index 31fa1bbc06..dd8f96dfc3 100644 --- a/runtime/sam/expr/udf.go +++ b/runtime/sam/expr/udf.go @@ -1,43 +1,3 @@ package expr -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/scode" -) - const MaxStackDepth = 10_000 - -type UDF struct { - Body Evaluator - sctx *super.Context - name string - fields []super.Field - stackDepth int - builder scode.Builder -} - -func NewUDF(sctx *super.Context, name string, params []string) *UDF { - var fields []super.Field - for _, p := range params { - fields = append(fields, super.Field{Name: p}) - } - return &UDF{sctx: sctx, name: name, fields: fields} -} - -func (u *UDF) Call(args []super.Value) super.Value { - u.stackDepth++ - if u.stackDepth > MaxStackDepth { - return u.sctx.NewErrorf("stack overflow in function %q", u.name) - } - defer func() { u.stackDepth-- }() - if len(args) == 0 { - return u.Body.Eval(super.Null) - } - u.builder.Reset() - for i, a := range args { - u.fields[i].Type = a.Type() - u.builder.Append(a.Bytes()) - } - typ := u.sctx.MustLookupTypeRecord(u.fields) - return u.Body.Eval(super.NewValue(typ, u.builder.Bytes())) -} diff --git a/runtime/sam/expr/values.go b/runtime/sam/expr/values.go deleted file mode 100644 index a5878222db..0000000000 --- a/runtime/sam/expr/values.go +++ /dev/null @@ -1,423 +0,0 @@ -package expr - -import ( - "slices" - - "github.com/brimdata/super" - "github.com/brimdata/super/scode" -) - -type recordExpr struct { - sctx *super.Context - typ *super.TypeRecord - builder *scode.Builder - fields []super.Field - exprs []Evaluator - options []*super.TypeUnion - somes []bool -} - -func NewRecordExpr(sctx *super.Context, elems []RecordElem) Evaluator { - if evaluator := newRecordExpr(sctx, elems); evaluator != nil { - return evaluator - } - return newRecordSpreadExpr(sctx, elems) -} - -func newRecordExpr(sctx *super.Context, elems []RecordElem) *recordExpr { - fields := make([]super.Field, 0, len(elems)) - exprs := make([]Evaluator, 0, len(elems)) - options := make([]*super.TypeUnion, 0, len(elems)) - somes := make([]bool, 0, len(elems)) - for _, elem := range elems { - var name string - var typ super.Type - switch elem := elem.(type) { - case *NoneElem: - name = elem.Name - exprs = append(exprs, nil) - options = append(options, sctx.Option(elem.Type)) - somes = append(somes, false) - case *FieldElem: - name = elem.Name - exprs = append(exprs, elem.Expr) - options = append(options, nil) - somes = append(somes, elem.Opt) - case *SpreadElem: - return nil - } - fields = append(fields, super.NewField(name, typ)) - } - return &recordExpr{ - sctx: sctx, - builder: scode.NewBuilder(), - fields: fields, - exprs: exprs, - options: options, - somes: somes, - } -} - -func (r *recordExpr) Eval(this super.Value) super.Value { - var changed bool - b := r.builder - b.Reset() - b.BeginContainer() - for k, e := range r.exprs { - var val super.Value - if e == nil { - val = super.None(r.options[k]) - } else { - val = e.Eval(this) - if r.somes[k] { - optionType := r.sctx.Option(val.Type()) - val = super.Some(optionType, val.Type(), val.Bytes()) - } - } - if r.fields[k].Type != val.Type() { - r.fields[k].Type = val.Type() - changed = true - } - b.Append(val.Bytes()) - } - if changed || r.typ == nil { - r.typ = r.sctx.MustLookupTypeRecord(r.fields) - } - b.EndContainer() - return super.NewValue(r.typ, b.Bytes().Body()) -} - -type RecordElem interface { - recordElemSum() -} - -type SpreadElem struct { - Expr Evaluator -} - -type FieldElem struct { - Name string - Expr Evaluator - Opt bool -} - -type NoneElem struct { - Name string - Type super.Type -} - -func (*SpreadElem) recordElemSum() {} -func (*FieldElem) recordElemSum() {} -func (*NoneElem) recordElemSum() {} - -type recordSpreadExpr struct { - sctx *super.Context - elems []RecordElem - builder scode.Builder - fields []super.Field - vals []fieldValue - cache *super.TypeRecord -} - -func newRecordSpreadExpr(sctx *super.Context, elems []RecordElem) *recordSpreadExpr { - return &recordSpreadExpr{ - sctx: sctx, - elems: elems, - } -} - -type fieldValue struct { - index int - value super.Value -} - -func get(rec map[string]fieldValue, name string) fieldValue { - fv, ok := rec[name] - if !ok { - fv = fieldValue{index: len(rec)} - rec[name] = fv - } - return fv -} - -func (r *recordSpreadExpr) Eval(this super.Value) super.Value { - rec := make(map[string]fieldValue) - for _, elem := range r.elems { - switch elem := elem.(type) { - case *SpreadElem: - val := elem.Expr.Eval(this) - if val.IsMissing() { - continue - } - typ := super.TypeRecordOf(val.Type()) - if typ == nil { - // Treat non-record spread values like missing. - continue - } - it := val.Bytes().Iter() - for _, f := range typ.Fields { - fv := get(rec, f.Name) - fv.value = super.NewValue(f.Type, it.Next()) - rec[f.Name] = fv - } - case *FieldElem: - val := elem.Expr.Eval(this) - fv := get(rec, elem.Name) - if elem.Opt { - optionType := r.sctx.Option(val.Type()) - val = super.Some(optionType, val.Type(), val.Bytes()) - } - fv.value = val - rec[elem.Name] = fv - case *NoneElem: - fv := get(rec, elem.Name) - fv.value = super.None(r.sctx.Option(elem.Type)) - rec[elem.Name] = fv - default: - panic(elem) - } - } - if len(rec) == 0 { - return super.NewValue(r.sctx.MustLookupTypeRecord([]super.Field{}), []byte{}) - } - r.update(rec) - b := r.builder - b.Reset() - b.BeginContainer() - for _, fv := range r.vals { - b.Append(fv.value.Bytes()) - } - b.EndContainer() - return super.NewValue(r.cache, b.Bytes().Body()) -} - -// update maps the object into the receiver's vals slice while also -// seeing if we can reuse the cached record type. If not we look up -// a new type, cache it, and save the field for the cache check. -func (r *recordSpreadExpr) update(rec map[string]fieldValue) { - if len(r.fields) != len(rec) { - r.invalidate(rec) - return - } - for name, fv := range rec { - if r.fields[fv.index] != super.NewField(name, fv.value.Type()) { - r.invalidate(rec) - return - } - r.vals[fv.index] = fv - } -} - -func (r *recordSpreadExpr) invalidate(rec map[string]fieldValue) { - n := len(rec) - r.fields = slices.Grow(r.fields[:0], n)[:n] - r.vals = slices.Grow(r.vals[:0], n)[:n] - for name, fv := range rec { - r.fields[fv.index] = super.NewField(name, fv.value.Type()) - r.vals[fv.index] = fv - } - r.cache = r.sctx.MustLookupTypeRecord(r.fields) -} - -type VectorElem struct { - Value Evaluator - Spread Evaluator -} - -type ArrayExpr struct { - elems []VectorElem - sctx *super.Context - - builder scode.Builder - collection collectionBuilder -} - -func NewArrayExpr(sctx *super.Context, elems []VectorElem) *ArrayExpr { - return &ArrayExpr{ - elems: elems, - sctx: sctx, - } -} - -func (a *ArrayExpr) Eval(this super.Value) super.Value { - a.builder.Reset() - a.collection.reset() - for _, e := range a.elems { - if e.Value != nil { - a.collection.append(e.Value.Eval(this)) - continue - } - val := e.Spread.Eval(this).Under() - inner := super.InnerType(val.Type()) - if inner == nil { - // Treat non-list spread values values like missing. - continue - } - a.collection.appendSpread(inner, val.Bytes()) - } - if len(a.collection.types) == 0 { - return super.NewValue(a.sctx.LookupTypeArray(super.TypeNone), []byte{}) - } - it := a.collection.iter(a.sctx) - for !it.done() { - it.appendNext(&a.builder) - } - return super.NewValue(a.sctx.LookupTypeArray(it.typ), a.builder.Bytes()) -} - -type SetExpr struct { - builder scode.Builder - collection collectionBuilder - elems []VectorElem - sctx *super.Context -} - -func NewSetExpr(sctx *super.Context, elems []VectorElem) *SetExpr { - return &SetExpr{ - elems: elems, - sctx: sctx, - } -} - -func (a *SetExpr) Eval(this super.Value) super.Value { - a.builder.Reset() - a.collection.reset() - for _, e := range a.elems { - if e.Value != nil { - a.collection.append(e.Value.Eval(this)) - continue - } - val := e.Spread.Eval(this).Under() - inner := super.InnerType(val.Type()) - if inner == nil { - // Treat non-list spread values values like missing. - continue - } - a.collection.appendSpread(inner, val.Bytes()) - } - if len(a.collection.types) == 0 { - return super.NewValue(a.sctx.LookupTypeSet(super.TypeNone), []byte{}) - } - it := a.collection.iter(a.sctx) - for !it.done() { - it.appendNext(&a.builder) - } - return super.NewValue(a.sctx.LookupTypeSet(it.typ), super.NormalizeSet(a.builder.Bytes())) -} - -type Entry struct { - Key Evaluator - Val Evaluator -} - -type MapExpr struct { - builder scode.Builder - entries []Entry - keys collectionBuilder - vals collectionBuilder - sctx *super.Context -} - -func NewMapExpr(sctx *super.Context, entries []Entry) *MapExpr { - return &MapExpr{ - entries: entries, - sctx: sctx, - } -} - -func (m *MapExpr) Eval(this super.Value) super.Value { - m.keys.reset() - m.vals.reset() - for _, e := range m.entries { - m.keys.append(e.Key.Eval(this)) - m.vals.append(e.Val.Eval(this)) - } - if len(m.keys.types) == 0 { - typ := m.sctx.LookupTypeMap(super.TypeNone, super.TypeNone) - return super.NewValue(typ, []byte{}) - } - m.builder.Reset() - kIter, vIter := m.keys.iter(m.sctx), m.vals.iter(m.sctx) - for !kIter.done() { - kIter.appendNext(&m.builder) - vIter.appendNext(&m.builder) - } - bytes := m.builder.Bytes() - typ := m.sctx.LookupTypeMap(kIter.typ, vIter.typ) - return super.NewValue(typ, super.NormalizeMap(bytes)) -} - -type collectionBuilder struct { - types []super.Type - uniqueTypes []super.Type - bytes []scode.Bytes -} - -func (c *collectionBuilder) reset() { - c.types = c.types[:0] - c.uniqueTypes = c.uniqueTypes[:0] - c.bytes = c.bytes[:0] -} - -func (c *collectionBuilder) append(val super.Value) { - c.types = append(c.types, val.Type()) - c.bytes = append(c.bytes, val.Bytes()) -} - -func (c *collectionBuilder) appendSpread(inner super.Type, b scode.Bytes) { - union, _ := super.TypeUnder(inner).(*super.TypeUnion) - for it := b.Iter(); !it.Done(); { - typ := inner - bytes := it.Next() - if union != nil { - typ, bytes = union.Untag(bytes) - } - c.types = append(c.types, typ) - c.bytes = append(c.bytes, bytes) - } -} - -func (c *collectionBuilder) iter(sctx *super.Context) collectionIter { - // uniqueTypes must be copied since super.UniqueTypes operates on the type - // array in place and thus we'll lose order. - c.uniqueTypes = append(c.uniqueTypes[:0], c.types...) - return collectionIter{ - typ: unionOf(sctx, c.uniqueTypes), - bytes: c.bytes, - types: c.types, - uniq: len(c.uniqueTypes), - } -} - -type collectionIter struct { - typ super.Type - bytes []scode.Bytes - types []super.Type - uniq int -} - -func (c *collectionIter) appendNext(b *scode.Builder) { - if union, ok := c.typ.(*super.TypeUnion); ok && c.uniq > 1 { - val := super.NewValue(c.types[0], c.bytes[0]).Deunion() - super.BuildUnion(b, union.TagOf(val.Type()), val.Bytes()) - } else { - b.Append(c.bytes[0]) - } - c.bytes = c.bytes[1:] - c.types = c.types[1:] -} - -func (c *collectionIter) done() bool { - return len(c.types) == 0 -} - -func unionOf(sctx *super.Context, types []super.Type) super.Type { - if len(types) == 0 { - return super.TypeNull - } - unique := super.Flatten(super.UniqueTypes(types)) - if len(unique) == 1 { - return unique[0] - } - return sctx.MustLookupTypeUnion(unique) -} diff --git a/runtime/sam/op/meta/lister.go b/runtime/sam/op/meta/lister.go index 2f4b05b89d..82d8d323b4 100644 --- a/runtime/sam/op/meta/lister.go +++ b/runtime/sam/op/meta/lister.go @@ -44,14 +44,6 @@ func NewSortedLister(ctx context.Context, sctx *super.Context, pool *db.Pool, co return NewSortedListerFromSnap(ctx, sctx, pool, snap, pruner), nil } -func NewSortedListerByID(ctx context.Context, sctx *super.Context, r *db.Root, poolID, commit ksuid.KSUID, pruner expr.Evaluator) (*Lister, error) { - pool, err := r.OpenPool(ctx, poolID) - if err != nil { - return nil, err - } - return NewSortedLister(ctx, sctx, pool, commit, pruner) -} - func NewSortedListerFromSnap(ctx context.Context, sctx *super.Context, pool *db.Pool, snap commits.View, pruner expr.Evaluator) *Lister { m := sup.NewBSUPMarshalerWithContext(sctx) m.Decorate(sup.StylePackage) diff --git a/runtime/sam/op/spill/file.go b/runtime/sam/op/spill/file.go index 8b8d12424c..890e659d82 100644 --- a/runtime/sam/op/spill/file.go +++ b/runtime/sam/op/spill/file.go @@ -35,14 +35,6 @@ func NewFile(f *os.File) *File { } } -func NewTempFile() (*File, error) { - f, err := TempFile() - if err != nil { - return nil, err - } - return NewFile(f), nil -} - func NewFileWithPath(path string) (*File, error) { f, err := fs.Create(path) if err != nil { diff --git a/runtime/sam/op/spill/merge.go b/runtime/sam/op/spill/merge.go index 8fc98ae902..d3af08a93a 100644 --- a/runtime/sam/op/spill/merge.go +++ b/runtime/sam/op/spill/merge.go @@ -30,10 +30,6 @@ func TempDir() (string, error) { return os.MkdirTemp("", TempPrefix) } -func TempFile() (*os.File, error) { - return os.CreateTemp("", TempPrefix) -} - // NewMergeSort returns a MergeSort to implement external merge sorts of a large // BSUP stream. It creates a temporary directory to hold the collection // of spilled chunks. Call Cleanup to remove it.