From 40b1edd51b3263bbdc5284c544b410257b7d3bd1 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 17 Sep 2024 14:26:03 +0200 Subject: [PATCH] remove trace and reuse vexplain engine primitive Signed-off-by: Andres Taylor --- go/vt/vtgate/engine/cached_size.go | 92 +++++++-------------- go/vt/vtgate/engine/trace.go | 116 --------------------------- go/vt/vtgate/engine/vexplain.go | 109 +++++++++++++++++++------ go/vt/vtgate/planbuilder/vexplain.go | 5 +- go/vt/vtgate/vcursor_impl.go | 13 ++- 5 files changed, 122 insertions(+), 213 deletions(-) delete mode 100644 go/vt/vtgate/engine/trace.go diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 280556d4315..4c0d1009bd1 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -57,7 +57,7 @@ func (cached *AlterVSchema) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(24) + size += int64(16) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) @@ -89,7 +89,7 @@ func (cached *Concatenate) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(48) + size += int64(32) } // field Sources []vitess.io/vitess/go/vt/vtgate/engine.Primitive { @@ -119,7 +119,7 @@ func (cached *DBDDL) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(48) + size += int64(32) } // field name string size += hack.RuntimeAllocSize(int64(len(cached.name))) @@ -131,7 +131,7 @@ func (cached *DDL) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(80) + size += int64(64) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) @@ -252,7 +252,7 @@ func (cached *Distinct) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(64) + size += int64(48) } // field Source vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Source.(cachedObject); ok { @@ -461,7 +461,7 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(240) + size += int64(224) } // field InsertCommon vitess.io/vitess/go/vt/vtgate/engine.InsertCommon size += cached.InsertCommon.CachedSize(false) @@ -542,7 +542,7 @@ func (cached *InsertSelect) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(192) + size += int64(176) } // field InsertCommon vitess.io/vitess/go/vt/vtgate/engine.InsertCommon size += cached.InsertCommon.CachedSize(false) @@ -605,7 +605,7 @@ func (cached *Limit) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(64) + size += int64(48) } // field Count vitess.io/vitess/go/vt/vtgate/evalengine.Expr if cc, ok := cached.Count.(cachedObject); ok { @@ -627,7 +627,7 @@ func (cached *Lock) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(80) + size += int64(64) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) @@ -686,7 +686,7 @@ func (cached *MemorySort) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(80) + size += int64(64) } // field UpperLimit vitess.io/vitess/go/vt/vtgate/evalengine.Expr if cc, ok := cached.UpperLimit.(cachedObject); ok { @@ -749,7 +749,7 @@ func (cached *OnlineDDL) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(80) + size += int64(64) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) @@ -833,7 +833,7 @@ func (cached *Projection) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(80) + size += int64(64) } // field Cols []string { @@ -897,7 +897,7 @@ func (cached *RenameFields) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(80) + size += int64(64) } // field Cols []string { @@ -922,7 +922,7 @@ func (cached *ReplaceVariables) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(24) + size += int64(16) } // field Input vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Input.(cachedObject); ok { @@ -936,7 +936,7 @@ func (cached *RevertMigration) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(64) + size += int64(48) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) @@ -1038,7 +1038,7 @@ func (cached *Rows) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(64) + size += int64(48) } // field rows [][]vitess.io/vitess/go/sqltypes.Value { @@ -1067,7 +1067,7 @@ func (cached *SQLCalcFoundRows) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(48) + size += int64(32) } // field LimitPrimitive vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.LimitPrimitive.(cachedObject); ok { @@ -1085,7 +1085,7 @@ func (cached *ScalarAggregate) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(64) + size += int64(48) } // field Aggregates []*vitess.io/vitess/go/vt/vtgate/engine.AggregateParams { @@ -1158,7 +1158,7 @@ func (cached *Sequential) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(32) + size += int64(24) } // field Sources []vitess.io/vitess/go/vt/vtgate/engine.Primitive { @@ -1177,7 +1177,7 @@ func (cached *SessionPrimitive) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(32) + size += int64(24) } // field name string size += hack.RuntimeAllocSize(int64(len(cached.name))) @@ -1212,7 +1212,7 @@ func (cached *ShowExec) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(24) + size += int64(16) } // field ShowFilter *vitess.io/vitess/go/vt/sqlparser.ShowFilter size += cached.ShowFilter.CachedSize(true) @@ -1224,7 +1224,7 @@ func (cached *SimpleProjection) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(80) + size += int64(64) } // field Cols []int { @@ -1243,16 +1243,6 @@ func (cached *SimpleProjection) CachedSize(alloc bool) int64 { } return size } -func (cached *SingleRow) CachedSize(alloc bool) int64 { - if cached == nil { - return int64(0) - } - size := int64(0) - if alloc { - size += int64(16) - } - return size -} func (cached *SysVarCheckAndIgnore) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) @@ -1329,7 +1319,7 @@ func (cached *ThrottleApp) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(24) + size += int64(16) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) @@ -1337,27 +1327,13 @@ func (cached *ThrottleApp) CachedSize(alloc bool) int64 { size += cached.ThrottledAppRule.CachedSize(true) return size } -func (cached *Trace) CachedSize(alloc bool) int64 { - if cached == nil { - return int64(0) - } - size := int64(0) - if alloc { - size += int64(24) - } - // field Inner vitess.io/vitess/go/vt/vtgate/engine.Primitive - if cc, ok := cached.Inner.(cachedObject); ok { - size += cc.CachedSize(true) - } - return size -} func (cached *TransactionStatus) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) } size := int64(0) if alloc { - size += int64(48) + size += int64(32) } // field Keyspace string size += hack.RuntimeAllocSize(int64(len(cached.Keyspace))) @@ -1387,16 +1363,6 @@ func (cached *UncorrelatedSubquery) CachedSize(alloc bool) int64 { } return size } -func (cached *Unlock) CachedSize(alloc bool) int64 { - if cached == nil { - return int64(0) - } - size := int64(0) - if alloc { - size += int64(16) - } - return size -} //go:nocheckptr func (cached *Update) CachedSize(alloc bool) int64 { @@ -1405,7 +1371,7 @@ func (cached *Update) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(24) + size += int64(16) } // field DML *vitess.io/vitess/go/vt/vtgate/engine.DML size += cached.DML.CachedSize(true) @@ -1432,7 +1398,7 @@ func (cached *UpdateTarget) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(24) + size += int64(16) } // field Target string size += hack.RuntimeAllocSize(int64(len(cached.Target))) @@ -1444,7 +1410,7 @@ func (cached *Upsert) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(32) + size += int64(24) } // field Upserts []vitess.io/vitess/go/vt/vtgate/engine.upsert { @@ -1477,7 +1443,7 @@ func (cached *VExplain) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(32) + size += int64(24) } // field Input vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Input.(cachedObject); ok { @@ -1491,7 +1457,7 @@ func (cached *VStream) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(80) + size += int64(64) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) diff --git a/go/vt/vtgate/engine/trace.go b/go/vt/vtgate/engine/trace.go deleted file mode 100644 index fc256c2c8fd..00000000000 --- a/go/vt/vtgate/engine/trace.go +++ /dev/null @@ -1,116 +0,0 @@ -/* -Copyright 2024 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package engine - -import ( - "context" - "encoding/json" - - "vitess.io/vitess/go/mysql/collations" - - "vitess.io/vitess/go/sqltypes" - querypb "vitess.io/vitess/go/vt/proto/query" -) - -var _ Primitive = (*Trace)(nil) - -type Trace struct { - Inner Primitive -} - -type RowsReceived []int - -func (t *Trace) RouteType() string { - return t.Inner.RouteType() -} - -func (t *Trace) GetKeyspaceName() string { - return t.Inner.GetKeyspaceName() -} - -func (t *Trace) GetTableName() string { - return t.Inner.GetTableName() -} - -func getFields() []*querypb.Field { - return []*querypb.Field{{ - Name: "Trace", - Type: sqltypes.VarChar, - Charset: uint32(collations.SystemCollation.Collation), - Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), - }} -} - -func (t *Trace) GetFields(context.Context, VCursor, map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - return &sqltypes.Result{Fields: getFields()}, nil -} - -func (t *Trace) NeedsTransaction() bool { - return t.Inner.NeedsTransaction() -} - -func (t *Trace) TryExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool) (*sqltypes.Result, error) { - getOpStats := vcursor.StartPrimitiveTrace() - _, err := vcursor.ExecutePrimitive(ctx, t.Inner, bindVars, wantfields) - if err != nil { - return nil, err - } - - return t.getExplainTraceOutput(getOpStats) -} - -func (t *Trace) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, callback func(*sqltypes.Result) error) error { - getOpsStats := vcursor.StartPrimitiveTrace() - noop := func(result *sqltypes.Result) error { return nil } - err := vcursor.StreamExecutePrimitive(ctx, t.Inner, bindVars, wantfields, noop) - if err != nil { - return err - } - - res, err := t.getExplainTraceOutput(getOpsStats) - if err != nil { - return err - } - - return callback(res) -} - -func (t *Trace) getExplainTraceOutput(getOpStats func() map[Primitive]RowsReceived) (*sqltypes.Result, error) { - description := PrimitiveToPlanDescription(t.Inner, getOpStats()) - - output, err := json.MarshalIndent(description, "", "\t") - if err != nil { - return nil, err - } - - return &sqltypes.Result{ - Fields: getFields(), - Rows: []sqltypes.Row{{ - sqltypes.NewVarChar(string(output)), - }}, - }, nil -} - -func (t *Trace) Inputs() ([]Primitive, []map[string]any) { - return []Primitive{t.Inner}, nil -} - -func (t *Trace) description() PrimitiveDescription { - return PrimitiveDescription{ - OperatorType: "Trace", - } -} diff --git a/go/vt/vtgate/engine/vexplain.go b/go/vt/vtgate/engine/vexplain.go index 010901021fa..0aed4493ddc 100644 --- a/go/vt/vtgate/engine/vexplain.go +++ b/go/vt/vtgate/engine/vexplain.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -42,6 +43,8 @@ type ( Input Primitive Type sqlparser.VExplainType } + + RowsReceived []int ) var _ Primitive = (*VExplain)(nil) @@ -62,8 +65,43 @@ func (v *VExplain) GetTableName() string { } // GetFields implements the Primitive interface -func (v *VExplain) GetFields(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - return v.Input.GetFields(ctx, vcursor, bindVars) +func (v *VExplain) GetFields(context.Context, VCursor, map[string]*querypb.BindVariable) (*sqltypes.Result, error) { + var fields []*querypb.Field + switch v.Type { + case sqlparser.QueriesVExplainType: + fields = getVExplainQueriesFields() + case sqlparser.AllVExplainType: + fields = getVExplainAllFields() + case sqlparser.TraceVExplainType: + fields = getVExplainTraceFields() + default: + return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "Unknown type of VExplain plan") + } + return &sqltypes.Result{Fields: fields}, nil +} + +func getVExplainTraceFields() []*querypb.Field { + return []*querypb.Field{{ + Name: "Trace", + Type: sqltypes.VarChar, + Charset: uint32(collations.SystemCollation.Collation), + Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), + }} +} + +func getVExplainQueriesFields() []*querypb.Field { + return []*querypb.Field{ + {Name: "#", Type: sqltypes.Int32}, + {Name: "keyspace", Type: sqltypes.VarChar}, + {Name: "shard", Type: sqltypes.VarChar}, + {Name: "query", Type: sqltypes.VarChar}} + +} + +func getVExplainAllFields() []*querypb.Field { + return []*querypb.Field{{ + Name: "VExplain", Type: sqltypes.VarChar, + }} } // NeedsTransaction implements the Primitive interface @@ -73,42 +111,74 @@ func (v *VExplain) NeedsTransaction() bool { // TryExecute implements the Primitive interface func (v *VExplain) TryExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool) (*sqltypes.Result, error) { - vcursor.Session().VExplainLogging() + var stats func() map[Primitive]RowsReceived + if v.Type == sqlparser.TraceVExplainType { + stats = vcursor.StartPrimitiveTrace() + } else { + vcursor.Session().VExplainLogging() + } _, err := vcursor.ExecutePrimitive(ctx, v.Input, bindVars, wantfields) if err != nil { return nil, err } - return v.convertToResult(ctx, vcursor) + return v.convertToResult(ctx, vcursor, stats) +} + +func noOpCallback(*sqltypes.Result) error { + return nil } // TryStreamExecute implements the Primitive interface func (v *VExplain) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, callback func(*sqltypes.Result) error) error { - vcursor.Session().VExplainLogging() - err := vcursor.StreamExecutePrimitive(ctx, v.Input, bindVars, wantfields, func(result *sqltypes.Result) error { - return nil - }) + var stats func() map[Primitive]RowsReceived + if v.Type == sqlparser.TraceVExplainType { + stats = vcursor.StartPrimitiveTrace() + } else { + vcursor.Session().VExplainLogging() + } + + err := vcursor.StreamExecutePrimitive(ctx, v.Input, bindVars, wantfields, noOpCallback) if err != nil { return err } - result, err := v.convertToResult(ctx, vcursor) + result, err := v.convertToResult(ctx, vcursor, stats) if err != nil { return err } return callback(result) } -func (v *VExplain) convertToResult(ctx context.Context, vcursor VCursor) (*sqltypes.Result, error) { +func (v *VExplain) convertToResult(ctx context.Context, vcursor VCursor, stats func() map[Primitive]RowsReceived) (*sqltypes.Result, error) { switch v.Type { case sqlparser.QueriesVExplainType: result := convertToVExplainQueriesResult(vcursor.Session().GetVExplainLogs()) return result, nil case sqlparser.AllVExplainType: return v.convertToVExplainAllResult(ctx, vcursor) + case sqlparser.TraceVExplainType: + return v.getExplainTraceOutput(stats) + default: return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "Unknown type of VExplain plan") } } +func (v *VExplain) getExplainTraceOutput(getOpStats func() map[Primitive]RowsReceived) (*sqltypes.Result, error) { + description := PrimitiveToPlanDescription(v.Input, getOpStats()) + + output, err := json.MarshalIndent(description, "", "\t") + if err != nil { + return nil, err + } + + return &sqltypes.Result{ + Fields: getVExplainTraceFields(), + Rows: []sqltypes.Row{{ + sqltypes.NewVarChar(string(output)), + }}, + }, nil +} + func (v *VExplain) convertToVExplainAllResult(ctx context.Context, vcursor VCursor) (*sqltypes.Result, error) { logEntries := vcursor.Session().GetVExplainLogs() explainResults := make(map[Primitive]string) @@ -144,18 +214,14 @@ func (v *VExplain) convertToVExplainAllResult(ctx context.Context, vcursor VCurs } result := string(resultBytes) - fields := []*querypb.Field{ - { - Name: "VExplain", Type: sqltypes.VarChar, - }, - } + rows := []sqltypes.Row{ { sqltypes.NewVarChar(result), }, } qr := &sqltypes.Result{ - Fields: fields, + Fields: getVExplainAllFields(), Rows: rows, } return qr, nil @@ -193,17 +259,8 @@ func primitiveToPlanDescriptionWithSQLResults(in Primitive, res map[Primitive]st } func convertToVExplainQueriesResult(logs []ExecuteEntry) *sqltypes.Result { - fields := []*querypb.Field{{ - Name: "#", Type: sqltypes.Int32, - }, { - Name: "keyspace", Type: sqltypes.VarChar, - }, { - Name: "shard", Type: sqltypes.VarChar, - }, { - Name: "query", Type: sqltypes.VarChar, - }} qr := &sqltypes.Result{ - Fields: fields, + Fields: getVExplainQueriesFields(), } for _, line := range logs { qr.Rows = append(qr.Rows, sqltypes.Row{ diff --git a/go/vt/vtgate/planbuilder/vexplain.go b/go/vt/vtgate/planbuilder/vexplain.go index 9b14a8bbf97..f31eeee1714 100644 --- a/go/vt/vtgate/planbuilder/vexplain.go +++ b/go/vt/vtgate/planbuilder/vexplain.go @@ -176,6 +176,9 @@ func buildVExplainTracePlan(ctx context.Context, explainStatement sqlparser.Stat } // We'll set the trace engine as the root primitive - innerInstruction.primitive = &engine.Trace{Inner: innerInstruction.primitive} + innerInstruction.primitive = &engine.VExplain{ + Input: innerInstruction.primitive, + Type: sqlparser.TraceVExplainType, + } return innerInstruction, nil } diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index cfa0b382ae0..375f930a50f 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -135,7 +135,7 @@ type ( // this is a map of the number of rows that every primitive has returned // if this field is nil, it means that we are not logging operator traffic - primitiveStats map[int]engine.RowsReceived + primitiveStats map[engine.Primitive]engine.RowsReceived } ) @@ -285,9 +285,9 @@ func (vc *vcursorImpl) UnresolvedTransactions(ctx context.Context, keyspace stri return vc.executor.UnresolvedTransactions(ctx, targets) } -func (vc *vcursorImpl) StartPrimitiveTrace() func() map[int]engine.RowsReceived { - vc.primitiveStats = make(map[int]engine.RowsReceived) - return func() map[int]engine.RowsReceived { +func (vc *vcursorImpl) StartPrimitiveTrace() func() map[engine.Primitive]engine.RowsReceived { + vc.primitiveStats = make(map[engine.Primitive]engine.RowsReceived) + return func() map[engine.Primitive]engine.RowsReceived { return vc.primitiveStats } } @@ -533,14 +533,13 @@ func (vc *vcursorImpl) ExecutePrimitive(ctx context.Context, primitive engine.Pr func (vc *vcursorImpl) logOpTraffic(primitive engine.Primitive, res *sqltypes.Result) { if vc.primitiveStats != nil { - key := int(primitive.GetID()) - rows := vc.primitiveStats[key] + rows := vc.primitiveStats[primitive] if res == nil { rows = append(rows, 0) } else { rows = append(rows, len(res.Rows)) } - vc.primitiveStats[key] = rows + vc.primitiveStats[primitive] = rows } }