From 35a3e04534bc28bbfab3d856d0a71d97529a8525 Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Tue, 9 Jun 2026 15:08:09 -0700 Subject: [PATCH 1/4] stats --- index/scorch/scorch.go | 4 ++++ index/scorch/segment_plugin.go | 16 +++++++++------- index/scorch/train_vector.go | 1 - 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go index 2cbea29ac..c89bfaf17 100644 --- a/index/scorch/scorch.go +++ b/index/scorch/scorch.go @@ -853,6 +853,10 @@ func (s *Scorch) StatsMap() map[string]interface{} { m["field:"+fieldName+":"+statName] = val } } + + // these stats are the segment level stats captured throughout the index lifecycle + zapStats := s.segPlugin.StatsMap() + m["segment_stats"] = zapStats return m } diff --git a/index/scorch/segment_plugin.go b/index/scorch/segment_plugin.go index 16be8e440..5be62ade5 100644 --- a/index/scorch/segment_plugin.go +++ b/index/scorch/segment_plugin.go @@ -75,6 +75,8 @@ type SegmentPlugin interface { MergeUsing(segments []segment.Segment, drops []*roaring.Bitmap, path string, closeCh chan struct{}, s segment.StatsReporter, config map[string]interface{}) ( [][]uint64, uint64, error) + + StatsMap() map[string]interface{} } var supportedSegmentPlugins map[string]map[uint32]SegmentPlugin @@ -82,13 +84,13 @@ var defaultSegmentPlugin SegmentPlugin func init() { ResetSegmentPlugins() - RegisterSegmentPlugin(&zapv17.ZapPlugin{}, true) - RegisterSegmentPlugin(&zapv16.ZapPlugin{}, false) - RegisterSegmentPlugin(&zapv15.ZapPlugin{}, false) - RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false) - RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false) - RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false) - RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false) + RegisterSegmentPlugin(zapv17.InitPlugin(), true) + RegisterSegmentPlugin(zapv16.InitPlugin(), false) + RegisterSegmentPlugin(zapv15.InitPlugin(), false) + RegisterSegmentPlugin(zapv14.InitPlugin(), false) + RegisterSegmentPlugin(zapv13.InitPlugin(), false) + RegisterSegmentPlugin(zapv12.InitPlugin(), false) + RegisterSegmentPlugin(zapv11.InitPlugin(), false) } func ResetSegmentPlugins() { diff --git a/index/scorch/train_vector.go b/index/scorch/train_vector.go index 8a606975b..40abdaa99 100644 --- a/index/scorch/train_vector.go +++ b/index/scorch/train_vector.go @@ -393,7 +393,6 @@ func (t *vectorTrainer) train(batch *index.Batch) error { // is complete, the template will be used for other operations down the line // like merge and search. // - // note: this might index text data too, how to handle this? s.segmentConfig? // todo: updates/deletes -> data drift detection if len(trainData) > 0 { trainReq.sample, _, err = t.parent.segPlugin.NewUsing(trainData, config) From 2426b826259c423ff454b15efbf2c147161234f3 Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Tue, 7 Jul 2026 13:39:35 -0700 Subject: [PATCH 2/4] add more training stats --- index/scorch/stats.go | 6 ++++-- index/scorch/train_vector.go | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/index/scorch/stats.go b/index/scorch/stats.go index da648ed4c..20bda3f47 100644 --- a/index/scorch/stats.go +++ b/index/scorch/stats.go @@ -144,8 +144,10 @@ type Stats struct { TotMemorySegmentsAtRoot uint64 - TotTrainedSamples uint64 - TotTrainTime uint64 + TotTrainedSamples uint64 + TotTrainTime uint64 + TotTrainFireIndexEventTime uint64 + TotTrainFireIndexEvents uint64 } // atomically populates the returned map diff --git a/index/scorch/train_vector.go b/index/scorch/train_vector.go index 40abdaa99..829426f6d 100644 --- a/index/scorch/train_vector.go +++ b/index/scorch/train_vector.go @@ -340,7 +340,10 @@ func (t *vectorTrainer) loadTrainedData(bucket *util.BoltBucketImpl) error { func (t *vectorTrainer) train(batch *index.Batch) error { // regulate the Train function + start := time.Now() t.parent.FireIndexEvent() + atomic.AddUint64(&t.parent.stats.TotTrainFireIndexEventTime, uint64(time.Since(start))) + atomic.AddUint64(&t.parent.stats.TotTrainFireIndexEvents, 1) if t.trainingComplete.Load() { return fmt.Errorf("training is already complete, cannot accept more training data") } From ffa9539c46edbbc89c28cac2f9ece9dbd9f8d856 Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Wed, 19 Aug 2026 13:53:45 -0700 Subject: [PATCH 3/4] fix the zap stats to be per bleve index --- index/scorch/scorch.go | 9 ++++++--- index/scorch/segment_plugin.go | 16 +++++++--------- index/scorch/train_vector.go | 5 ++++- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go index c89bfaf17..af65f303f 100644 --- a/index/scorch/scorch.go +++ b/index/scorch/scorch.go @@ -44,6 +44,7 @@ var ErrClosed = fmt.Errorf("scorch closed") type Scorch struct { nextSegmentID uint64 stats Stats + zapStats segment.Stats iStats internalStats readOnly bool @@ -213,6 +214,9 @@ func NewScorch(storeName string, rv.segmentConfig = segConfig } + // NOTE: always register the stats handler in the config map to track zap layer stats + rv.segmentConfig[segment.StatsKey] = &rv.zapStats + typ, ok := config["spatialPlugin"].(string) if ok { if err := rv.loadSpatialAnalyzerPlugin(typ); err != nil { @@ -854,9 +858,8 @@ func (s *Scorch) StatsMap() map[string]interface{} { } } - // these stats are the segment level stats captured throughout the index lifecycle - zapStats := s.segPlugin.StatsMap() - m["segment_stats"] = zapStats + // zap layer stats that are continously updated throughout the index lifecycle + m[segment.StatsKey] = s.zapStats.StatsMap() return m } diff --git a/index/scorch/segment_plugin.go b/index/scorch/segment_plugin.go index 5be62ade5..16be8e440 100644 --- a/index/scorch/segment_plugin.go +++ b/index/scorch/segment_plugin.go @@ -75,8 +75,6 @@ type SegmentPlugin interface { MergeUsing(segments []segment.Segment, drops []*roaring.Bitmap, path string, closeCh chan struct{}, s segment.StatsReporter, config map[string]interface{}) ( [][]uint64, uint64, error) - - StatsMap() map[string]interface{} } var supportedSegmentPlugins map[string]map[uint32]SegmentPlugin @@ -84,13 +82,13 @@ var defaultSegmentPlugin SegmentPlugin func init() { ResetSegmentPlugins() - RegisterSegmentPlugin(zapv17.InitPlugin(), true) - RegisterSegmentPlugin(zapv16.InitPlugin(), false) - RegisterSegmentPlugin(zapv15.InitPlugin(), false) - RegisterSegmentPlugin(zapv14.InitPlugin(), false) - RegisterSegmentPlugin(zapv13.InitPlugin(), false) - RegisterSegmentPlugin(zapv12.InitPlugin(), false) - RegisterSegmentPlugin(zapv11.InitPlugin(), false) + RegisterSegmentPlugin(&zapv17.ZapPlugin{}, true) + RegisterSegmentPlugin(&zapv16.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv15.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false) } func ResetSegmentPlugins() { diff --git a/index/scorch/train_vector.go b/index/scorch/train_vector.go index 829426f6d..8032bd862 100644 --- a/index/scorch/train_vector.go +++ b/index/scorch/train_vector.go @@ -69,7 +69,7 @@ func initTrainer(s *Scorch, config map[string]interface{}) *vectorTrainer { if ok && feature { trainer := vectorTrainer{ parent: s, - config: maps.Clone(s.config), + config: maps.Clone(s.segmentConfig), trainCh: make(chan *trainRequest, 1), doneCh: make(chan struct{}), } @@ -398,6 +398,9 @@ func (t *vectorTrainer) train(batch *index.Batch) error { // // todo: updates/deletes -> data drift detection if len(trainData) > 0 { + if _, ok := config[segment.StatsKey]; !ok { + fmt.Println("missing stats in config") + } trainReq.sample, _, err = t.parent.segPlugin.NewUsing(trainData, config) if err != nil { return err From 161ed7c2f0c764a1324bff6a71c225199edad88b Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Fri, 21 Aug 2026 08:31:48 -0700 Subject: [PATCH 4/4] go.mod update --- go.mod | 4 ++-- go.sum | 8 ++++---- index/scorch/train_vector.go | 3 --- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c6210bacf..c2c01a784 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/blevesearch/go-porterstemmer v1.0.3 github.com/blevesearch/goleveldb v1.0.1 github.com/blevesearch/gtreap v0.1.1 - github.com/blevesearch/scorch_segment_api/v2 v2.4.9-0.20260729090843-4313bda09bee + github.com/blevesearch/scorch_segment_api/v2 v2.4.9 github.com/blevesearch/segment v0.9.1 github.com/blevesearch/snowball v0.6.1 github.com/blevesearch/snowballstem v0.9.0 @@ -25,7 +25,7 @@ require ( github.com/blevesearch/zapx/v14 v14.4.3 github.com/blevesearch/zapx/v15 v15.4.3 github.com/blevesearch/zapx/v16 v16.3.4 - github.com/blevesearch/zapx/v17 v17.2.2-0.20260806075952-6594f3d6bd0d + github.com/blevesearch/zapx/v17 v17.2.2 github.com/couchbase/moss v0.2.0 github.com/spf13/cobra v1.10.2 go.etcd.io/bbolt v1.4.0 diff --git a/go.sum b/go.sum index ce1b629f5..f1efe06cd 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= github.com/blevesearch/mmap-go v1.2.0 h1:l33nNKPFcBjJUMwem6sAYJPUzhUCABoK9FxZDGiFNBI= github.com/blevesearch/mmap-go v1.2.0/go.mod h1:Vd6+20GBhEdwJnU1Xohgt88XCD/CTWcqbCNxkZpyBo0= -github.com/blevesearch/scorch_segment_api/v2 v2.4.9-0.20260729090843-4313bda09bee h1:RmX8uCxp4RDbsW0PRFsFED5f7X5xUvz6/9cYb9HrNTU= -github.com/blevesearch/scorch_segment_api/v2 v2.4.9-0.20260729090843-4313bda09bee/go.mod h1:WUUkAocbkDlNK/kgAE13NvS9oxe+u618mYZ8sOvcCc4= +github.com/blevesearch/scorch_segment_api/v2 v2.4.9 h1:q1shQvpwVGp0IISmX08Ead5XFOPNmZrB761Snpru3vk= +github.com/blevesearch/scorch_segment_api/v2 v2.4.9/go.mod h1:WUUkAocbkDlNK/kgAE13NvS9oxe+u618mYZ8sOvcCc4= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A= @@ -45,8 +45,8 @@ github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2l github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= github.com/blevesearch/zapx/v16 v16.3.4 h1:hDAqA8qusZTNbPEL7//w5P65UZ2de6yhSeUaTbp0Po0= github.com/blevesearch/zapx/v16 v16.3.4/go.mod h1:zqkPPqs9GS9FzVWzCO3Wf1X044yWAV17+4zb+FTiEHg= -github.com/blevesearch/zapx/v17 v17.2.2-0.20260806075952-6594f3d6bd0d h1:NsgFVBOgq3lg7ClW4e7iJc5OsW7MFlsSIGxiGW7++Fw= -github.com/blevesearch/zapx/v17 v17.2.2-0.20260806075952-6594f3d6bd0d/go.mod h1:hPt57M7CxaMgVQF1lYak7rRaO7Ge5j/sdYzrs8t8Htk= +github.com/blevesearch/zapx/v17 v17.2.2 h1:v3as9jp2CFsrPKpBotYWn/Evp/EjElH4xpNvOhgsNTU= +github.com/blevesearch/zapx/v17 v17.2.2/go.mod h1:r/ddAUuomMMcx12ZUBoDxf/9Fa4aqvFATf7AJzWp/Ac= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o= diff --git a/index/scorch/train_vector.go b/index/scorch/train_vector.go index 8032bd862..c717eab33 100644 --- a/index/scorch/train_vector.go +++ b/index/scorch/train_vector.go @@ -398,9 +398,6 @@ func (t *vectorTrainer) train(batch *index.Batch) error { // // todo: updates/deletes -> data drift detection if len(trainData) > 0 { - if _, ok := config[segment.StatsKey]; !ok { - fmt.Println("missing stats in config") - } trainReq.sample, _, err = t.parent.segPlugin.NewUsing(trainData, config) if err != nil { return err