diff --git a/go.mod b/go.mod index c6210bacf..c2c01a784 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/blevesearch/go-porterstemmer v1.0.3 github.com/blevesearch/goleveldb v1.0.1 github.com/blevesearch/gtreap v0.1.1 - github.com/blevesearch/scorch_segment_api/v2 v2.4.9-0.20260729090843-4313bda09bee + github.com/blevesearch/scorch_segment_api/v2 v2.4.9 github.com/blevesearch/segment v0.9.1 github.com/blevesearch/snowball v0.6.1 github.com/blevesearch/snowballstem v0.9.0 @@ -25,7 +25,7 @@ require ( github.com/blevesearch/zapx/v14 v14.4.3 github.com/blevesearch/zapx/v15 v15.4.3 github.com/blevesearch/zapx/v16 v16.3.4 - github.com/blevesearch/zapx/v17 v17.2.2-0.20260806075952-6594f3d6bd0d + github.com/blevesearch/zapx/v17 v17.2.2 github.com/couchbase/moss v0.2.0 github.com/spf13/cobra v1.10.2 go.etcd.io/bbolt v1.4.0 diff --git a/go.sum b/go.sum index ce1b629f5..f1efe06cd 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= github.com/blevesearch/mmap-go v1.2.0 h1:l33nNKPFcBjJUMwem6sAYJPUzhUCABoK9FxZDGiFNBI= github.com/blevesearch/mmap-go v1.2.0/go.mod h1:Vd6+20GBhEdwJnU1Xohgt88XCD/CTWcqbCNxkZpyBo0= -github.com/blevesearch/scorch_segment_api/v2 v2.4.9-0.20260729090843-4313bda09bee h1:RmX8uCxp4RDbsW0PRFsFED5f7X5xUvz6/9cYb9HrNTU= -github.com/blevesearch/scorch_segment_api/v2 v2.4.9-0.20260729090843-4313bda09bee/go.mod h1:WUUkAocbkDlNK/kgAE13NvS9oxe+u618mYZ8sOvcCc4= +github.com/blevesearch/scorch_segment_api/v2 v2.4.9 h1:q1shQvpwVGp0IISmX08Ead5XFOPNmZrB761Snpru3vk= +github.com/blevesearch/scorch_segment_api/v2 v2.4.9/go.mod h1:WUUkAocbkDlNK/kgAE13NvS9oxe+u618mYZ8sOvcCc4= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A= @@ -45,8 +45,8 @@ github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2l github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= github.com/blevesearch/zapx/v16 v16.3.4 h1:hDAqA8qusZTNbPEL7//w5P65UZ2de6yhSeUaTbp0Po0= github.com/blevesearch/zapx/v16 v16.3.4/go.mod h1:zqkPPqs9GS9FzVWzCO3Wf1X044yWAV17+4zb+FTiEHg= -github.com/blevesearch/zapx/v17 v17.2.2-0.20260806075952-6594f3d6bd0d h1:NsgFVBOgq3lg7ClW4e7iJc5OsW7MFlsSIGxiGW7++Fw= -github.com/blevesearch/zapx/v17 v17.2.2-0.20260806075952-6594f3d6bd0d/go.mod h1:hPt57M7CxaMgVQF1lYak7rRaO7Ge5j/sdYzrs8t8Htk= +github.com/blevesearch/zapx/v17 v17.2.2 h1:v3as9jp2CFsrPKpBotYWn/Evp/EjElH4xpNvOhgsNTU= +github.com/blevesearch/zapx/v17 v17.2.2/go.mod h1:r/ddAUuomMMcx12ZUBoDxf/9Fa4aqvFATf7AJzWp/Ac= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o= diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go index 2cbea29ac..af65f303f 100644 --- a/index/scorch/scorch.go +++ b/index/scorch/scorch.go @@ -44,6 +44,7 @@ var ErrClosed = fmt.Errorf("scorch closed") type Scorch struct { nextSegmentID uint64 stats Stats + zapStats segment.Stats iStats internalStats readOnly bool @@ -213,6 +214,9 @@ func NewScorch(storeName string, rv.segmentConfig = segConfig } + // NOTE: always register the stats handler in the config map to track zap layer stats + rv.segmentConfig[segment.StatsKey] = &rv.zapStats + typ, ok := config["spatialPlugin"].(string) if ok { if err := rv.loadSpatialAnalyzerPlugin(typ); err != nil { @@ -853,6 +857,9 @@ func (s *Scorch) StatsMap() map[string]interface{} { m["field:"+fieldName+":"+statName] = val } } + + // zap layer stats that are continously updated throughout the index lifecycle + m[segment.StatsKey] = s.zapStats.StatsMap() return m } diff --git a/index/scorch/stats.go b/index/scorch/stats.go index da648ed4c..20bda3f47 100644 --- a/index/scorch/stats.go +++ b/index/scorch/stats.go @@ -144,8 +144,10 @@ type Stats struct { TotMemorySegmentsAtRoot uint64 - TotTrainedSamples uint64 - TotTrainTime uint64 + TotTrainedSamples uint64 + TotTrainTime uint64 + TotTrainFireIndexEventTime uint64 + TotTrainFireIndexEvents uint64 } // atomically populates the returned map diff --git a/index/scorch/train_vector.go b/index/scorch/train_vector.go index 8a606975b..c717eab33 100644 --- a/index/scorch/train_vector.go +++ b/index/scorch/train_vector.go @@ -69,7 +69,7 @@ func initTrainer(s *Scorch, config map[string]interface{}) *vectorTrainer { if ok && feature { trainer := vectorTrainer{ parent: s, - config: maps.Clone(s.config), + config: maps.Clone(s.segmentConfig), trainCh: make(chan *trainRequest, 1), doneCh: make(chan struct{}), } @@ -340,7 +340,10 @@ func (t *vectorTrainer) loadTrainedData(bucket *util.BoltBucketImpl) error { func (t *vectorTrainer) train(batch *index.Batch) error { // regulate the Train function + start := time.Now() t.parent.FireIndexEvent() + atomic.AddUint64(&t.parent.stats.TotTrainFireIndexEventTime, uint64(time.Since(start))) + atomic.AddUint64(&t.parent.stats.TotTrainFireIndexEvents, 1) if t.trainingComplete.Load() { return fmt.Errorf("training is already complete, cannot accept more training data") } @@ -393,7 +396,6 @@ func (t *vectorTrainer) train(batch *index.Batch) error { // is complete, the template will be used for other operations down the line // like merge and search. // - // note: this might index text data too, how to handle this? s.segmentConfig? // todo: updates/deletes -> data drift detection if len(trainData) > 0 { trainReq.sample, _, err = t.parent.segPlugin.NewUsing(trainData, config)