From 1577da18a29a59bc774ed1c1dd80c6c0d9791476 Mon Sep 17 00:00:00 2001 From: biast12 <53872542+biast12@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:27:05 +0200 Subject: [PATCH 1/2] Harden purge flow and one-shot shutdown Refactors daemon purge handling to return errors, purge database data before irreversible transcript deletion, and add bounded polling with timeout/backoff when waiting on logarchiver status. Improves failure reporting by surfacing detailed per-object errors and fixes context usage for guild checks. Also updates main loop cleanup so one-shot runs flush Sentry/loggers before exit, and bumps TicketsBot cloud dependencies (with local common/database replaces enabled). --- cmd/cleanupdaemon/main.go | 19 +++- go.mod | 12 +-- go.sum | 16 ++-- pkg/daemon/daemon.go | 195 +++++++++++++++++++++----------------- 4 files changed, 135 insertions(+), 107 deletions(-) diff --git a/cmd/cleanupdaemon/main.go b/cmd/cleanupdaemon/main.go index d4df581..0e0a697 100644 --- a/cmd/cleanupdaemon/main.go +++ b/cmd/cleanupdaemon/main.go @@ -73,12 +73,21 @@ func main() { }) daemon := daemon.NewDaemon(logger, conf, client, db) + + // sentry-go buffers on an async transport; ONESHOT would otherwise drop the + // final run's events on return. + defer sentry.Flush(time.Second * 2) + defer logger.Sync() + defer pool.Close() + daemon.Run() - if !conf.OneShot { - for { - time.Sleep(time.Hour * 6) - daemon.Run() - } + if conf.OneShot { + return + } + + for { + time.Sleep(time.Hour * 6) + daemon.Run() } } diff --git a/go.mod b/go.mod index 0fea474..2d87655 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,14 @@ go 1.24.0 //replace github.com/TicketsBot-cloud/archiverclient => ../archiverclient -//replace github.com/TicketsBot-cloud/common => ../common +replace github.com/TicketsBot-cloud/common => ../common -//replace github.com/TicketsBot-cloud/database => ../database +replace github.com/TicketsBot-cloud/database => ../database require ( - github.com/TicketsBot-cloud/archiverclient v0.0.0-20260106203614-3f2fdd061b8b - github.com/TicketsBot-cloud/common v0.0.0-20251026182733-99fa0dc31d90 - github.com/TicketsBot-cloud/database v0.0.0-20260106203748-e9a3db5cdcd6 + github.com/TicketsBot-cloud/archiverclient v0.0.0-20260425075746-c37717f34d16 + github.com/TicketsBot-cloud/common v0.0.0-20260827064609-69131fc7bd3e + github.com/TicketsBot-cloud/database v0.0.0-20260829063441-2dc7109928ef github.com/caarlos0/env/v6 v6.10.1 github.com/getsentry/sentry-go v0.40.0 github.com/jackc/pgx/v4 v4.18.3 @@ -20,7 +20,7 @@ require ( ) require ( - github.com/TicketsBot-cloud/gdl v0.0.0-20260103195545-cf76b185bdbc // indirect + github.com/TicketsBot-cloud/gdl v0.0.0-20260306134952-cccb0116fef6 // indirect github.com/TicketsBot-cloud/logarchiver v0.0.0-20251018211319-7a7df5cacbdc // indirect github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 // indirect github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 // indirect diff --git a/go.sum b/go.sum index adf0c65..271b65e 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,14 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/ReneKroon/ttlcache v1.6.0/go.mod h1:DG6nbhXKUQhrExfwwLuZUdH7UnRDDRA1IW+nBuCssvs= -github.com/TicketsBot-cloud/archiverclient v0.0.0-20260106203614-3f2fdd061b8b h1:rzTKxzhl0oZusxro/9tIqDBmXrHmxjFdKQdPvaEOy4k= -github.com/TicketsBot-cloud/archiverclient v0.0.0-20260106203614-3f2fdd061b8b/go.mod h1:Mux1bEPpOHwRw1wo6Fa6qJLJH9Erk9qv1yAIfLi1Wmw= -github.com/TicketsBot-cloud/common v0.0.0-20251026182733-99fa0dc31d90 h1:gv7uVneGf22eTwT976iWpCPWA6yPY51nfWPadcDreWI= -github.com/TicketsBot-cloud/common v0.0.0-20251026182733-99fa0dc31d90/go.mod h1:uRra7EnPtmd7Og9dLZ7SYgb0AwODeBnd5qSaDdWcjtU= -github.com/TicketsBot-cloud/database v0.0.0-20260106203748-e9a3db5cdcd6 h1:OP+Fqvyffc2ZP44blvzlaeiJOvhNmLaCGpHrE24xoBY= -github.com/TicketsBot-cloud/database v0.0.0-20260106203748-e9a3db5cdcd6/go.mod h1:HQXAgmNSm7/FmBYwcsa6qpZqMrDhbLoEl+AyqFQ+RwY= -github.com/TicketsBot-cloud/gdl v0.0.0-20260103195545-cf76b185bdbc h1:V8ZVITR6CTJm9uwwrneAN/3FKR/epOPJIHhiEh0HMIQ= -github.com/TicketsBot-cloud/gdl v0.0.0-20260103195545-cf76b185bdbc/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= +github.com/TicketsBot-cloud/archiverclient v0.0.0-20260425075746-c37717f34d16 h1:VMuquj0D0OSGxUNYzJAtSetOkWlkE2Fg1x/eX9WrL1s= +github.com/TicketsBot-cloud/archiverclient v0.0.0-20260425075746-c37717f34d16/go.mod h1:Mux1bEPpOHwRw1wo6Fa6qJLJH9Erk9qv1yAIfLi1Wmw= +github.com/TicketsBot-cloud/common v0.0.0-20260827064609-69131fc7bd3e h1:66mDH3lvotbWSgQXNVbssCvUgy6Yg1mqKl7RmEdr6y4= +github.com/TicketsBot-cloud/common v0.0.0-20260827064609-69131fc7bd3e/go.mod h1:yL+VPSYNVK5gxUkA+fbb0WdTjGfWXnr7ScfL0rwk29g= +github.com/TicketsBot-cloud/database v0.0.0-20260829063441-2dc7109928ef h1:MRQgmmfAUP2tDXfiWpCvV6GBiSXWjOMpOrH3iHf6TUk= +github.com/TicketsBot-cloud/database v0.0.0-20260829063441-2dc7109928ef/go.mod h1:HQXAgmNSm7/FmBYwcsa6qpZqMrDhbLoEl+AyqFQ+RwY= +github.com/TicketsBot-cloud/gdl v0.0.0-20260306134952-cccb0116fef6 h1:ucG0xLPt7xixW7/LvL0hXDBDouDRS1Nf+77qP8iJ/X0= +github.com/TicketsBot-cloud/gdl v0.0.0-20260306134952-cccb0116fef6/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= github.com/TicketsBot-cloud/logarchiver v0.0.0-20251018211319-7a7df5cacbdc h1:qTLNpCvIqM7UwZ6MdWQ9EztcDsIJfHh+VJdG+ULLEaA= github.com/TicketsBot-cloud/logarchiver v0.0.0-20251018211319-7a7df5cacbdc/go.mod h1:pZqkzPNNTqnwKZvCT8kCaTHxrG7HJbxZV83S0p7mmzM= github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 h1:FqC1KGOsmB+ikvbmDkyNQU6bGUWyfYq8Ip9r4KxTveY= diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 7a904e8..467d3fc 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -3,8 +3,9 @@ package daemon import ( "context" "errors" - "log" - "math" + "fmt" + "sort" + "strings" "time" "github.com/TicketsBot-cloud/archiverclient" @@ -15,7 +16,17 @@ import ( "go.uber.org/zap" ) -const BreakTime = time.Second +const ( + BreakTime = time.Second + + RetentionPeriod = time.Hour * 24 * 28 + + // Without this the status poll below has no exit of its own. + purgeTimeout = 5 * time.Minute + + minPollInterval = time.Second * 10 + maxPollInterval = time.Minute +) type Daemon struct { logger *zap.Logger @@ -37,9 +48,9 @@ func (d *Daemon) Run() { d.logger.Info("Starting run...") ctx := context.Background() - guildIds, err := d.database.GuildLeaveTime.GetBefore(ctx, time.Hour * 24 * 28) + guildIds, err := d.database.GuildLeaveTime.GetBefore(ctx, RetentionPeriod) if err != nil { - log.Printf("error occurred while fetching guild ids: %s\n", err.Error()) + d.logger.Error("Error while fetching guild ids", zap.Error(err)) return } @@ -65,106 +76,114 @@ func (d *Daemon) Run() { continue } - if d.purgeGuild(ctx, guildId) { - if err := d.database.GuildLeaveTime.Delete(ctx, guildId); err != nil { - logger.Error("error while deleting leave times", zap.Error(err)) - } + if err := d.purgeGuild(ctx, guildId); err != nil { + logger.Error("Failed to purge guild", zap.Error(err)) + continue + } + + if err := d.database.GuildLeaveTime.Delete(ctx, guildId); err != nil { + logger.Error("error while deleting leave times", zap.Error(err)) } } } -func (d *Daemon) purgeGuild(ctx context.Context, guildId uint64) bool { +// Database first: the archiver delete is irreversible, so the other order lets a +// database failure destroy the transcripts. Both halves are idempotent. +func (d *Daemon) purgeGuild(ctx context.Context, guildId uint64) error { + if err := d.database.PurgeGuildData(ctx, guildId, d.logger); err != nil { + return fmt.Errorf("purging guild data from database: %w", err) + } + + return d.purgeTranscripts(ctx, guildId) +} + +func (d *Daemon) purgeTranscripts(ctx context.Context, guildId uint64) error { + ctx, cancel := context.WithTimeout(ctx, purgeTimeout) + defer cancel() + if err := d.client.PurgeGuild(ctx, guildId); err != nil { - d.logger.Error("Error sending purge request", zap.Error(err), zap.Uint64("guild", guildId)) - return false + return fmt.Errorf("sending purge request to logarchiver: %w", err) } - var attempt int - for { + for attempt := 0; ; attempt++ { + if err := ctx.Err(); err != nil { + return fmt.Errorf("timed out waiting for logarchiver purge: %w", err) + } + status, err := d.client.PurgeStatus(ctx, guildId) if err != nil { - if err == archiverclient.ErrOperationNotFound { - d.logger.Warn( - "logarchiver return not found when fetching purge status", - zap.Uint64("guild", guildId), - ) - } else { - d.logger.Error( - "Error when fetching purge status from logarchiver", - zap.Uint64("guild", guildId), - zap.Error(err), - ) + if errors.Is(err, archiverclient.ErrOperationNotFound) { + return fmt.Errorf("logarchiver lost the purge operation: %w", err) } - return false + return fmt.Errorf("fetching purge status from logarchiver: %w", err) } switch status.Status { - case archiverclient.StatusComplete: - d.logger.Info( - "logarchiver purge completed successfully", - zap.Uint64("guild", guildId), - ) - - // Purge all guild data from the database - if err := d.database.PurgeGuildData(ctx, guildId, d.logger); err != nil { - d.logger.Error( - "Failed to purge guild data from database", - zap.Uint64("guild", guildId), - zap.Error(err), - ) - return false - } - - return true - case archiverclient.StatusFailed: - d.logger.Error( - "logarchiver purge failed", - zap.Uint64("guild", guildId), - ) - - if len(status.Errors) > 0 { - for objectName, errStr := range status.Errors { - d.logger.Error( - "logarchiver error detail", - zap.Uint64("guild", guildId), - zap.String("object", objectName), - zap.String("error", errStr), - ) - } - } - - return false - case archiverclient.StatusTimeout: - d.logger.Error( - "logarchiver purge timed out after inactivity", - zap.Uint64("guild", guildId), - ) - return false - case archiverclient.StatusInProgress: - d.logger.Debug( - "Purge in progress...", - zap.Uint64("guild", guildId), - zap.Int("status_check_attempt", attempt), - zap.Strings("objects", status.Removed), - zap.Strings("failed", status.Failed), - ) - - attempt++ - - time.Sleep(time.Second * time.Duration(math.Max(10, float64(attempt)))) - default: - d.logger.Error( - "logarchiver returned unexpected status", - zap.Uint64("guild", guildId), - zap.String("status", string(status.Status)), - ) - return false + case archiverclient.StatusComplete: + d.logger.Info( + "logarchiver purge completed successfully", + zap.Uint64("guild", guildId), + ) + + return nil + + case archiverclient.StatusFailed: + return fmt.Errorf("logarchiver purge failed%s", formatPurgeErrors(status)) + + case archiverclient.StatusTimeout: + return errors.New("logarchiver purge timed out after inactivity") + + case archiverclient.StatusInProgress: + d.logger.Debug( + "Purge in progress...", + zap.Uint64("guild", guildId), + zap.Int("status_check_attempt", attempt), + zap.Strings("objects", status.Removed), + zap.Strings("failed", status.Failed), + ) + + time.Sleep(pollInterval(attempt)) + + default: + return fmt.Errorf("logarchiver returned unexpected status %q", status.Status) + } + } +} + +func pollInterval(attempt int) time.Duration { + interval := minPollInterval + time.Duration(attempt)*time.Second + if interval > maxPollInterval { + return maxPollInterval + } + + return interval +} + +func formatPurgeErrors(status archiverclient.PurgeStatus) string { + if len(status.Errors) == 0 { + if len(status.Failed) == 0 { + return " (logarchiver reported no detail; check its own logs)" } + + return fmt.Sprintf(" (failed objects: %s)", strings.Join(status.Failed, ", ")) } + + objects := make([]string, 0, len(status.Errors)) + for object := range status.Errors { + objects = append(objects, object) + } + sort.Strings(objects) + + details := make([]string, 0, len(objects)) + for _, object := range objects { + details = append(details, fmt.Sprintf("%s: %s", object, status.Errors[object])) + } + + return fmt.Sprintf(" (%s)", strings.Join(details, "; ")) } -func (d *Daemon) isBotInServer(ctx context.Context ,guildId uint64) (bool, error) { +func (d *Daemon) isBotInServer(ctx context.Context, guildId uint64) (bool, error) { botId, ok, err := d.database.WhitelabelGuilds.GetBotByGuild(ctx, guildId) if err != nil { return false, err @@ -182,7 +201,7 @@ func (d *Daemon) isBotInServer(ctx context.Context ,guildId uint64) (bool, error token = d.config.MainBotToken } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(ctx, time.Second*5) defer cancel() if _, err := rest.GetGuild(ctx, token, nil, guildId); err != nil { From bbbed465e1fc5a3d2d999d3c5b20ddf7917e84f0 Mon Sep 17 00:00:00 2001 From: Tobias <53872542+biast12@users.noreply.github.com> Date: Sat, 5 Sep 2026 12:40:52 +0200 Subject: [PATCH 2/2] Uncomment replace directive for archiverclient --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2d87655..58838bf 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/TicketsBot-cloud/cleanupdaemon go 1.24.0 -//replace github.com/TicketsBot-cloud/archiverclient => ../archiverclient +replace github.com/TicketsBot-cloud/archiverclient => ../archiverclient replace github.com/TicketsBot-cloud/common => ../common