Latest-attempt filter, and the retry chains on the dashboard

SW.Bitween.Api/Resources/Dashboard/RetrySummary.cs

@@ -0,0 +1,169 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using SW.Bitween.Domain;
+using SW.PrimitiveTypes;
+
+namespace SW.Bitween.Resources.Dashboard;
+
+/// <summary>
+/// The two things a retry chain can say that a count of failures cannot: which pieces of work
+/// have been retried over and over and are still failing, and whether retrying is achieving
+/// anything at all.
+/// </summary>
+/// <remarks>
+/// Both in one handler because both belong to the same panel — a repeat offender means little
+/// without knowing whether retries generally work here. Nine attempts against the same connection
+/// error is a configuration problem someone has to fix; nine attempts where most retries do
+/// eventually succeed is bad luck.
+/// </remarks>
+[HandlerName("retrysummary")]
+public class RetrySummary : IQueryHandler<object>
+{
+    /// <summary>
+    /// How many failures the chain lengths are measured over, newest first. The walk below costs a
+    /// query per level for the whole set at once, so this bounds the work without narrowing the
+    /// answer in practice: a chain long enough to be listed is recent, because something has been
+    /// retrying it.
+    /// </summary>
+    private const int Candidates = 500;
+
+    /// <summary>Enough to show a pattern; the full list is one link away.</summary>
+    private const int Listed = 5;
+
+    /// <summary>Guards the walk against a cycle in the data, as elsewhere.</summary>
+    private const int MaxDepth = 100;
+
+    private const int WindowDays = 7;
+
+    private readonly BitweenDbContext _dbContext;
+    private readonly RequestContext _requestContext;
+
+    public RetrySummary(BitweenDbContext dbContext, RequestContext requestContext)
+    {
+        _dbContext = dbContext;
+        _requestContext = requestContext;
+    }
+
+    public async Task<object> Handle()
+    {
+        await _requestContext.EnsurePermission(_dbContext, Model.Permissions.Dashboard.View);
+
+        var since = DateTime.UtcNow.AddDays(-WindowDays);
+
+        // Did retrying help? Every attempt that was itself a retry, and how it ended. A retry with
+        // no result yet is counted in neither, so the two never add up to more than what finished.
+        var retries = await (
+            from xchange in _dbContext.Set<Xchange>()
+            join result in _dbContext.Set<XchangeResult>() on xchange.Id equals result.Id
+            where xchange.RetryFor != null && xchange.StartedOn >= since
+            group result by result.Success && !result.ResponseBad into worked
+            select new { Worked = worked.Key, Count = worked.Count() }).AsNoTracking().ToListAsync();
+
+        var succeeded = retries.Where(r => r.Worked).Sum(r => r.Count);
+        var finished = retries.Sum(r => r.Count);
+
+        // Chains that keep failing: a failure nothing has been retried from — so it is where that
+        // chain has got to — which is itself a retry, so the chain is at least two attempts long.
+        var candidates = await (
+            from xchange in _dbContext.Set<Xchange>()
+            join result in _dbContext.Set<XchangeResult>() on xchange.Id equals result.Id
+            where xchange.RetryFor != null
+                  && !result.Success
+                  && !_dbContext.Set<Xchange>().Any(child => child.RetryFor == xchange.Id)
+            orderby xchange.StartedOn descending
+            select new
+            {
+                xchange.Id,
+                xchange.RetryFor,
+                xchange.SubscriptionId,
+                xchange.DocumentId,
+                xchange.StartedOn,
+                result.Exception
+            }).AsNoTracking().Take(Candidates).ToListAsync();
+
+        var attempts = await CountAttempts(candidates.ToDictionary(c => c.Id, c => c.RetryFor));
+
+        var worst = candidates
+            .OrderByDescending(c => attempts[c.Id])
+            .ThenByDescending(c => c.StartedOn)
+            .Take(Listed)
+            .ToList();
+
+        // Names for the few that are actually listed, rather than for all five hundred.
+        var subscriptionIds = worst.Where(w => w.SubscriptionId != null).Select(w => w.SubscriptionId!.Value).ToList();
+        var subscriptionNames = await _dbContext.Set<Subscription>().AsNoTracking()
+            .Where(s => subscriptionIds.Contains(s.Id))
+            .ToDictionaryAsync(s => s.Id, s => s.Name);
+
+        var documentIds = worst.Select(w => w.DocumentId).Distinct().ToList();
+        var documentNames = await _dbContext.Set<Document>().AsNoTracking()
+            .Where(d => documentIds.Contains(d.Id))
+            .ToDictionaryAsync(d => d.Id, d => d.Name);
+
+        return new
+        {
+            RetriesLast7Days = new { Finished = finished, Succeeded = succeeded },
+            FailingChains = worst.Select(w => new
+            {
+                w.Id,
+                Attempts = attempts[w.Id],
+                w.SubscriptionId,
+                SubscriptionName = w.SubscriptionId != null && subscriptionNames.ContainsKey(w.SubscriptionId.Value)
+                    ? subscriptionNames[w.SubscriptionId.Value]
+                    : null,
+                InformationTypeCode = documentNames.GetValueOrDefault(w.DocumentId),
+                w.StartedOn,
+                w.Exception
+            })
+        };
+    }
+
+    /// <summary>
+    /// How many attempts each chain has taken, counting the original as one.
+    /// </summary>
+    /// <remarks>
+    /// A level of every chain per query rather than a walk per chain — five hundred chains four
+    /// attempts deep is four queries, not two thousand. <c>XchangeResult.AttemptNumber</c> holds
+    /// this already, but only for failures a retry policy's group matched, so a chain retried by
+    /// hand has none and it cannot be read from there.
+    /// </remarks>
+    private async Task<Dictionary<string, int>> CountAttempts(Dictionary<string, string> parentOf)
+    {
+        // Every chain starts at two: the candidate is a retry, so something came before it.
+        var attempts = parentOf.Keys.ToDictionary(id => id, _ => 2);
+        var frontier = new Dictionary<string, string>(parentOf);
+
+        for (var depth = 0; depth < MaxDepth; depth++)
+        {
+            var ancestors = frontier.Values.Where(id => id != null).Distinct().ToList();
+            if (ancestors.Count == 0) break;
+
+            var next = await _dbContext.Set<Xchange>().AsNoTracking()
+                .Where(x => ancestors.Contains(x.Id))
+                .Select(x => new { x.Id, x.RetryFor })
+                .ToDictionaryAsync(x => x.Id, x => x.RetryFor);
+
+            var moved = false;
+            foreach (var id in frontier.Keys.ToList())
+            {
+                var at = frontier[id];
+                if (at == null || !next.TryGetValue(at, out var parent) || parent == null)
+                {
+                    frontier[id] = null;
+                    continue;
+                }
+
+                frontier[id] = parent;
+                attempts[id]++;
+                moved = true;
+            }
+
+            if (!moved) break;
+        }
+
+        return attempts;
+    }
+}

SW.Bitween.Api/Resources/Xchanges/XchangeFilters.cs

@@ -100,6 +100,21 @@ internal static IQueryable<XchangeRow> ApplySpecialFilters(this IQueryable<Xchan
             condition.Filters.Remove(statusFilter);
         }
 
+        // "Only the newest attempt of each chain." An exchange that has been retried is superseded
+        // by that retry, so a list of failures otherwise counts one piece of work as many rows —
+        // a chain retried nine times fills nine of them, none of which is the current state.
+        //
+        // NOT EXISTS rather than a left join for the same reason the still-running status uses it:
+        // Postgres estimates an anti-join, and RetryFor is indexed.
+        var latestFilters = condition.Filters.Where(f => f.Field == "LatestOnly").ToList();
+        foreach (var latestFilter in latestFilters)
+        {
+            if (latestFilter.Value?.ToString()?.ToLower() is "true" or "1")
+                query = query.Where(i => !dbContext.Set<Xchange>().Any(r => r.RetryFor == i.Id));
+
+            condition.Filters.Remove(latestFilter);
+        }
+
         var propertiesFilters = condition.Filters
             .Where(f => f.Field == "PromotedPropertiesRaw").ToList();
         foreach (var propertyFilter in propertiesFilters)

SW.Bitween.IntegrationTests/Tests/RetryChainTests.cs

@@ -383,6 +383,54 @@ await Assert.ThrowsAsync<SWValidationException>(() =>
         Assert.False(await db.Set<Xchange>().AnyAsync(x => x.RetryFor == xchange.Id));
     }
 
+    // ─── What the dashboard reads ─────────────────────────────────────────────
+
+    /// <summary>
+    /// The dashboard's retry panel: which chains have been retried repeatedly and are still
+    /// failing, and whether retrying is achieving anything.
+    /// </summary>
+    [Fact]
+    public async Task RetrySummary_ranks_the_chains_that_keep_failing_and_says_whether_retries_work()
+    {
+        await using var scope = _fixture.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<BitweenDbContext>();
+        var xs = scope.ServiceProvider.GetRequiredService<XchangeService>();
+        var ctx = scope.Superuser();
+
+        // A chain of four failed attempts.
+        var (_, deep) = await FailedXchange(db, xs, "Summary Deep Doc");
+        var second = await RetryOnce(db, xs, ctx, deep.Id);
+        var third = await RetryOnce(db, xs, ctx, second.Id);
+        var fourth = await RetryOnce(db, xs, ctx, third.Id);
+
+        // And one that was retried once, where the retry worked.
+        var (_, healed) = await FailedXchange(db, xs, "Summary Healed Doc");
+        await new Resources.Xchanges.Retry(db, ctx, xs).Handle(healed.Id, new XchangeRetry());
+        await db.SaveChangesAsync();
+        var healedRetry = await db.Set<Xchange>().FirstAsync(x => x.RetryFor == healed.Id);
+        db.Set<XchangeResult>().Add(new XchangeResult(healedRetry.Id, null, null));
+        await db.SaveChangesAsync();
+
+        var summary = await new Resources.Dashboard.RetrySummary(db, ctx).Handle();
+        var json = System.Text.Json.JsonSerializer.SerializeToElement(summary);
+
+        // The still-failing chain is listed, at its true length. The one that came good is not:
+        // its newest attempt succeeded, so there is nothing left to act on.
+        var chains = json.GetProperty("FailingChains").EnumerateArray().ToList();
+        var listed = chains.FirstOrDefault(c => c.GetProperty("Id").GetString() == fourth.Id);
+        Assert.Equal(4, listed.GetProperty("Attempts").GetInt32());
+        Assert.DoesNotContain(chains, c => c.GetProperty("Id").GetString() == healedRetry.Id);
+
+        // Superseded attempts are not listed either — only where each chain got to.
+        foreach (var stale in new[] { deep.Id, second.Id, third.Id })
+            Assert.DoesNotContain(chains, c => c.GetProperty("Id").GetString() == stale);
+
+        // And retries that worked are counted among the ones that finished.
+        var retries = json.GetProperty("RetriesLast7Days");
+        Assert.True(retries.GetProperty("Succeeded").GetInt32() >= 1);
+        Assert.True(retries.GetProperty("Finished").GetInt32() >= retries.GetProperty("Succeeded").GetInt32());
+    }
+
     // ─── Reading the chain back ───────────────────────────────────────────────
 
     [Fact]
@@ -410,6 +458,47 @@ public async Task RetryTree_returns_the_whole_chain_whichever_attempt_is_asked_a
         }
     }
 
+    /// <summary>
+    /// A chain is one piece of work, however many attempts it took. Asking for the newest attempt
+    /// only is what turns a list of failures into a list of things still to deal with — and, used
+    /// with a bulk retry, means the selection is already the attempts that can be retried.
+    /// </summary>
+    [Fact]
+    public async Task Exchange_search_can_return_only_the_newest_attempt_of_each_chain()
+    {
+        await using var scope = _fixture.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<BitweenDbContext>();
+        var xs = scope.ServiceProvider.GetRequiredService<XchangeService>();
+        var ctx = scope.Superuser();
+
+        var (sub, first) = await FailedXchange(db, xs, "Latest Only Doc");
+        var second = await RetryOnce(db, xs, ctx, first.Id);
+        var third = await RetryOnce(db, xs, ctx, second.Id);
+
+        // A second, unrelated failure on the same subscription that was never retried.
+        var alone = await xs.CreateXchange(sub, new XchangeFile("{}"));
+        await db.SaveChangesAsync();
+        db.Set<XchangeResult>().Add(new XchangeResult(alone.Id, null, null, exception: "boom"));
+        await db.SaveChangesAsync();
+
+        var search = new Resources.Xchanges.Search(db, xs, ctx);
+
+        var all = (SearchyResponse<XchangeRow>)await search.Handle(
+            new SearchyRequest($"filter=SubscriptionId:1:{sub.Id}") { PageSize = 50 });
+        Assert.Equal(4, all.Result.Count());
+
+        var latest = (SearchyResponse<XchangeRow>)await search.Handle(
+            new SearchyRequest($"filter=SubscriptionId:1:{sub.Id}&filter=LatestOnly:1:true") { PageSize = 50 });
+
+        // The chain collapses to its own end, and the standalone failure is its own end too.
+        var ids = latest.Result.Select(r => r.Id).ToList();
+        Assert.Equal(2, ids.Count);
+        Assert.Contains(third.Id, ids);
+        Assert.Contains(alone.Id, ids);
+        Assert.Equal(2, latest.TotalCount);
+        Assert.All(latest.Result, r => Assert.False(r.HasRetry));
+    }
+
     [Fact]
     public async Task Exchange_search_reports_which_rows_have_been_retried()
     {

SW.Bitween.Web/ClientApp/e2e/dashboard.spec.ts

@@ -16,4 +16,28 @@ test("dashboard loads with real aggregated data", async ({ page }) => {
   await expect(page.getByText("Success rate (7 days)")).toBeVisible();
   await expect(page.getByText("undefined")).toHaveCount(0);
   await expect(page.getByText("NaN")).toHaveCount(0);
+
+  // "Failures to act on" counts problems rather than attempts, and has to agree with the list it
+  // opens — a tile whose number changes when you click it is worse than no tile.
+  // Anchored: the "Chains that keep failing" panel ends with an "All failures to act on" link,
+  // which an unanchored match picks up as well.
+  const tile = page.getByRole("link").filter({ hasText: /^Failures to act on/ });
+  await expect(tile).toBeVisible();
+  const count = (await tile.innerText()).match(/([\d,]+)/)?.[1];
+  expect(count).toBeTruthy();
+
+  await tile.click();
+  await expect(page).toHaveURL(/status=failed&latest=1/);
+  await expect(page.getByText(`of ${count}`)).toBeVisible({ timeout: 15000 });
+
+  // The panel that says which of those failures are not getting better, however often they are
+  // retried — the number on each row is how many attempts that one piece of work has taken.
+  await page.goBack();
+  const chains = page.getByText("Chains that keep failing");
+  await expect(chains).toBeVisible();
+  const rows = page.getByRole("link").filter({ hasText: /^\d+ attempts$/ });
+  if ((await rows.count()) > 0) {
+    await rows.first().click();
+    await expect(page).toHaveURL(/\/exchanges\?ids=/);
+  }
 });

SW.Bitween.Web/ClientApp/e2e/exchanges.spec.ts

@@ -170,3 +170,40 @@ test("select all matching covers the whole filter, and the confirm says what wil
   await dialog.locator("button", { hasText: /^(Cancel|Close)$/ }).click();
   await expect(dialog).toHaveCount(0);
 });
+
+/**
+ * A chain is one piece of work however many attempts it took, so the list needs to be able to
+ * show the newest attempt of each — otherwise a chain retried nine times fills nine rows, none
+ * of which is the current state of anything.
+ */
+test("the list can show only the newest attempt of each chain", async ({ page }) => {
+  await page.goto("exchanges?status=failed");
+  await expect(page.getByRole("row").nth(1)).toBeVisible({ timeout: 15000 });
+  await page.getByLabel("Refresh interval").selectOption("0");
+
+  const total = async () =>
+    Number(
+      (await page.locator("text=/Showing .* of [\\d,]+/").first().innerText())
+        .match(/of ([\d,]+)/)![1]
+        .replace(/,/g, ""),
+    );
+
+  const attempts = await total();
+  const pill = page.getByRole("button", { name: "Latest attempt only" });
+  await expect(pill).toHaveAttribute("aria-pressed", "false");
+
+  await pill.click();
+  await expect(page).toHaveURL(/latest=1/);
+  await expect(pill).toHaveAttribute("aria-pressed", "true");
+
+  // Never more than the attempts, and fewer as soon as anything has been retried.
+  const problems = await total();
+  expect(problems).toBeLessThanOrEqual(attempts);
+
+  // And it survives a reload, since it lives in the URL like every other filter.
+  await page.reload();
+  await expect(page.getByRole("button", { name: "Latest attempt only" })).toHaveAttribute(
+    "aria-pressed",
+    "true",
+  );
+});