Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/discord-bot/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"test": "vp test run"
},
"dependencies": {
"@azure/storage-blob": "^12.33.0",
"@effect/platform-node": "catalog:",
"@microsoft/teams.apps": "2.0.14",
"@t3tools/client-runtime": "workspace:*",
Expand Down
4 changes: 4 additions & 0 deletions apps/discord-bot/src/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ const baseConfig = {
teamsPort: 3978,
teamsMessagingEndpoint: "/api/messages" as const,
teamsDefaultProjectShortName: undefined,
azureStorageConnectionString: undefined,
azureStorageAccountName: undefined,
azureStorageAccountKey: undefined,
azureStorageContainer: "discord-bot-attachments",
} satisfies DiscordBotConfig;

describe("preferredModelSelection", () => {
Expand Down
30 changes: 30 additions & 0 deletions apps/discord-bot/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,15 @@ export interface DiscordBotConfig {
readonly teamsMessagingEndpoint: `/${string}`;
/** Fallback alias for personal/group chats that do not map to a configured channel. */
readonly teamsDefaultProjectShortName: string | undefined;
/**
* Optional Azure Blob offload for Discord attachments over ~10MB.
* Container must be private (not listable). Bot issues 3-day read SAS links.
* Prefer connection string; account name+key is also accepted.
*/
readonly azureStorageConnectionString: string | undefined;
readonly azureStorageAccountName: string | undefined;
readonly azureStorageAccountKey: string | undefined;
readonly azureStorageContainer: string;
}

const RuntimeModeConfig = Schema.Literals([
Expand Down Expand Up @@ -220,6 +229,23 @@ export const DiscordBotConfig: Effect.Effect<DiscordBotConfig, Config.ConfigErro
const teamsDefaultProjectShortName = yield* Config.string(
"TEAMS_DEFAULT_PROJECT_SHORT_NAME",
).pipe(Config.option, Config.map(Option.getOrUndefined));
const azureStorageConnectionString = yield* Config.redacted(
"AZURE_STORAGE_CONNECTION_STRING",
).pipe(
Config.option,
Config.map((value) => (Option.isSome(value) ? Redacted.value(value.value) : undefined)),
);
const azureStorageAccountName = yield* Config.string("AZURE_STORAGE_ACCOUNT_NAME").pipe(
Config.option,
Config.map(Option.getOrUndefined),
);
const azureStorageAccountKey = yield* Config.redacted("AZURE_STORAGE_ACCOUNT_KEY").pipe(
Config.option,
Config.map((value) => (Option.isSome(value) ? Redacted.value(value.value) : undefined)),
);
const azureStorageContainer = yield* Config.string("AZURE_STORAGE_CONTAINER").pipe(
Config.withDefault("discord-bot-attachments"),
);

return {
discordToken,
Expand Down Expand Up @@ -255,6 +281,10 @@ export const DiscordBotConfig: Effect.Effect<DiscordBotConfig, Config.ConfigErro
teamsPort,
teamsMessagingEndpoint,
teamsDefaultProjectShortName,
azureStorageConnectionString,
azureStorageAccountName,
azureStorageAccountKey,
azureStorageContainer,
} satisfies DiscordBotConfig;
},
);
Expand Down
13 changes: 13 additions & 0 deletions apps/discord-bot/src/features/ResponseBridge.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1343,6 +1343,19 @@ describe("rewriteMarkdownLocalFileLinksForDiscord", () => {
expect(rewritten).toBe("report.csv (attached below)");
});

it("rewrites oversized local files to temporary Azure download links", () => {
const rewritten = rewriteMarkdownLocalFileLinksForDiscord({
text: "[big.mp4](/tmp/big.mp4)",
githubUrlsBySrc: new Map(),
attachedFileNames: new Set(),
oversizedByName: new Set(["big.mp4"]),
externalUrlsByName: new Map([
["big.mp4", "https://example.blob.core.windows.net/c/big?sv=1&sig=abc"],
]),
});
expect(rewritten).toBe("[big.mp4](https://example.blob.core.windows.net/c/big?sv=1&sig=abc)");
});

it("keeps attachable documents as attachments even when source refs become links", () => {
const rewritten = rewriteMarkdownLocalFileLinksForDiscord({
text: [
Expand Down
193 changes: 180 additions & 13 deletions apps/discord-bot/src/features/ResponseBridge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,15 @@ import {
streamHistoryHasAdditionalContent,
unpostedAttachments,
} from "../presentation/attachments.ts";
import {
AzureBlobUploadError,
DISCORD_CONSERVATIVE_UPLOAD_LIMIT_BYTES,
formatOversizedAttachmentNote,
isAzureBlobUploadConfigured,
uploadOversizedFilesToAzureBlob,
type AzureBlobUploadConfig,
type UploadedAzureBlobLink,
} from "../presentation/azureBlobUpload.ts";
import { buildOmegentThreadMessageUrl } from "../presentation/discordPrAttribution.ts";
import {
createMessageWithAttachments,
Expand Down Expand Up @@ -107,7 +116,6 @@ import { upsertThreadInfoPin } from "./ThreadInfoPin.ts";

const DISCORD_LIMIT = 2000;
const STREAM_CHUNK_LIMIT = inProgressChunkLimit(DISCORD_LIMIT);
const DISCORD_CONSERVATIVE_UPLOAD_LIMIT_BYTES = 10_000_000;

interface BridgeState {
/** T3 orchestration turn currently tracked by this bridge. */
Expand Down Expand Up @@ -2017,6 +2025,7 @@ function formatMarkdownLocalFileRefForDiscord(input: {
readonly githubUrlsBySrc: ReadonlyMap<string, string>;
readonly attachedFileNames?: ReadonlySet<string> | undefined;
readonly oversizedByName?: ReadonlySet<string> | undefined;
readonly externalUrlsByName?: ReadonlyMap<string, string> | undefined;
}): string {
const display =
input.ref.label.trim() !== "" ? input.ref.label : fileNameForLocalFileRef(input.ref);
Expand All @@ -2026,13 +2035,17 @@ function formatMarkdownLocalFileRefForDiscord(input: {
}

const uploadName = fileNameForLocalFileRef(input.ref);
const externalUrl = input.externalUrlsByName?.get(uploadName);
if (externalUrl) {
return `[${display}](${externalUrl})`;
}
if (input.oversizedByName?.has(uploadName)) {
return `${display} (too large to attach in Discord)`;
}
if (input.attachedFileNames?.has(uploadName)) {
return `${display} (attached below)`;
}
if (input.attachedFileNames || input.oversizedByName) {
if (input.attachedFileNames || input.oversizedByName || input.externalUrlsByName) {
return `${display} (attachment unavailable)`;
}
return input.ref.match;
Expand All @@ -2043,13 +2056,15 @@ export function rewriteMarkdownLocalFileLinksForDiscord(input: {
readonly githubUrlsBySrc: ReadonlyMap<string, string>;
readonly attachedFileNames?: ReadonlySet<string> | undefined;
readonly oversizedByName?: ReadonlySet<string> | undefined;
readonly externalUrlsByName?: ReadonlyMap<string, string> | undefined;
}): string {
return replaceMarkdownLocalFileLinks(input.text, (ref) =>
formatMarkdownLocalFileRefForDiscord({
ref,
githubUrlsBySrc: input.githubUrlsBySrc,
attachedFileNames: input.attachedFileNames,
oversizedByName: input.oversizedByName,
externalUrlsByName: input.externalUrlsByName,
}),
);
}
Expand Down Expand Up @@ -3082,6 +3097,131 @@ export const runBridge = (
};
};

const azureBlobConfigFromBot = (botConfig: {
readonly azureStorageConnectionString: string | undefined;
readonly azureStorageAccountName: string | undefined;
readonly azureStorageAccountKey: string | undefined;
readonly azureStorageContainer: string;
}): AzureBlobUploadConfig => ({
connectionString: botConfig.azureStorageConnectionString,
accountName: botConfig.azureStorageAccountName,
accountKey: botConfig.azureStorageAccountKey,
containerName: botConfig.azureStorageContainer,
});

/**
* Offload files over Discord's ~10MB limit to a private Azure container and
* return 3-day read-only SAS links. No-ops gracefully when Azure is unset.
*/
const offloadOversizedFiles = (oversized: ReadonlyArray<DiscordUploadFile>) =>
Effect.gen(function* () {
if (oversized.length === 0) {
return {
uploaded: [] as ReadonlyArray<UploadedAzureBlobLink>,
failed: [] as ReadonlyArray<{ readonly fileName: string; readonly sizeBytes: number }>,
unconfigured: [] as ReadonlyArray<{
readonly fileName: string;
readonly sizeBytes: number;
}>,
externalUrlsByName: new Map<string, string>(),
};
}

const botConfig = yield* DiscordBotConfig;
const azureConfig = azureBlobConfigFromBot(botConfig);
if (!isAzureBlobUploadConfigured(azureConfig)) {
yield* Effect.logWarning(
"Oversized Discord attachments skipped (Azure blob upload not configured)",
{
files: oversized.map((file) => ({
name: file.name,
bytes: file.data.byteLength,
})),
},
);
return {
uploaded: [] as ReadonlyArray<UploadedAzureBlobLink>,
failed: [] as ReadonlyArray<{ readonly fileName: string; readonly sizeBytes: number }>,
unconfigured: oversized.map((file) => ({
fileName: file.name,
sizeBytes: file.data.byteLength,
})),
externalUrlsByName: new Map<string, string>(),
};
}

yield* Effect.logInfo("Uploading oversized Discord attachments to Azure Blob", {
container: azureConfig.containerName,
files: oversized.map((file) => ({
name: file.name,
bytes: file.data.byteLength,
})),
});

const result = yield* Effect.tryPromise({
try: () =>
uploadOversizedFilesToAzureBlob({
config: azureConfig,
files: oversized,
}),
catch: (cause) =>
cause instanceof AzureBlobUploadError
? cause
: new AzureBlobUploadError(
cause instanceof Error ? cause.message : String(cause),
cause,
),
}).pipe(
Effect.catch((cause) =>
Effect.logError("Azure blob bulk upload failed").pipe(
Effect.andThen(Effect.logError(cause)),
Effect.as({
uploaded: [] as ReadonlyArray<UploadedAzureBlobLink>,
failed: oversized.map((file) => ({
fileName: file.name,
error: cause instanceof Error ? cause.message : String(cause),
})),
}),
),
),
);

for (const failure of result.failed) {
yield* Effect.logWarning("Azure blob upload failed for attachment", failure);
}
if (result.uploaded.length > 0) {
yield* Effect.logInfo("Azure blob upload complete", {
uploaded: result.uploaded.map((entry) => ({
fileName: entry.fileName,
blobName: entry.blobName,
expiresAt: entry.expiresAt.toISOString(),
bytes: entry.sizeBytes,
})),
});
}

const failedWithSize = result.failed.map((failure) => {
const match = oversized.find((file) => file.name === failure.fileName);
return {
fileName: failure.fileName,
sizeBytes: match?.data.byteLength ?? 0,
};
});
const externalUrlsByName = new Map(
result.uploaded.map((entry) => [entry.fileName, entry.url] as const),
);

return {
uploaded: result.uploaded,
failed: failedWithSize,
unconfigured: [] as ReadonlyArray<{
readonly fileName: string;
readonly sizeBytes: number;
}>,
externalUrlsByName,
};
});

/**
* Create a Discord message. Binary files use native multipart FormData + fetch
* (HTTP/1.1). dfx `withFiles` goes through Effect/Undici HTTP2 and dies with
Expand Down Expand Up @@ -3679,7 +3819,26 @@ export const runBridge = (
);
const files = [...imageFiles, ...mdLoaded.files, ...linkedFilesLoaded.files];
if (files.length === 0) return;
const created = yield* createMessageWithFiles("", files);
const { batches: lateBatches, oversized: lateOversized } =
splitFilesForDiscordUpload(files);
const lateOffload = yield* offloadOversizedFiles(lateOversized);
const finalIds: string[] = [];
for (const batch of lateBatches) {
const created = yield* createMessageWithFiles("", batch);
finalIds.push(created.id);
}
if (lateOversized.length > 0) {
const note = formatOversizedAttachmentNote({
uploaded: lateOffload.uploaded,
failed: lateOffload.failed,
unconfigured: lateOffload.unconfigured,
});
if (note !== null) {
const created = yield* rest.createMessage(input.discordChannelId, { content: note });
finalIds.push(created.id);
}
}
if (finalIds.length === 0) return;
const postedFromFiles = pendingImages
.slice(0, imageFiles.length)
.map((entry) => entry.id);
Expand All @@ -3692,7 +3851,7 @@ export const runBridge = (
postedMarkdownFileSrcs: [
...new Set([...current.postedMarkdownFileSrcs, ...linkedFilesLoaded.loadedSrcs]),
],
finalDiscordMessageIds: [...current.finalDiscordMessageIds, created.id],
finalDiscordMessageIds: [...current.finalDiscordMessageIds, ...finalIds],
}));
return;
}
Expand Down Expand Up @@ -3771,9 +3930,15 @@ export const runBridge = (
const postedFromFiles = pendingImages.slice(0, imageFiles.length).map((entry) => entry.id);

// Split once for local-file rewrite notes (no table .txt attachments).
// Oversized files go to private Azure Blob with 3-day SAS download links.
const { batches: uploadBatches, oversized: oversizedFiles } =
splitFilesForDiscordUpload(files);
const oversizedByName = new Set(oversizedFiles.map((file) => file.name));
const oversizedOffload = yield* offloadOversizedFiles(oversizedFiles);
const oversizedByName = new Set(
oversizedFiles
.map((file) => file.name)
.filter((name) => !oversizedOffload.externalUrlsByName.has(name)),
);
const attachedFileNames = new Set(
uploadBatches.flatMap((batch) => batch.map((file) => file.name)),
);
Expand All @@ -3789,6 +3954,7 @@ export const runBridge = (
githubUrlsBySrc,
attachedFileNames,
oversizedByName,
externalUrlsByName: oversizedOffload.externalUrlsByName,
})
.replace(/_\(attachment will attach when done\)_/giu, "")
.replace(/_\(\d+ attachments will attach when done\)_/giu, "")
Expand Down Expand Up @@ -3932,14 +4098,15 @@ export const runBridge = (
}

if (oversizedFiles.length > 0) {
const note = [
"**Some files could not be attached due to Discord upload limits:**",
...oversizedFiles.map(
(file) => `- \`${file.name}\` (${Math.ceil(file.data.byteLength / 1_000_000)} MB)`,
),
].join("\n");
const created = yield* rest.createMessage(input.discordChannelId, { content: note });
ids.push(created.id);
const note = formatOversizedAttachmentNote({
uploaded: oversizedOffload.uploaded,
failed: oversizedOffload.failed,
unconfigured: oversizedOffload.unconfigured,
});
if (note !== null) {
const created = yield* rest.createMessage(input.discordChannelId, { content: note });
ids.push(created.id);
}
}
return ids;
});
Expand Down
Loading
Loading