From 509927760a472b37c5bbf366b2a807d23bd4999c Mon Sep 17 00:00:00 2001 From: Thant Sin Toe Date: Fri, 14 Aug 2026 21:31:54 +0700 Subject: [PATCH 1/7] initial group message txs implementation --- src/@types/accountTypeGuards.ts | 8 + src/@types/index.ts | 196 +++++++- src/@types/transactionSchemas.ts | 161 +++++++ src/accounts/groupAccount.ts | 153 ++++++ src/accounts/index.ts | 13 + src/api/group/group.ts | 231 +++++++++ src/api/group/index.ts | 11 + src/api/index.ts | 11 + src/config/index.ts | 27 ++ src/index.ts | 15 + src/transactions/group_commit.ts | 468 +++++++++++++++++++ src/transactions/group_create.ts | 239 ++++++++++ src/transactions/group_keypackage_publish.ts | 223 +++++++++ src/transactions/group_leave.ts | 221 +++++++++ src/transactions/group_message.ts | 269 +++++++++++ src/transactions/index.ts | 10 + src/utils/index.ts | 11 + 17 files changed, 2266 insertions(+), 1 deletion(-) create mode 100644 src/accounts/groupAccount.ts create mode 100644 src/api/group/group.ts create mode 100644 src/api/group/index.ts create mode 100644 src/transactions/group_commit.ts create mode 100644 src/transactions/group_create.ts create mode 100644 src/transactions/group_keypackage_publish.ts create mode 100644 src/transactions/group_leave.ts create mode 100644 src/transactions/group_message.ts diff --git a/src/@types/accountTypeGuards.ts b/src/@types/accountTypeGuards.ts index 45263551..1fa6f9c1 100644 --- a/src/@types/accountTypeGuards.ts +++ b/src/@types/accountTypeGuards.ts @@ -11,6 +11,7 @@ import { DevAccount, DaoProposalsMeta, DaoProposalAccount, + GroupAccount, } from '.' /** @@ -27,6 +28,13 @@ export function isChatAccount(account: unknown): account is ChatAccount { return !!account && typeof account === 'object' && 'type' in account && account.type === 'ChatAccount' } +/** + * Type guard to check if an account is a GroupAccount + */ +export function isGroupAccount(account: unknown): account is GroupAccount { + return !!account && typeof account === 'object' && 'type' in account && account.type === 'GroupAccount' +} + /** * Type guard to check if an account is an AliasAccount */ diff --git a/src/@types/index.ts b/src/@types/index.ts index 99212ae7..8acbdaf1 100644 --- a/src/@types/index.ts +++ b/src/@types/index.ts @@ -154,6 +154,12 @@ export enum TXTypes { dao_claim_reward = 'dao_claim_reward', dao_burn_reward = 'dao_burn_reward', dao_cancel = 'dao_cancel', + // MLS (RFC 9420) group chat + group_create = 'group_create', + group_keypackage_publish = 'group_keypackage_publish', + group_message = 'group_message', + group_commit = 'group_commit', + group_leave = 'group_leave', } export interface BaseLiberdusTx { @@ -273,6 +279,123 @@ export namespace Tx { export type MessageRecord = Message + /** + * ------------------------- MLS GROUP CHAT (RFC 9420) ------------------------- + * + * Group messaging uses its own transaction family because `Message` is + * irreducibly two-party: its chatId is hash(from,to) and its toll state is + * [sender, receiver] pairs. + * + * All group transactions target a single GroupAccount, so Shardus orders them + * deterministically by timestamp. That ordering is what makes MLS — which + * requires every member to apply the same commits in the same sequence — + * workable on-chain, and `GroupAccount.epoch` fences concurrent commits. + * + * Every blob below is opaque base64 produced by the client's MLS stack. The + * network never sees plaintext, group name, or who said what inside a message. + */ + + /** ML-KEM-1024 sealed post-quantum PSK, addressed to one joining member. */ + export interface GroupSealedPsk { + cipherText: string // ML-KEM-1024 ciphertext, b64 + nonce: string // AEAD nonce, b64 + ct: string // wrapped 32-byte group PSK, b64 + } + + /** Everything a newly added member needs to join. */ + export interface GroupWelcomeEnvelope { + welcome: string // b64 MLS Welcome + ratchetTree: string // b64 ratchet tree + sealedPsk: GroupSealedPsk + pskId: string // b64 + pskNonce: string // b64 + epoch: number + timestamp: number + } + + export interface GroupCreate extends BaseLiberdusTx { + from: string + groupId: string // must equal hash(from + groupNonce) + groupNonce: string // 32-byte hex, client-chosen + mlsGroupId: string // hex of the MLS group_id + cipherSuite: number // pinned; members must agree + meta: string // client-encrypted {name, avatar, ...} + maxMembers: number + fee: bigint + } + + /** Publish single-use MLS KeyPackages so others can add this account. */ + export interface GroupKeyPackagePublish extends BaseLiberdusTx { + from: string + keyPackages: string[] // b64, each consumed on use + lastResortKeyPackage?: string // b64, reusable fallback when the pool empties + cipherSuite: number + fee: bigint + } + + /** An application message: one MLS PrivateMessage. The hot path. */ + export interface GroupMessage extends BaseLiberdusTx { + from: string + groupId: string + epoch: number // recorded, NOT enforced (see group_message.ts) + message: string // b64 MLS PrivateMessage + fee: bigint + } + + export interface GroupMessageRecord { + type: TXTypes.group_message + txId: string + from: string + groupId: string + epoch: number + message: string + timestamp: number + sign: Signature + } + + /** A membership change: MLS proposals + commit, fenced on `epoch`. */ + export interface GroupCommit extends BaseLiberdusTx { + from: string + groupId: string + epoch: number // MUST equal GroupAccount.epoch + commit: string // b64 MLS commit + proposals: string[] // b64, applied before the commit + pskId: string // b64, external PSK referenced by the commit + pskNonce: string // b64 + welcomes: { address: string; envelope: GroupWelcomeEnvelope }[] + groupInfo: string // b64 GroupInfo w/ external_pub, for recovery + ratchetTree: string // b64 post-commit tree + addedMembers: string[] + removedMembers: string[] + consumedKeyPackages: { address: string; keyPackage: string }[] + meta?: string + fee: bigint + } + + /** Trimmed transcript record. Welcomes and trees live elsewhere to keep this small. */ + export interface GroupCommitRecord { + type: TXTypes.group_commit + txId: string + from: string + groupId: string + epoch: number // epoch BEFORE this commit applied + commit: string + proposals: string[] + pskId: string + pskNonce: string + addedMembers: string[] + removedMembers: string[] + timestamp: number + sign: Signature + } + + /** Self-removal. Not cryptographically effective until an admin commits a Remove. */ + export interface GroupLeave extends BaseLiberdusTx { + from: string + groupId: string + fee: bigint + } + export interface Read extends BaseLiberdusTx { from: string to: string @@ -609,6 +732,15 @@ export interface UserAccount { stake?: bigint remove_stake_request: number | null payments: DeveloperPayment[] + /** + * Pool of single-use MLS KeyPackages, b64. Popped by group_commit when this + * account is added to a group; the client tops the pool back up. + */ + mlsKeyPackages?: string[] + /** Reusable fallback used when the pool empties (RFC 9420 s10, weaker PCS). */ + mlsLastResortKeyPackage?: string + /** Ciphersuite the published KeyPackages were generated for. */ + mlsCipherSuite?: number } alias: string | null emailHash: string | null @@ -698,6 +830,66 @@ export interface ChatAccount { hasChats: boolean // if chat has messages } +/** + * One account per MLS group. Because every group transaction targets this single + * account, Shardus serializes them into a deterministic, consensus-agreed order — + * which is exactly the total-order broadcast MLS needs from a delivery service. + * + * The network stores only ciphertext and the minimum public metadata required to + * authorize writes (who is a member) and to fence concurrent commits (`epoch`). + */ +export interface GroupAccount { + id: string + type: string + hash: string + timestamp: number + + // --- MLS coordination ----------------------------------------------------- + mlsGroupId: string + cipherSuite: number + /** + * AUTHORITATIVE MLS epoch. A group_commit must name this exact value or it is + * rejected, so exactly one commit can land per epoch no matter how many + * members race. Incremented on every applied commit. + */ + epoch: number + + // --- membership (public metadata; required for authorization) ------------- + members: string[] + admins: string[] + memberSince: { [address: string]: { epoch: number; timestamp: number } } + + // --- transcript ----------------------------------------------------------- + /** Application messages. Safe to prune: losing them costs history only. */ + messages: Tx.GroupMessageRecord[] + /** + * Commits. MUST NOT be pruned on the ordinary retention timer — dropping a + * commit a member has not applied locks that member out of the group forever. + * Only prune below `checkpoint.epoch`, from which stragglers can re-join + * externally. + */ + handshakes: Tx.GroupCommitRecord[] + + /** Welcome + sealed PQ PSK awaiting collection by each newly added member. */ + pendingWelcomes: { [address: string]: Tx.GroupWelcomeEnvelope } + + // --- recovery ------------------------------------------------------------- + checkpoint: { + epoch: number + groupInfo: string // b64 GroupInfo with external_pub + ratchetTree: string // b64 + timestamp: number + } | null + + // --- misc ----------------------------------------------------------------- + meta: string // client-encrypted group name/avatar; opaque here + maxMembers: number + /** Per-member send throttle, address -> last group_message timestamp. */ + lastMessageAt: { [address: string]: number } + createdBy: string + hasChats: boolean +} + export interface AliasAccount { id: string type: string @@ -897,7 +1089,8 @@ export type Accounts = NetworkAccount & NodeAccount & ChatAccount & DaoProposalsMeta & - DaoProposalAccount + DaoProposalAccount & + GroupAccount export type AccountVariant = | NetworkAccount @@ -912,6 +1105,7 @@ export type AccountVariant = | DevAccount | DaoProposalsMeta | DaoProposalAccount + | GroupAccount /** * ---------------------- NETWORK DATA export interfaceS ---------------------- diff --git a/src/@types/transactionSchemas.ts b/src/@types/transactionSchemas.ts index fc8c7747..4fb00832 100644 --- a/src/@types/transactionSchemas.ts +++ b/src/@types/transactionSchemas.ts @@ -261,6 +261,162 @@ export const schemaMessageTX = { additionalProperties: false, } +/** + * ------------------------- MLS GROUP CHAT SCHEMAS ------------------------- + * + * NOTE: `baseTxProperties` omits `networkId` and `fee`, even though every + * transaction carries them. Combined with `additionalProperties: false` that + * makes several existing schemas (schemaMessageTX among them) reject real + * traffic — they only pass today because LiberdusFlags.enableAJVValidation is + * false. The group schemas below declare both explicitly so they are correct + * when that flag is eventually turned on. + */ +const groupBaseProperties = { + ...baseTxProperties, + networkId: { type: 'string' }, + fee: { isBigInt: true }, +} +const groupBaseRequired = [...baseTxRequired, 'networkId', 'fee'] + +const schemaGroupSealedPsk = { + type: 'object', + properties: { + cipherText: { type: 'string' }, + nonce: { type: 'string' }, + ct: { type: 'string' }, + }, + required: ['cipherText', 'nonce', 'ct'], + additionalProperties: false, +} + +const schemaGroupWelcomeEnvelope = { + type: 'object', + properties: { + welcome: { type: 'string' }, + ratchetTree: { type: 'string' }, + sealedPsk: schemaGroupSealedPsk, + pskId: { type: 'string' }, + pskNonce: { type: 'string' }, + epoch: { type: 'number', minimum: 0 }, + timestamp: { type: 'number', minimum: 0 }, + }, + required: ['welcome', 'ratchetTree', 'sealedPsk', 'pskId', 'pskNonce'], + additionalProperties: false, +} + +export const schemaGroupCreateTX = { + type: 'object', + properties: { + ...groupBaseProperties, + from: { type: 'string' }, + groupId: { type: 'string', minLength: 64, maxLength: 64 }, + groupNonce: { type: 'string', minLength: 64, maxLength: 64 }, + mlsGroupId: { type: 'string', minLength: 1 }, + cipherSuite: { type: 'number', minimum: 1 }, + meta: { type: 'string' }, + maxMembers: { type: 'number', minimum: 1 }, + }, + required: [...groupBaseRequired, 'from', 'groupId', 'groupNonce', 'mlsGroupId', 'cipherSuite', 'meta', 'maxMembers'], + additionalProperties: false, +} + +export const schemaGroupKeyPackagePublishTX = { + type: 'object', + properties: { + ...groupBaseProperties, + from: { type: 'string' }, + keyPackages: { type: 'array', items: { type: 'string' } }, + lastResortKeyPackage: { type: 'string' }, + cipherSuite: { type: 'number', minimum: 1 }, + }, + required: [...groupBaseRequired, 'from', 'keyPackages', 'cipherSuite'], + additionalProperties: false, +} + +export const schemaGroupMessageTX = { + type: 'object', + properties: { + ...groupBaseProperties, + from: { type: 'string' }, + groupId: { type: 'string', minLength: 64, maxLength: 64 }, + epoch: { type: 'number', minimum: 0 }, + message: { type: 'string', minLength: 1 }, + }, + required: [...groupBaseRequired, 'from', 'groupId', 'epoch', 'message'], + additionalProperties: false, +} + +export const schemaGroupCommitTX = { + type: 'object', + properties: { + ...groupBaseProperties, + from: { type: 'string' }, + groupId: { type: 'string', minLength: 64, maxLength: 64 }, + epoch: { type: 'number', minimum: 0 }, + commit: { type: 'string', minLength: 1 }, + proposals: { type: 'array', items: { type: 'string' } }, + pskId: { type: 'string' }, + pskNonce: { type: 'string' }, + welcomes: { + type: 'array', + items: { + type: 'object', + properties: { + address: { type: 'string' }, + envelope: schemaGroupWelcomeEnvelope, + }, + required: ['address', 'envelope'], + additionalProperties: false, + }, + }, + groupInfo: { type: 'string' }, + ratchetTree: { type: 'string' }, + addedMembers: { type: 'array', items: { type: 'string' } }, + removedMembers: { type: 'array', items: { type: 'string' } }, + consumedKeyPackages: { + type: 'array', + items: { + type: 'object', + properties: { + address: { type: 'string' }, + keyPackage: { type: 'string' }, + }, + required: ['address', 'keyPackage'], + additionalProperties: false, + }, + }, + meta: { type: 'string' }, + }, + required: [ + ...groupBaseRequired, + 'from', + 'groupId', + 'epoch', + 'commit', + 'proposals', + 'pskId', + 'pskNonce', + 'welcomes', + 'groupInfo', + 'ratchetTree', + 'addedMembers', + 'removedMembers', + 'consumedKeyPackages', + ], + additionalProperties: false, +} + +export const schemaGroupLeaveTX = { + type: 'object', + properties: { + ...groupBaseProperties, + from: { type: 'string' }, + groupId: { type: 'string', minLength: 64, maxLength: 64 }, + }, + required: [...groupBaseRequired, 'from', 'groupId'], + additionalProperties: false, +} + export const schemaReadTX = { type: 'object', properties: { @@ -935,6 +1091,11 @@ function addSchemas(): void { [TXTypes.issue]: schemaIssueTX, [TXTypes.dev_issue]: schemaDevIssueTX, [TXTypes.message]: schemaMessageTX, + [TXTypes.group_create]: schemaGroupCreateTX, + [TXTypes.group_keypackage_publish]: schemaGroupKeyPackagePublishTX, + [TXTypes.group_message]: schemaGroupMessageTX, + [TXTypes.group_commit]: schemaGroupCommitTX, + [TXTypes.group_leave]: schemaGroupLeaveTX, [TXTypes.read]: schemaReadTX, [TXTypes.reclaim_toll]: schemeReclaimTollTX, [TXTypes.update_chat_toll]: schemaUpdateChatTollTX, diff --git a/src/accounts/groupAccount.ts b/src/accounts/groupAccount.ts new file mode 100644 index 00000000..721315b0 --- /dev/null +++ b/src/accounts/groupAccount.ts @@ -0,0 +1,153 @@ +import * as crypto from '../crypto' +import { VectorBufferStream } from '@shardus/core' +import { Utils } from '@shardus/lib-types' +import { SerdeTypeIdent } from '.' +import { GroupAccount, Tx } from '../@types' + +/** + * Creates the account backing one MLS group. + * + * The founder is the sole member at epoch 0; everyone else arrives via + * group_commit, which is fenced on `epoch` so only one commit can land per + * epoch regardless of how many admins act at once. + */ +export const groupAccount = (accountId: string, tx: Tx.GroupCreate, timestamp: number): GroupAccount => { + // Ensure lowercase accountId + accountId = accountId.toLowerCase() + const from = tx.from.toLowerCase() + + const group: GroupAccount = { + id: accountId, + type: 'GroupAccount', + hash: '', + timestamp: 0, + + mlsGroupId: tx.mlsGroupId, + cipherSuite: tx.cipherSuite, + epoch: 0, + + members: [from], + admins: [from], + memberSince: { [from]: { epoch: 0, timestamp } }, + + messages: [], + handshakes: [], + pendingWelcomes: {}, + + checkpoint: null, + + meta: tx.meta, + maxMembers: tx.maxMembers, + lastMessageAt: {}, + createdBy: from, + hasChats: false, + } + + group.hash = crypto.hashObj(group) + return group +} + +/** + * The variable-length, deeply nested parts (transcript, welcomes, checkpoint) + * are written as JSON rather than field-by-field. They are already opaque + * base64 blobs whose shape is driven by the client's MLS stack, so a bespoke + * binary layout would buy little and would need a migration every time the + * envelope changes. + */ +export const serializeGroupAccount = (stream: VectorBufferStream, inp: GroupAccount, root = false): void => { + if (root) { + stream.writeUInt16(SerdeTypeIdent.GroupAccount) + } + + stream.writeString(inp.id) + stream.writeString(inp.type) + stream.writeString(inp.hash) + stream.writeBigUInt64(BigInt(inp.timestamp)) + + stream.writeString(inp.mlsGroupId) + stream.writeUInt16(inp.cipherSuite) + stream.writeUInt32(inp.epoch) + + stream.writeUInt32(inp.members.length) + for (const member of inp.members) { + stream.writeString(member) + } + + stream.writeUInt32(inp.admins.length) + for (const admin of inp.admins) { + stream.writeString(admin) + } + + stream.writeString(Utils.safeStringify(inp.memberSince)) + stream.writeString(Utils.safeStringify(inp.messages)) + stream.writeString(Utils.safeStringify(inp.handshakes)) + stream.writeString(Utils.safeStringify(inp.pendingWelcomes)) + stream.writeString(Utils.safeStringify(inp.checkpoint)) + stream.writeString(Utils.safeStringify(inp.lastMessageAt)) + + stream.writeString(inp.meta) + stream.writeUInt32(inp.maxMembers) + stream.writeString(inp.createdBy) + stream.writeUInt8(inp.hasChats ? 1 : 0) +} + +export const deserializeGroupAccount = (stream: VectorBufferStream, root = false): GroupAccount => { + if (root && stream.readUInt16() !== SerdeTypeIdent.GroupAccount) { + throw new Error('Unexpected bufferstream for GroupAccount type') + } + + const id = stream.readString() + const type = stream.readString() + const hash = stream.readString() + const timestamp = Number(stream.readBigUInt64()) + + const mlsGroupId = stream.readString() + const cipherSuite = stream.readUInt16() + const epoch = stream.readUInt32() + + const members: string[] = [] + const memberCount = stream.readUInt32() + for (let i = 0; i < memberCount; i++) { + members.push(stream.readString()) + } + + const admins: string[] = [] + const adminCount = stream.readUInt32() + for (let i = 0; i < adminCount; i++) { + admins.push(stream.readString()) + } + + const memberSince = Utils.safeJsonParse(stream.readString()) + const messages = Utils.safeJsonParse(stream.readString()) + const handshakes = Utils.safeJsonParse(stream.readString()) + const pendingWelcomes = Utils.safeJsonParse(stream.readString()) + const checkpoint = Utils.safeJsonParse(stream.readString()) + const lastMessageAt = Utils.safeJsonParse(stream.readString()) + + const meta = stream.readString() + const maxMembers = stream.readUInt32() + const createdBy = stream.readString() + const hasChats = stream.readUInt8() === 1 + + return { + id, + type, + hash, + timestamp, + mlsGroupId, + cipherSuite, + epoch, + members, + admins, + memberSince, + messages, + handshakes, + pendingWelcomes, + checkpoint, + meta, + maxMembers, + lastMessageAt, + createdBy, + hasChats, + } +} diff --git a/src/accounts/index.ts b/src/accounts/index.ts index a52b2cdc..df91089e 100644 --- a/src/accounts/index.ts +++ b/src/accounts/index.ts @@ -10,6 +10,7 @@ import { deserializeNodeAccount, nodeAccount, serializeNodeAccount } from './nod import { deserializeProposalAccount, proposalAccount, serializeProposalAccount } from './proposalAccount' import { daoProposalsMetaAccount, deserializeDaoProposalsMetaAccount, serializeDaoProposalsMetaAccount } from './daoProposalsMetaAccount' import { daoProposalAccount, deserializeDaoProposalAccount, serializeDaoProposalAccount } from './daoProposalAccount' +import { deserializeGroupAccount, groupAccount, serializeGroupAccount } from './groupAccount' import { VectorBufferStream } from '@shardus/core' import { DeveloperPayment, @@ -26,6 +27,7 @@ import { DevAccount, DaoProposalsMeta, DaoProposalAccount, + GroupAccount, } from '../@types' import { Utils } from '@shardus/lib-types' @@ -45,6 +47,7 @@ export enum SerdeTypeIdent { Fallback, DaoProposalsMeta, DaoProposalAccount, + GroupAccount, } export const serializeAccounts = (inp: AccountVariant): VectorBufferStream => { @@ -86,6 +89,13 @@ export const serializeAccounts = (inp: AccountVariant): VectorBufferStream => { case 'DaoProposalAccount': serializeDaoProposalAccount(stream, inp as DaoProposalAccount, true) break + // NOTE: the cases above use camelCase type strings while the account + // constructors set PascalCase ('chatAccount' vs 'ChatAccount'), so those + // accounts silently fall through to fallbackSerializer. GroupAccount is + // registered with the string it actually carries, so it uses this path. + case 'GroupAccount': + serializeGroupAccount(stream, inp as GroupAccount, true) + break default: fallbackSerializer(stream, inp, true) break @@ -122,6 +132,8 @@ export const deserializeAccounts = (buffer: Buffer): AccountVariant => { return deserializeDaoProposalsMetaAccount(stream) case SerdeTypeIdent.DaoProposalAccount: return deserializeDaoProposalAccount(stream) + case SerdeTypeIdent.GroupAccount: + return deserializeGroupAccount(stream) default: return fallbackDeserializer(stream) } @@ -218,4 +230,5 @@ export default { userAccount, daoProposalsMetaAccount, daoProposalAccount, + groupAccount, } diff --git a/src/api/group/group.ts b/src/api/group/group.ts new file mode 100644 index 00000000..556b5163 --- /dev/null +++ b/src/api/group/group.ts @@ -0,0 +1,231 @@ +import { Shardus } from '@shardus/core' +import { GroupAccount, UserAccount } from '../../@types' +import { isGroupAccount, isUserAccount } from '../../@types/accountTypeGuards' + +/** + * Read endpoints for MLS group chat. + * + * Clients poll a group account directly rather than being fanned out to, which + * is what keeps group_message a two-account transaction. The transcript is + * split so a client can catch up cheaply: handshakes by epoch, application + * messages by timestamp. + */ + +const loadGroup = async (dapp: Shardus, groupId: string): Promise => { + const account = await dapp.getLocalOrRemoteAccount(groupId) + if (!account || !account.data) return null + const group = account.data as unknown as GroupAccount + return isGroupAccount(group) ? group : null +} + +/** GET /group/:groupId — metadata only; no transcript. */ +export const info = + (dapp: Shardus) => + async (req, res): Promise => { + try { + const groupId = req.params['groupId'] + const group = await loadGroup(dapp, groupId) + if (!group) { + res.json({ error: 'No group with the given id' }) + return + } + res.json({ + group: { + id: group.id, + mlsGroupId: group.mlsGroupId, + cipherSuite: group.cipherSuite, + epoch: group.epoch, + members: group.members, + admins: group.admins, + memberSince: group.memberSince, + meta: group.meta, + maxMembers: group.maxMembers, + createdBy: group.createdBy, + hasChats: group.hasChats, + timestamp: group.timestamp, + checkpointEpoch: group.checkpoint ? group.checkpoint.epoch : null, + messageCount: group.messages.length, + handshakeCount: group.handshakes.length, + }, + }) + } catch (error) { + dapp.log(error) + res.json({ error }) + } + } + +/** GET /group/:groupId/messages/:timestamp — application messages at or after `timestamp`. */ +export const messages = + (dapp: Shardus) => + async (req, res): Promise => { + try { + const groupId = req.params['groupId'] + const timestamp = Number(req.params['timestamp']) || 0 + const group = await loadGroup(dapp, groupId) + if (!group) { + res.json({ error: 'No group with the given id' }) + return + } + res.json({ + messages: group.messages.filter((msg) => msg.timestamp >= timestamp), + epoch: group.epoch, + timestamp: group.timestamp, + }) + } catch (error) { + dapp.log(error) + res.json({ error }) + } + } + +/** + * GET /group/:groupId/handshakes/:epoch — commits from `epoch` onward. + * + * A client applies these in order to walk from its own epoch to the group's. + * If `oldestAvailableEpoch` is greater than the client's epoch the intervening + * commits have been pruned, and the client must recover via an external join + * using the checkpoint rather than replaying. + */ +export const handshakes = + (dapp: Shardus) => + async (req, res): Promise => { + try { + const groupId = req.params['groupId'] + const fromEpoch = Number(req.params['epoch']) || 0 + const group = await loadGroup(dapp, groupId) + if (!group) { + res.json({ error: 'No group with the given id' }) + return + } + const available = group.handshakes.map((h) => h.epoch) + res.json({ + handshakes: group.handshakes.filter((h) => h.epoch >= fromEpoch), + epoch: group.epoch, + oldestAvailableEpoch: available.length > 0 ? Math.min(...available) : group.epoch, + checkpoint: group.checkpoint, + }) + } catch (error) { + dapp.log(error) + res.json({ error }) + } + } + +/** GET /group/:groupId/welcome/:address — the pending Welcome for a new member. */ +export const welcome = + (dapp: Shardus) => + async (req, res): Promise => { + try { + const groupId = req.params['groupId'] + const address = String(req.params['address'] || '').toLowerCase() + const group = await loadGroup(dapp, groupId) + if (!group) { + res.json({ error: 'No group with the given id' }) + return + } + const envelope = group.pendingWelcomes[address] + if (!envelope) { + res.json({ error: 'No pending welcome for this address' }) + return + } + res.json({ welcome: envelope, cipherSuite: group.cipherSuite, mlsGroupId: group.mlsGroupId }) + } catch (error) { + dapp.log(error) + res.json({ error }) + } + } + +/** GET /group/:groupId/checkpoint — GroupInfo for recovering a desynced member. */ +export const checkpoint = + (dapp: Shardus) => + async (req, res): Promise => { + try { + const groupId = req.params['groupId'] + const group = await loadGroup(dapp, groupId) + if (!group) { + res.json({ error: 'No group with the given id' }) + return + } + if (!group.checkpoint) { + res.json({ error: 'No checkpoint yet for this group' }) + return + } + res.json({ checkpoint: group.checkpoint, epoch: group.epoch, cipherSuite: group.cipherSuite }) + } catch (error) { + dapp.log(error) + res.json({ error }) + } + } + +/** + * GET /account/:id/keypackages — published KeyPackages for adding this account. + * + * The caller picks one and names it in group_commit.consumedKeyPackages, which + * pops it from the pool so it is never reused. + */ +export const keyPackages = + (dapp: Shardus) => + async (req, res): Promise => { + try { + const id = req.params['id'] + const account = await dapp.getLocalOrRemoteAccount(id) + if (!account || !account.data) { + res.json({ error: 'No account with the given id' }) + return + } + const user = account.data as unknown as UserAccount + if (!isUserAccount(user)) { + res.json({ error: 'Account is not a UserAccount' }) + return + } + res.json({ + keyPackages: user.data.mlsKeyPackages || [], + lastResortKeyPackage: user.data.mlsLastResortKeyPackage || null, + cipherSuite: user.data.mlsCipherSuite || null, + pqPublicKey: user.pqPublicKey || null, + }) + } catch (error) { + dapp.log(error) + res.json({ error }) + } + } + +/** + * GET /account/:id/groups — group ids this account belongs to. + * + * Derived from the chats map, which group_create and group_commit maintain, so + * a client can enumerate its groups without a separate index. + */ +export const accountGroups = + (dapp: Shardus) => + async (req, res): Promise => { + try { + const id = req.params['id'] + const account = await dapp.getLocalOrRemoteAccount(id) + if (!account || !account.data) { + res.json({ error: 'No account with the given id' }) + return + } + const user = account.data as unknown as UserAccount + if (!isUserAccount(user)) { + res.json({ error: 'Account is not a UserAccount' }) + return + } + + const candidates = Object.keys(user.data.chats || {}) + const groups: { id: string; epoch: number; timestamp: number; members: number }[] = [] + for (const candidate of candidates) { + const group = await loadGroup(dapp, candidate) + if (group) { + groups.push({ + id: group.id, + epoch: group.epoch, + timestamp: group.timestamp, + members: group.members.length, + }) + } + } + res.json({ groups }) + } catch (error) { + dapp.log(error) + res.json({ error }) + } + } diff --git a/src/api/group/index.ts b/src/api/group/index.ts new file mode 100644 index 00000000..137f7a89 --- /dev/null +++ b/src/api/group/index.ts @@ -0,0 +1,11 @@ +import { info, messages, handshakes, welcome, checkpoint, keyPackages, accountGroups } from './group' + +export default { + info, + messages, + handshakes, + welcome, + checkpoint, + keyPackages, + accountGroups, +} diff --git a/src/api/index.ts b/src/api/index.ts index 08a1e728..3efbd502 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -10,6 +10,7 @@ import node from './node' import { handlePutAdminCertificate } from './admin_certificate' import { debug_liberdus_flags, set_liberdus_flag } from './liberdus_flags' import dao from './dao' +import group from './group' import { Shardus } from '@shardus/core' export default (dapp: Shardus): void => { dapp.registerExternalPost('inject', inject(dapp)) @@ -51,6 +52,16 @@ export default (dapp: Shardus): void => { dapp.registerExternalGet('account/:id/chats/:timestamp', accounts.chats(dapp)) // dapp.registerExternalGet('accounts', accounts.all(dapp)) + // MLS group chat. Same LIFO caveat as the DAO routes above: register the + // broadest path first so the specific ones win. + dapp.registerExternalGet('group/:groupId', group.info(dapp)) + dapp.registerExternalGet('group/:groupId/checkpoint', group.checkpoint(dapp)) + dapp.registerExternalGet('group/:groupId/messages/:timestamp', group.messages(dapp)) + dapp.registerExternalGet('group/:groupId/handshakes/:epoch', group.handshakes(dapp)) + dapp.registerExternalGet('group/:groupId/welcome/:address', group.welcome(dapp)) + dapp.registerExternalGet('account/:id/keypackages', group.keyPackages(dapp)) + dapp.registerExternalGet('account/:id/groups', group.accountGroups(dapp)) + dapp.registerExternalGet('transaction/:id', accounts.transactions(dapp)) dapp.registerExternalGet('messages/:chatId/:timestamp', messages.messages(dapp)) diff --git a/src/config/index.ts b/src/config/index.ts index 3d880e8f..37e76de5 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -271,6 +271,24 @@ interface LiberdusFlags { minCommitteeMembers: number maxCommitteeMembers: number enableAJVValidation: boolean + // --- MLS group chat ------------------------------------------------------- + // Kill-switch for the whole group_* transaction family. Everything below + // affects transaction validity, so it must stay identical across all nodes. + enableGroupChat: boolean + /** Hard cap on members per group. Bounds commit size and account hotspotting. */ + groupMaxMembers: number + /** Members addable/removable in one commit (bounds the transaction key set). */ + groupMaxMembersPerCommit: number + /** Max size of one MLS application message, in kB. */ + groupMessageSizeLimit: number + /** Application messages retained per group before the oldest are dropped. */ + groupMessageMaxLength: number + /** Days of application-message history retained. Commits are NOT pruned by this. */ + groupMessageRetentionDays: number + /** Minimum gap between group_message transactions from one member, in ms. */ + groupMessageMinIntervalMs: number + /** Max unconsumed KeyPackages an account may hold at once. */ + groupMaxKeyPackagesPerAccount: number versionFlags: { replierNoToll: boolean allowZeroToll: boolean @@ -322,6 +340,15 @@ export const LiberdusFlags: LiberdusFlags = { minCommitteeMembers: 4, maxCommitteeMembers: 10, enableAJVValidation: false, + // MLS group chat — off until the feature ships + enableGroupChat: true, + groupMaxMembers: 50, + groupMaxMembersPerCommit: 10, + groupMessageSizeLimit: 64, // 64 kB; an X-Wing commit is ~5.5 kB + groupMessageMaxLength: 500, + groupMessageRetentionDays: 7, + groupMessageMinIntervalMs: 1000, + groupMaxKeyPackagesPerAccount: 10, // X-Wing KeyPackages are ~2.6 kB each versionFlags: { replierNoToll: true, // turn on by 2.3.5 allowZeroToll: true, // turn on by 2.3.6 diff --git a/src/index.ts b/src/index.ts index 9d45d46c..241ecd9b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -77,6 +77,15 @@ const daoPreCrackTxTypes = new Set([ TXTypes.dao_cancel, ]) +/** MLS group chat transaction family, gated by LiberdusFlags.enableGroupChat. */ +const groupChatTxTypes = new Set([ + TXTypes.group_create, + TXTypes.group_keypackage_publish, + TXTypes.group_message, + TXTypes.group_commit, + TXTypes.group_leave, +]) + let isReadyToJoinLatestValue = false let mustUseAdminCert = false @@ -280,6 +289,12 @@ const shardusSetup = (): void => { return validationResult } + // 3.8. Reject MLS group chat transactions while the feature is off + if (!LiberdusFlags.enableGroupChat && groupChatTxTypes.has(tx.type)) { + validationResult.reason = 'Group chat transactions are not enabled on this network yet' + return validationResult + } + // 4. Validate the tx fields if (LiberdusFlags.enableAJVValidation) { const errors = verifyPayload(tx.type, tx) diff --git a/src/transactions/group_commit.ts b/src/transactions/group_commit.ts new file mode 100644 index 00000000..e0f6f057 --- /dev/null +++ b/src/transactions/group_commit.ts @@ -0,0 +1,468 @@ +import * as crypto from '../crypto' +import { Shardus, ShardusTypes } from '@shardus/core' +import * as utils from '../utils' +import * as config from '../config' +import { UserAccount, GroupAccount, WrappedStates, Tx, AppReceiptData, TXTypes } from '../@types' +import { SafeBigIntMath } from '../utils/safeBigIntMath' +import * as AccountsStorage from '../storage/accountStorage' +import { isUserAccount, isGroupAccount } from '../@types/accountTypeGuards' + +/** + * An MLS membership change: proposals + commit, plus the Welcomes for anyone + * being added. + * + * THE EPOCH FENCE + * --------------- + * MLS cannot tolerate two members committing at the same epoch — that is an + * unrecoverable state fork. Because every group transaction targets one + * account, Shardus orders them deterministically by timestamp, so requiring + * `tx.epoch === group.epoch` makes the network itself enforce + * exactly-one-commit-per-epoch: the first commit bumps the epoch and any racer + * still naming the old one is rejected with 'stale epoch'. The loser applies + * the winning commit, rebuilds its proposal and retries. + * + * This is stronger than what a conventional delivery service can offer, and it + * is the reason MLS fits a blockchain well. + */ +export const validate_fields = (tx: Tx.GroupCommit, response: ShardusTypes.IncomingTransactionResult): ShardusTypes.IncomingTransactionResult => { + if (utils.isValidAddress(tx.from) === false) { + response.reason = 'tx "from" is not a valid address.' + return response + } + if (utils.isValidAddress(tx.groupId) === false) { + response.reason = 'tx "groupId" is not a valid address.' + return response + } + if (typeof tx.epoch !== 'number' || !Number.isInteger(tx.epoch) || tx.epoch < 0) { + response.reason = 'tx "epoch" must be a non-negative integer.' + return response + } + if (typeof tx.commit !== 'string' || tx.commit.length === 0) { + response.reason = 'tx "commit" must be a non-empty string.' + return response + } + if (!Array.isArray(tx.proposals)) { + response.reason = 'tx "proposals" must be an array.' + return response + } + if (typeof tx.pskId !== 'string' || typeof tx.pskNonce !== 'string') { + response.reason = 'tx "pskId" and "pskNonce" must be strings.' + return response + } + if (typeof tx.groupInfo !== 'string' || typeof tx.ratchetTree !== 'string') { + response.reason = 'tx "groupInfo" and "ratchetTree" must be strings.' + return response + } + if (!Array.isArray(tx.addedMembers) || !Array.isArray(tx.removedMembers)) { + response.reason = 'tx "addedMembers" and "removedMembers" must be arrays.' + return response + } + if (!Array.isArray(tx.welcomes)) { + response.reason = 'tx "welcomes" must be an array.' + return response + } + if (!Array.isArray(tx.consumedKeyPackages)) { + response.reason = 'tx "consumedKeyPackages" must be an array.' + return response + } + /* + * Exactly one consumed KeyPackage per added member. The network cannot parse + * MLS, so this declaration is the only thing that lets apply() pop the used + * package from the addee's pool. Without it a committer could keep adding + * someone against the same init key, defeating forward secrecy. + */ + if (tx.consumedKeyPackages.length !== tx.addedMembers.length) { + response.reason = 'tx must declare exactly one consumed key package per added member.' + return response + } + const consumedFor = new Set(tx.consumedKeyPackages.map((c) => c && c.address)) + if (consumedFor.size !== tx.addedMembers.length || !tx.addedMembers.every((a) => consumedFor.has(a))) { + response.reason = 'tx "consumedKeyPackages" must cover each added member exactly once.' + return response + } + + const maxPerCommit = config.LiberdusFlags.groupMaxMembersPerCommit + if (tx.addedMembers.length + tx.removedMembers.length > maxPerCommit) { + response.reason = `a commit may change at most ${maxPerCommit} members.` + return response + } + for (const address of [...tx.addedMembers, ...tx.removedMembers]) { + if (utils.isValidAddress(address) === false) { + response.reason = 'tx contains an invalid member address.' + return response + } + } + if (new Set(tx.addedMembers).size !== tx.addedMembers.length) { + response.reason = 'tx "addedMembers" contains duplicates.' + return response + } + if (new Set(tx.removedMembers).size !== tx.removedMembers.length) { + response.reason = 'tx "removedMembers" contains duplicates.' + return response + } + // Every added member needs a Welcome, or they can never join. + if (tx.welcomes.length !== tx.addedMembers.length) { + response.reason = 'tx must contain exactly one welcome per added member.' + return response + } + for (const welcome of tx.welcomes) { + if (!welcome || !tx.addedMembers.includes(welcome.address)) { + response.reason = 'tx contains a welcome for an address that is not being added.' + return response + } + const env = welcome.envelope + if (!env || typeof env.welcome !== 'string' || typeof env.ratchetTree !== 'string' || !env.sealedPsk) { + response.reason = 'tx contains a malformed welcome envelope.' + return response + } + if ( + typeof env.sealedPsk.cipherText !== 'string' || + typeof env.sealedPsk.nonce !== 'string' || + typeof env.sealedPsk.ct !== 'string' + ) { + response.reason = 'tx contains a malformed sealed post-quantum PSK.' + return response + } + } + + const totalBytes = + Buffer.byteLength(tx.commit, 'utf8') + + Buffer.byteLength(tx.groupInfo, 'utf8') + + Buffer.byteLength(tx.ratchetTree, 'utf8') + + tx.proposals.reduce((sum, p) => sum + Buffer.byteLength(String(p), 'utf8'), 0) + if (totalBytes / 1024 > config.LiberdusFlags.groupMessageSizeLimit) { + response.reason = `commit payload exceeds ${config.LiberdusFlags.groupMessageSizeLimit} kB.` + return response + } + + if (typeof tx.fee !== 'bigint') { + response.reason = 'tx "fee" must be a bigint.' + return response + } + if (!tx.sign || !tx.sign.owner || !tx.sign.sig || tx.sign.owner !== tx.from) { + response.reason = 'not signed by from account' + return response + } + if (crypto.verifyObj(tx) === false) { + response.reason = 'incorrect signing' + return response + } + response.success = true + return response +} + +export const validate = ( + tx: Tx.GroupCommit, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId] && wrappedStates[tx.groupId].data + + if (typeof from === 'undefined' || from === null) { + response.reason = '"from" account does not exist.' + return response + } + if (!isUserAccount(from)) { + response.reason = 'from account is not a UserAccount' + return response + } + if (typeof group === 'undefined' || group === null) { + response.reason = '"groupId" account does not exist.' + return response + } + if (!isGroupAccount(group)) { + response.reason = 'groupId account is not a GroupAccount' + return response + } + if (!group.members.includes(tx.from)) { + response.reason = 'sender is not a member of this group.' + return response + } + + // THE FENCE. Everything else about MLS ordering depends on this check. + if (tx.epoch !== group.epoch) { + response.reason = `stale epoch: commit targets epoch ${tx.epoch} but the group is at ${group.epoch}. Apply the latest commit and retry.` + return response + } + + const changesMembership = tx.addedMembers.length > 0 || tx.removedMembers.length > 0 + if (changesMembership && !group.admins.includes(tx.from)) { + response.reason = 'only an admin may add or remove members.' + return response + } + + for (const address of tx.addedMembers) { + if (group.members.includes(address)) { + response.reason = `address ${address} is already a member.` + return response + } + const addee: UserAccount = wrappedStates[address] && wrappedStates[address].data + if (!addee || !isUserAccount(addee)) { + response.reason = `added member ${address} does not have a UserAccount.` + return response + } + } + for (const address of tx.removedMembers) { + if (!group.members.includes(address)) { + response.reason = `address ${address} is not a member.` + return response + } + } + if (tx.removedMembers.includes(tx.from)) { + response.reason = 'use group_leave to remove yourself.' + return response + } + + const nextSize = group.members.length + tx.addedMembers.length - tx.removedMembers.length + if (nextSize > group.maxMembers || nextSize > config.LiberdusFlags.groupMaxMembers) { + response.reason = `group would exceed its member limit (${group.maxMembers}).` + return response + } + if (nextSize < 1) { + response.reason = 'a group must retain at least one member.' + return response + } + + // A consumed KeyPackage must actually be one the addee published, otherwise a + // committer could add a key of its own choosing on someone else's behalf. + for (const consumed of tx.consumedKeyPackages) { + if (!tx.addedMembers.includes(consumed.address)) { + response.reason = 'consumedKeyPackages references an address that is not being added.' + return response + } + const addee: UserAccount = wrappedStates[consumed.address] && wrappedStates[consumed.address].data + const pool = addee.data.mlsKeyPackages || [] + const isLastResort = addee.data.mlsLastResortKeyPackage === consumed.keyPackage + if (!pool.includes(consumed.keyPackage) && !isLastResort) { + response.reason = `key package for ${consumed.address} was not published by that account.` + return response + } + if (addee.data.mlsCipherSuite !== undefined && addee.data.mlsCipherSuite !== group.cipherSuite) { + response.reason = `key package for ${consumed.address} uses a different ciphersuite than the group.` + return response + } + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.success = false + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < transactionFee) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + +export const apply = ( + tx: Tx.GroupCommit, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, +): void => { + const from: UserAccount = wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId].data + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + + const previousEpoch = group.epoch + + // Record the commit before mutating membership, so the transcript reads as + // "at epoch N, this commit was applied". + const commitRecord: Tx.GroupCommitRecord = { + type: TXTypes.group_commit, + txId, + from: tx.from, + groupId: tx.groupId, + epoch: previousEpoch, + commit: tx.commit, + proposals: [...tx.proposals], + pskId: tx.pskId, + pskNonce: tx.pskNonce, + addedMembers: [...tx.addedMembers], + removedMembers: [...tx.removedMembers], + timestamp: txTimestamp, + sign: tx.sign, + } + group.handshakes.push(commitRecord) + + group.epoch = previousEpoch + 1 + + const removed = new Set(tx.removedMembers) + group.members = [...group.members.filter((m) => !removed.has(m)), ...tx.addedMembers] + group.admins = group.admins.filter((a) => !removed.has(a)) + + for (const address of tx.addedMembers) { + group.memberSince[address] = { epoch: group.epoch, timestamp: txTimestamp } + } + for (const address of tx.removedMembers) { + delete group.memberSince[address] + delete group.lastMessageAt[address] + delete group.pendingWelcomes[address] + } + + // Park each Welcome (and its sealed PQ PSK) for collection by the new member. + for (const welcome of tx.welcomes) { + group.pendingWelcomes[welcome.address] = { + ...welcome.envelope, + epoch: group.epoch, + timestamp: txTimestamp, + } + } + + /* + * Checkpoint every commit. A member who was offline while older commits were + * pruned can rejoin from this GroupInfo via an external commit; without it + * they would be locked out permanently. + */ + group.checkpoint = { + epoch: group.epoch, + groupInfo: tx.groupInfo, + ratchetTree: tx.ratchetTree, + timestamp: txTimestamp, + } + + if (typeof tx.meta === 'string' && tx.meta.length > 0) { + group.meta = tx.meta + } + + // Consume the single-use KeyPackages this commit used up. + for (const consumed of tx.consumedKeyPackages) { + const addee: UserAccount = wrappedStates[consumed.address].data + if (Array.isArray(addee.data.mlsKeyPackages)) { + addee.data.mlsKeyPackages = addee.data.mlsKeyPackages.filter((kp) => kp !== consumed.keyPackage) + } + addee.timestamp = txTimestamp + } + + /* + * Point each added member's client at the group through the chats map, and + * bump chatTimestamp so the collector long-poll wakes them. This reuses the + * existing discovery path, so a new member needs no new notification channel + * and still finds the group after a week offline. + */ + for (const address of tx.addedMembers) { + const addee: UserAccount = wrappedStates[address].data + addee.data.chats[tx.groupId] = { + receivedTimestamp: txTimestamp, + chatId: tx.groupId, + } + addee.data.chatTimestamp = txTimestamp + addee.timestamp = txTimestamp + } + + for (const address of tx.removedMembers) { + const removee: UserAccount = wrappedStates[address] && wrappedStates[address].data + if (removee && removee.data && removee.data.chats) { + delete removee.data.chats[tx.groupId] + removee.data.chatTimestamp = txTimestamp + removee.timestamp = txTimestamp + } + } + + group.timestamp = txTimestamp + from.timestamp = txTimestamp + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: true, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { + groupId: tx.groupId, + epoch: group.epoch, + addedMembers: tx.addedMembers, + removedMembers: tx.removedMembers, + }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) + dapp.log('Applied group_commit tx', tx.groupId, `epoch ${previousEpoch} -> ${group.epoch}`) +} + +export const createFailedAppReceiptData = ( + tx: Tx.GroupCommit, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, + reason: string, +): void => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + let transactionFee = BigInt(0) + if (from !== undefined && from !== null) { + const networkFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (from.data.balance >= networkFee) { + transactionFee = networkFee + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } else { + transactionFee = from.data.balance + from.data.balance = BigInt(0) + } + from.timestamp = txTimestamp + } + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: false, + reason, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { groupId: tx.groupId, epoch: tx.epoch }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) +} + +/** + * Added and removed members are targets because their accounts are written: + * added members get the group pointer and lose a KeyPackage, removed members + * lose the pointer. Bounded by groupMaxMembersPerCommit, so unlike + * group_message this stays a small key set even for large groups. + */ +export const keys = (tx: Tx.GroupCommit, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { + result.sourceKeys = [tx.from] + result.targetKeys = [tx.groupId, ...tx.addedMembers, ...tx.removedMembers] + result.allKeys = [...result.sourceKeys, ...result.targetKeys] + return result +} + +export const memoryPattern = (tx: Tx.GroupCommit, result: ShardusTypes.TransactionKeys): ShardusTypes.ShardusMemoryPatternsInput => { + return { + rw: [tx.from, tx.groupId, ...tx.addedMembers, ...tx.removedMembers], + wo: [], + on: [], + ri: [], + ro: [], + } +} + +export const createRelevantAccount = ( + dapp: Shardus, + account: UserAccount | GroupAccount, + accountId: string, + tx: Tx.GroupCommit, + accountCreated = false, +): ShardusTypes.WrappedResponse => { + if (!account) { + throw Error('Account must exist in order to commit to a group') + } + return dapp.createWrappedResponse(accountId, accountCreated, account.hash, account.timestamp, account) +} diff --git a/src/transactions/group_create.ts b/src/transactions/group_create.ts new file mode 100644 index 00000000..48b76616 --- /dev/null +++ b/src/transactions/group_create.ts @@ -0,0 +1,239 @@ +import * as crypto from '../crypto' +import { Shardus, ShardusTypes } from '@shardus/core' +import * as utils from '../utils' +import create from '../accounts' +import * as config from '../config' +import { UserAccount, GroupAccount, WrappedStates, Tx, AppReceiptData } from '../@types' +import { SafeBigIntMath } from '../utils/safeBigIntMath' +import * as AccountsStorage from '../storage/accountStorage' +import { isUserAccount } from '../@types/accountTypeGuards' + +/** + * Creates the GroupAccount backing a new MLS group. The founder is the only + * member at epoch 0; everyone else joins through group_commit. + * + * The group id is client-computed as hash(from + groupNonce) so it can be named + * in keys() before the account exists — the same pattern dao_proposal_create + * uses for tx.proposalId. The nonce is random rather than a counter so two + * concurrent creations by one account cannot collide. + */ +export const validate_fields = (tx: Tx.GroupCreate, response: ShardusTypes.IncomingTransactionResult): ShardusTypes.IncomingTransactionResult => { + if (utils.isValidAddress(tx.from) === false) { + response.reason = 'tx "from" is not a valid address.' + return response + } + if (utils.isValidAddress(tx.groupId) === false) { + response.reason = 'tx "groupId" is not a valid address.' + return response + } + if (typeof tx.groupNonce !== 'string' || tx.groupNonce.length !== 64) { + response.reason = 'tx "groupNonce" must be a 32-byte hex string.' + return response + } + if (tx.groupId !== utils.calculateGroupId(tx.from, tx.groupNonce)) { + response.reason = 'groupId is not calculated correctly from "from" and "groupNonce".' + return response + } + if (typeof tx.mlsGroupId !== 'string' || tx.mlsGroupId.length === 0) { + response.reason = 'tx "mlsGroupId" must be a non-empty string.' + return response + } + if (typeof tx.cipherSuite !== 'number' || !Number.isInteger(tx.cipherSuite) || tx.cipherSuite <= 0) { + response.reason = 'tx "cipherSuite" must be a positive integer.' + return response + } + if (typeof tx.meta !== 'string') { + response.reason = 'tx "meta" must be a string.' + return response + } + if (Buffer.byteLength(tx.meta, 'utf8') / 1024 > config.LiberdusFlags.groupMessageSizeLimit) { + response.reason = `tx "meta" size must be less than ${config.LiberdusFlags.groupMessageSizeLimit} kB.` + return response + } + if ( + typeof tx.maxMembers !== 'number' || + !Number.isInteger(tx.maxMembers) || + tx.maxMembers < 1 || + tx.maxMembers > config.LiberdusFlags.groupMaxMembers + ) { + response.reason = `tx "maxMembers" must be an integer between 1 and ${config.LiberdusFlags.groupMaxMembers}.` + return response + } + if (typeof tx.fee !== 'bigint') { + response.reason = 'tx "fee" must be a bigint.' + return response + } + if (!tx.sign || !tx.sign.owner || !tx.sign.sig || tx.sign.owner !== tx.from) { + response.reason = 'not signed by from account' + return response + } + if (crypto.verifyObj(tx) === false) { + response.reason = 'incorrect signing' + return response + } + response.success = true + return response +} + +export const validate = ( + tx: Tx.GroupCreate, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId] && wrappedStates[tx.groupId].data + + if (typeof from === 'undefined' || from === null) { + response.reason = '"from" account does not exist.' + return response + } + if (!isUserAccount(from)) { + response.reason = 'from account is not a UserAccount' + return response + } + // createRelevantAccount produces a fresh GroupAccount; a populated one means + // this groupId is already taken. + if (group && group.epoch > 0) { + response.reason = 'group already exists' + return response + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.success = false + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < transactionFee) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + +export const apply = ( + tx: Tx.GroupCreate, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, +): void => { + const from: UserAccount = wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId].data + + if (!group) { + throw Error('getRelevantAccount must be called before apply') + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + + // Point the founder's own client at the group through the existing chats + // discovery path, and bump chatTimestamp so the collector long-poll fires. + if (!from.data.chats[tx.groupId]) { + from.data.chats[tx.groupId] = { + receivedTimestamp: txTimestamp, + chatId: tx.groupId, + } + } + from.data.chatTimestamp = txTimestamp + + group.timestamp = txTimestamp + from.timestamp = txTimestamp + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: true, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { + groupId: tx.groupId, + cipherSuite: tx.cipherSuite, + maxMembers: tx.maxMembers, + }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) + dapp.log('Applied group_create tx', group, from) +} + +export const createFailedAppReceiptData = ( + tx: Tx.GroupCreate, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, + reason: string, +): void => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + let transactionFee = BigInt(0) + if (from !== undefined && from !== null) { + const networkFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (from.data.balance >= networkFee) { + transactionFee = networkFee + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } else { + transactionFee = from.data.balance + from.data.balance = BigInt(0) + } + from.timestamp = txTimestamp + } + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: false, + reason, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { groupId: tx.groupId }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) +} + +export const keys = (tx: Tx.GroupCreate, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { + result.sourceKeys = [tx.from] + result.targetKeys = [tx.groupId] + result.allKeys = [...result.sourceKeys, ...result.targetKeys] + return result +} + +export const memoryPattern = (tx: Tx.GroupCreate, result: ShardusTypes.TransactionKeys): ShardusTypes.ShardusMemoryPatternsInput => { + return { + rw: [tx.from, tx.groupId], + wo: [], + on: [], + ri: [], + ro: [], + } +} + +export const createRelevantAccount = ( + dapp: Shardus, + account: UserAccount | GroupAccount, + accountId: string, + tx: Tx.GroupCreate, + accountCreated = false, +): ShardusTypes.WrappedResponse => { + if (!account) { + if (accountId === tx.groupId) { + account = create.groupAccount(accountId, tx, tx.timestamp) + accountCreated = true + } else { + throw Error('Account must exist in order to create a group') + } + } + return dapp.createWrappedResponse(accountId, accountCreated, account.hash, account.timestamp, account) +} diff --git a/src/transactions/group_keypackage_publish.ts b/src/transactions/group_keypackage_publish.ts new file mode 100644 index 00000000..fd3f0b5a --- /dev/null +++ b/src/transactions/group_keypackage_publish.ts @@ -0,0 +1,223 @@ +import * as crypto from '../crypto' +import { Shardus, ShardusTypes } from '@shardus/core' +import * as utils from '../utils' +import * as config from '../config' +import { UserAccount, WrappedStates, Tx, AppReceiptData } from '../@types' +import { SafeBigIntMath } from '../utils/safeBigIntMath' +import * as AccountsStorage from '../storage/accountStorage' +import { isUserAccount } from '../@types/accountTypeGuards' + +/** + * Publishes MLS KeyPackages so other accounts can add this one to a group. + * + * KeyPackages are single-use: group_commit pops the one it consumes from this + * pool in the same transaction. Reusing an init key across two adds would + * undermine forward secrecy, which is why the pool exists rather than a single + * static key. `lastResortKeyPackage` is the RFC 9420 s10 fallback for when the + * pool empties — reusable, with a documented post-compromise-security caveat. + * + * Publishing replaces the pool wholesale, so a client that has rotated its MLS + * identity (for example after an account restore, where the signature key is + * recoverable from pqSeed but the HPKE keys are not) can cleanly invalidate the + * stale packages. + */ +export const validate_fields = ( + tx: Tx.GroupKeyPackagePublish, + response: ShardusTypes.IncomingTransactionResult, +): ShardusTypes.IncomingTransactionResult => { + if (utils.isValidAddress(tx.from) === false) { + response.reason = 'tx "from" is not a valid address.' + return response + } + if (!Array.isArray(tx.keyPackages)) { + response.reason = 'tx "keyPackages" must be an array.' + return response + } + if (tx.keyPackages.length === 0 && !tx.lastResortKeyPackage) { + response.reason = 'tx must publish at least one key package.' + return response + } + if (tx.keyPackages.length > config.LiberdusFlags.groupMaxKeyPackagesPerAccount) { + response.reason = `tx "keyPackages" must contain at most ${config.LiberdusFlags.groupMaxKeyPackagesPerAccount} entries.` + return response + } + for (const kp of tx.keyPackages) { + if (typeof kp !== 'string' || kp.length === 0) { + response.reason = 'each key package must be a non-empty base64 string.' + return response + } + } + if (tx.lastResortKeyPackage !== undefined && typeof tx.lastResortKeyPackage !== 'string') { + response.reason = 'tx "lastResortKeyPackage" must be a string when present.' + return response + } + if (typeof tx.cipherSuite !== 'number' || !Number.isInteger(tx.cipherSuite) || tx.cipherSuite <= 0) { + response.reason = 'tx "cipherSuite" must be a positive integer.' + return response + } + + const totalBytes = tx.keyPackages.reduce((sum, kp) => sum + Buffer.byteLength(kp, 'utf8'), 0) + + (tx.lastResortKeyPackage ? Buffer.byteLength(tx.lastResortKeyPackage, 'utf8') : 0) + if (totalBytes / 1024 > config.LiberdusFlags.groupMessageSizeLimit) { + response.reason = `published key packages exceed ${config.LiberdusFlags.groupMessageSizeLimit} kB.` + return response + } + + if (typeof tx.fee !== 'bigint') { + response.reason = 'tx "fee" must be a bigint.' + return response + } + if (!tx.sign || !tx.sign.owner || !tx.sign.sig || tx.sign.owner !== tx.from) { + response.reason = 'not signed by from account' + return response + } + if (crypto.verifyObj(tx) === false) { + response.reason = 'incorrect signing' + return response + } + response.success = true + return response +} + +export const validate = ( + tx: Tx.GroupKeyPackagePublish, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + + if (typeof from === 'undefined' || from === null) { + response.reason = '"from" account does not exist.' + return response + } + if (!isUserAccount(from)) { + response.reason = 'from account is not a UserAccount' + return response + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.success = false + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < transactionFee) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + +export const apply = ( + tx: Tx.GroupKeyPackagePublish, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, +): void => { + const from: UserAccount = wrappedStates[tx.from].data + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + + from.data.mlsKeyPackages = [...tx.keyPackages] + from.data.mlsCipherSuite = tx.cipherSuite + if (tx.lastResortKeyPackage) { + from.data.mlsLastResortKeyPackage = tx.lastResortKeyPackage + } + + from.timestamp = txTimestamp + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: true, + from: tx.from, + to: tx.from, + type: tx.type, + transactionFee, + additionalInfo: { + keyPackageCount: tx.keyPackages.length, + cipherSuite: tx.cipherSuite, + }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) + dapp.log('Applied group_keypackage_publish tx', from.id, tx.keyPackages.length) +} + +export const createFailedAppReceiptData = ( + tx: Tx.GroupKeyPackagePublish, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, + reason: string, +): void => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + let transactionFee = BigInt(0) + if (from !== undefined && from !== null) { + const networkFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (from.data.balance >= networkFee) { + transactionFee = networkFee + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } else { + transactionFee = from.data.balance + from.data.balance = BigInt(0) + } + from.timestamp = txTimestamp + } + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: false, + reason, + from: tx.from, + to: tx.from, + type: tx.type, + transactionFee, + additionalInfo: {}, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) +} + +export const keys = (tx: Tx.GroupKeyPackagePublish, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { + result.sourceKeys = [tx.from] + result.targetKeys = [] + result.allKeys = [...result.sourceKeys, ...result.targetKeys] + return result +} + +export const memoryPattern = ( + tx: Tx.GroupKeyPackagePublish, + result: ShardusTypes.TransactionKeys, +): ShardusTypes.ShardusMemoryPatternsInput => { + return { + rw: [tx.from], + wo: [], + on: [], + ri: [], + ro: [], + } +} + +export const createRelevantAccount = ( + dapp: Shardus, + account: UserAccount, + accountId: string, + tx: Tx.GroupKeyPackagePublish, + accountCreated = false, +): ShardusTypes.WrappedResponse => { + if (!account) { + throw Error('Account must exist in order to publish key packages') + } + return dapp.createWrappedResponse(accountId, accountCreated, account.hash, account.timestamp, account) +} diff --git a/src/transactions/group_leave.ts b/src/transactions/group_leave.ts new file mode 100644 index 00000000..76e0ebef --- /dev/null +++ b/src/transactions/group_leave.ts @@ -0,0 +1,221 @@ +import * as crypto from '../crypto' +import { Shardus, ShardusTypes } from '@shardus/core' +import * as utils from '../utils' +import { UserAccount, GroupAccount, WrappedStates, Tx, AppReceiptData } from '../@types' +import { SafeBigIntMath } from '../utils/safeBigIntMath' +import * as AccountsStorage from '../storage/accountStorage' +import { isUserAccount, isGroupAccount } from '../@types/accountTypeGuards' + +/** + * Self-removal from a group. + * + * IMPORTANT: this drops the member from the roster and stops the network + * accepting their messages, but it is NOT cryptographically effective on its + * own — the leaver still holds the current epoch's group secret and could + * decrypt traffic until a remaining member commits a Remove, which rotates the + * keys. Clients should prompt an admin to commit promptly and should surface + * the group as "leaving" until that lands. + * + * Leaving is deliberately not fenced on epoch: it does not advance the MLS + * epoch, and blocking someone from leaving because a commit raced them would be + * hostile. + */ +export const validate_fields = (tx: Tx.GroupLeave, response: ShardusTypes.IncomingTransactionResult): ShardusTypes.IncomingTransactionResult => { + if (utils.isValidAddress(tx.from) === false) { + response.reason = 'tx "from" is not a valid address.' + return response + } + if (utils.isValidAddress(tx.groupId) === false) { + response.reason = 'tx "groupId" is not a valid address.' + return response + } + if (typeof tx.fee !== 'bigint') { + response.reason = 'tx "fee" must be a bigint.' + return response + } + if (!tx.sign || !tx.sign.owner || !tx.sign.sig || tx.sign.owner !== tx.from) { + response.reason = 'not signed by from account' + return response + } + if (crypto.verifyObj(tx) === false) { + response.reason = 'incorrect signing' + return response + } + response.success = true + return response +} + +export const validate = ( + tx: Tx.GroupLeave, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId] && wrappedStates[tx.groupId].data + + if (typeof from === 'undefined' || from === null) { + response.reason = '"from" account does not exist.' + return response + } + if (!isUserAccount(from)) { + response.reason = 'from account is not a UserAccount' + return response + } + if (typeof group === 'undefined' || group === null) { + response.reason = '"groupId" account does not exist.' + return response + } + if (!isGroupAccount(group)) { + response.reason = 'groupId account is not a GroupAccount' + return response + } + if (!group.members.includes(tx.from)) { + response.reason = 'sender is not a member of this group.' + return response + } + // The last member cannot leave, or the group would be unrecoverable while + // still holding a transcript. + if (group.members.length === 1) { + response.reason = 'the last member cannot leave the group.' + return response + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.success = false + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < transactionFee) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + +export const apply = ( + tx: Tx.GroupLeave, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, +): void => { + const from: UserAccount = wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId].data + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + + group.members = group.members.filter((m) => m !== tx.from) + group.admins = group.admins.filter((a) => a !== tx.from) + delete group.memberSince[tx.from] + delete group.lastMessageAt[tx.from] + delete group.pendingWelcomes[tx.from] + + // If the last admin walked out, promote the longest-standing remaining member + // so the group can still be managed. + if (group.admins.length === 0 && group.members.length > 0) { + const successor = group.members + .slice() + .sort((a, b) => (group.memberSince[a]?.timestamp || 0) - (group.memberSince[b]?.timestamp || 0))[0] + group.admins.push(successor) + } + + if (from.data.chats && from.data.chats[tx.groupId]) { + delete from.data.chats[tx.groupId] + } + from.data.chatTimestamp = txTimestamp + + group.timestamp = txTimestamp + from.timestamp = txTimestamp + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: true, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { + groupId: tx.groupId, + remainingMembers: group.members.length, + }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) + dapp.log('Applied group_leave tx', tx.groupId, tx.from) +} + +export const createFailedAppReceiptData = ( + tx: Tx.GroupLeave, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, + reason: string, +): void => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + let transactionFee = BigInt(0) + if (from !== undefined && from !== null) { + const networkFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (from.data.balance >= networkFee) { + transactionFee = networkFee + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } else { + transactionFee = from.data.balance + from.data.balance = BigInt(0) + } + from.timestamp = txTimestamp + } + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: false, + reason, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { groupId: tx.groupId }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) +} + +export const keys = (tx: Tx.GroupLeave, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { + result.sourceKeys = [tx.from] + result.targetKeys = [tx.groupId] + result.allKeys = [...result.sourceKeys, ...result.targetKeys] + return result +} + +export const memoryPattern = (tx: Tx.GroupLeave, result: ShardusTypes.TransactionKeys): ShardusTypes.ShardusMemoryPatternsInput => { + return { + rw: [tx.from, tx.groupId], + wo: [], + on: [], + ri: [], + ro: [], + } +} + +export const createRelevantAccount = ( + dapp: Shardus, + account: UserAccount | GroupAccount, + accountId: string, + tx: Tx.GroupLeave, + accountCreated = false, +): ShardusTypes.WrappedResponse => { + if (!account) { + throw Error('Account must exist in order to leave a group') + } + return dapp.createWrappedResponse(accountId, accountCreated, account.hash, account.timestamp, account) +} diff --git a/src/transactions/group_message.ts b/src/transactions/group_message.ts new file mode 100644 index 00000000..f80f76f4 --- /dev/null +++ b/src/transactions/group_message.ts @@ -0,0 +1,269 @@ +import * as crypto from '../crypto' +import { Shardus, ShardusTypes, nestedCountersInstance } from '@shardus/core' +import * as utils from '../utils' +import * as config from '../config' +import { UserAccount, GroupAccount, WrappedStates, Tx, AppReceiptData, TXTypes } from '../@types' +import { SafeBigIntMath } from '../utils/safeBigIntMath' +import * as AccountsStorage from '../storage/accountStorage' +import { isUserAccount, isGroupAccount } from '../@types/accountTypeGuards' + +/** + * One MLS application message appended to the group transcript. + * + * This is the hot path, so it deliberately touches only two accounts — + * [from, groupId] — exactly like a 1:1 message. Writing to every member's + * account would make a 50-member group a 50-account transaction and destroy + * sharding; members are notified by polling the group account instead. + * + * There is no toll: groups have no two-party toll analogue. Spam is bounded by + * the network fee plus a per-member send interval held in the group account. + */ +export const validate_fields = (tx: Tx.GroupMessage, response: ShardusTypes.IncomingTransactionResult): ShardusTypes.IncomingTransactionResult => { + if (utils.isValidAddress(tx.from) === false) { + response.reason = 'tx "from" is not a valid address.' + return response + } + if (utils.isValidAddress(tx.groupId) === false) { + response.reason = 'tx "groupId" is not a valid address.' + return response + } + if (typeof tx.message !== 'string' || tx.message.length === 0) { + response.reason = 'tx "message" field must be a non-empty string.' + return response + } + const messageSizeInKb = Buffer.byteLength(tx.message, 'utf8') / 1024 + if (messageSizeInKb > config.LiberdusFlags.groupMessageSizeLimit) { + response.reason = `tx "message" size must be less than ${config.LiberdusFlags.groupMessageSizeLimit} kB.` + return response + } + if (typeof tx.epoch !== 'number' || !Number.isInteger(tx.epoch) || tx.epoch < 0) { + response.reason = 'tx "epoch" must be a non-negative integer.' + return response + } + if (typeof tx.fee !== 'bigint') { + response.reason = 'tx "fee" must be a bigint.' + return response + } + if (!tx.sign || !tx.sign.owner || !tx.sign.sig || tx.sign.owner !== tx.from) { + response.reason = 'not signed by from account' + return response + } + if (crypto.verifyObj(tx) === false) { + response.reason = 'incorrect signing' + return response + } + response.success = true + return response +} + +export const validate = ( + tx: Tx.GroupMessage, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId] && wrappedStates[tx.groupId].data + + if (typeof from === 'undefined' || from === null) { + response.reason = '"from" account does not exist.' + return response + } + if (!isUserAccount(from)) { + response.reason = 'from account is not a UserAccount' + return response + } + if (typeof group === 'undefined' || group === null) { + response.reason = '"groupId" account does not exist.' + return response + } + if (!isGroupAccount(group)) { + response.reason = 'groupId account is not a GroupAccount' + return response + } + if (!group.members.includes(tx.from)) { + response.reason = 'sender is not a member of this group.' + return response + } + + /* + * The epoch is recorded but NOT enforced. + * + * A member may legitimately send at epoch N while a commit to N+1 is already + * in flight; MLS clients retain old-epoch keys and can still decrypt. Failing + * the transaction here would drop valid messages during every membership + * change. Clients use the recorded epoch to pick the right key schedule. + */ + + // Per-member send throttle. Cheap griefing defence until group economics land. + const lastSentAt = group.lastMessageAt[tx.from] || 0 + const minInterval = config.LiberdusFlags.groupMessageMinIntervalMs + if (minInterval > 0 && lastSentAt > 0 && tx.timestamp - lastSentAt < minInterval) { + response.reason = `sending too fast; minimum interval between group messages is ${minInterval}ms.` + return response + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.success = false + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < transactionFee) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + +/** + * Trims application-message history. Handshakes are deliberately untouched: + * dropping a commit a member has not yet applied would lock that member out of + * the group permanently, with no error the network can observe. + */ +const pruneMessages = (group: GroupAccount, txTimestamp: number, dapp: Shardus): void => { + const retentionDays = config.LiberdusFlags.groupMessageRetentionDays + if (retentionDays > 0) { + const cutoffTimestamp = txTimestamp - retentionDays * 24 * 60 * 60 * 1000 + const before = group.messages.length + group.messages = group.messages.filter((msg) => msg.timestamp >= cutoffTimestamp) + if (config.LiberdusFlags.VerboseLogs && before !== group.messages.length) { + dapp.log(`group messages after retention cleanup, kept ${group.messages.length} of ${before}`) + nestedCountersInstance.countEvent('liberdus-group-retention', `cleaned-up older than ${retentionDays} days`) + } + } + + const maxLength = config.LiberdusFlags.groupMessageMaxLength + if (maxLength > 0 && group.messages.length > maxLength) { + group.messages = group.messages.slice(-maxLength) + nestedCountersInstance.countEvent('liberdus-group-retention', `cleaned-up more than ${maxLength} messages`) + } +} + +export const apply = ( + tx: Tx.GroupMessage, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, +): void => { + const from: UserAccount = wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId].data + const network = AccountsStorage.cachedNetworkAccount + + const transactionFee = utils.getTransactionFeeWei(network) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + + const maintenanceFee = utils.maintenanceAmount(txTimestamp, from, network) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, maintenanceFee) + + pruneMessages(group, txTimestamp, dapp) + + const messageRecord: Tx.GroupMessageRecord = { + type: TXTypes.group_message, + txId, + from: tx.from, + groupId: tx.groupId, + epoch: tx.epoch, + message: tx.message, + timestamp: txTimestamp, + sign: tx.sign, + } + group.messages.push(messageRecord) + group.hasChats = true + group.lastMessageAt[tx.from] = txTimestamp + + // Bumping the group timestamp is what wakes every member's poller. + group.timestamp = txTimestamp + from.timestamp = txTimestamp + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: true, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { + maintenanceFee, + groupId: tx.groupId, + epoch: tx.epoch, + }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) + dapp.log('Applied group_message tx', tx.groupId, tx.from) +} + +export const createFailedAppReceiptData = ( + tx: Tx.GroupMessage, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, + reason: string, +): void => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + let transactionFee = BigInt(0) + if (from !== undefined && from !== null) { + const networkFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (from.data.balance >= networkFee) { + transactionFee = networkFee + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } else { + transactionFee = from.data.balance + from.data.balance = BigInt(0) + } + from.timestamp = txTimestamp + } + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: false, + reason, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { groupId: tx.groupId }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) +} + +export const keys = (tx: Tx.GroupMessage, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { + result.sourceKeys = [tx.from] + result.targetKeys = [tx.groupId] + result.allKeys = [...result.sourceKeys, ...result.targetKeys] + return result +} + +export const memoryPattern = (tx: Tx.GroupMessage, result: ShardusTypes.TransactionKeys): ShardusTypes.ShardusMemoryPatternsInput => { + return { + rw: [tx.from, tx.groupId], + wo: [], + on: [], + ri: [], + ro: [], + } +} + +export const createRelevantAccount = ( + dapp: Shardus, + account: UserAccount | GroupAccount, + accountId: string, + tx: Tx.GroupMessage, + accountCreated = false, +): ShardusTypes.WrappedResponse => { + if (!account) { + throw Error('Account must exist in order to send a group message') + } + return dapp.createWrappedResponse(accountId, accountCreated, account.hash, account.timestamp, account) +} diff --git a/src/transactions/index.ts b/src/transactions/index.ts index 56ba0b2f..7c444d05 100644 --- a/src/transactions/index.ts +++ b/src/transactions/index.ts @@ -57,6 +57,11 @@ import * as dao_unapply_parameters from './dao/dao_unapply_parameters' import * as dao_claim_reward from './dao/dao_claim_reward' import * as dao_burn_reward from './dao/dao_burn_reward' import * as dao_cancel from './dao/dao_cancel' +import * as group_create from './group_create' +import * as group_keypackage_publish from './group_keypackage_publish' +import * as group_message from './group_message' +import * as group_commit from './group_commit' +import * as group_leave from './group_leave' export default { init_network, @@ -118,4 +123,9 @@ export default { dao_claim_reward, dao_burn_reward, dao_cancel, + group_create, + group_keypackage_publish, + group_message, + group_commit, + group_leave, } diff --git a/src/utils/index.ts b/src/utils/index.ts index e86c2ef9..9c033a2b 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -613,6 +613,17 @@ export function calculateChatId(from: string, to: string): string { return crypto.hash([from, to].sort((a, b) => a.localeCompare(b)).join('')) } +/** + * Deterministic address for an MLS group account. + * + * The creator picks a random 32-byte nonce, so two groups created concurrently + * by the same account cannot collide. Because the id is derived client-side it + * can be named in keys() before the account exists. + */ +export function calculateGroupId(creator: string, groupNonce: string): string { + return crypto.hash(`${creator.toLowerCase()}${groupNonce.toLowerCase()}`) +} + export function validateTxTimestamp(txnTimestamp: number): { success: boolean; reason: string } { const validationResult = { success: false, reason: '' } try { From b4a4e3e08a02bd5817f4c5c7886e0cc8589b29f8 Mon Sep 17 00:00:00 2001 From: Thant Sin Toe Date: Thu, 20 Aug 2026 18:28:36 +0700 Subject: [PATCH 2/7] group request and group join logic --- src/@types/accountTypeGuards.ts | 5 + src/@types/index.ts | 207 +++++++++- src/@types/transactionSchemas.ts | 68 +++- src/accounts/groupAccount.ts | 28 +- src/accounts/groupTreeAccount.ts | 107 ++++++ src/accounts/index.ts | 10 + src/api/group/group.ts | 151 +++++++- src/api/group/index.ts | 4 +- src/api/index.ts | 2 + src/config/index.ts | 67 +++- src/transactions/group_commit.ts | 394 +++++++++++++++++++- src/transactions/group_create.ts | 15 + src/transactions/group_fee_claim.ts | 198 ++++++++++ src/transactions/group_join_reclaim.ts | 196 ++++++++++ src/transactions/group_join_request.ts | 259 +++++++++++++ src/transactions/group_leave.ts | 8 +- src/transactions/index.ts | 8 + src/transactions/update_group_add_policy.ts | 178 +++++++++ src/utils/index.ts | 12 + 19 files changed, 1863 insertions(+), 54 deletions(-) create mode 100644 src/accounts/groupTreeAccount.ts create mode 100644 src/transactions/group_fee_claim.ts create mode 100644 src/transactions/group_join_reclaim.ts create mode 100644 src/transactions/group_join_request.ts create mode 100644 src/transactions/update_group_add_policy.ts diff --git a/src/@types/accountTypeGuards.ts b/src/@types/accountTypeGuards.ts index 1fa6f9c1..80a39df8 100644 --- a/src/@types/accountTypeGuards.ts +++ b/src/@types/accountTypeGuards.ts @@ -12,6 +12,7 @@ import { DaoProposalsMeta, DaoProposalAccount, GroupAccount, + GroupTreeAccount, } from '.' /** @@ -35,6 +36,10 @@ export function isGroupAccount(account: unknown): account is GroupAccount { return !!account && typeof account === 'object' && 'type' in account && account.type === 'GroupAccount' } +export function isGroupTreeAccount(account: unknown): account is GroupTreeAccount { + return !!account && typeof account === 'object' && 'type' in account && account.type === 'GroupTreeAccount' +} + /** * Type guard to check if an account is an AliasAccount */ diff --git a/src/@types/index.ts b/src/@types/index.ts index 8acbdaf1..768c8a2d 100644 --- a/src/@types/index.ts +++ b/src/@types/index.ts @@ -160,6 +160,10 @@ export enum TXTypes { group_message = 'group_message', group_commit = 'group_commit', group_leave = 'group_leave', + update_group_add_policy = 'update_group_add_policy', + group_join_request = 'group_join_request', + group_join_reclaim = 'group_join_reclaim', + group_fee_claim = 'group_fee_claim', } export interface BaseLiberdusTx { @@ -321,6 +325,8 @@ export namespace Tx { cipherSuite: number // pinned; members must agree meta: string // client-encrypted {name, avatar, ...} maxMembers: number + /** Price of admission, escrowed by a requester and earned by the approving admin. */ + joinFee: bigint fee: bigint } @@ -364,7 +370,21 @@ export namespace Tx { pskNonce: string // b64 welcomes: { address: string; envelope: GroupWelcomeEnvelope }[] groupInfo: string // b64 GroupInfo w/ external_pub, for recovery - ratchetTree: string // b64 post-commit tree + /** + * BASELINE ONLY: the full b64 post-commit tree. Sent on a group's first + * commit, or once per group when migrating an existing group onto the + * delta scheme. Empty on every other commit — see `treeDelta`. + */ + ratchetTree: string + /** + * Ratchet-tree nodes this commit changed, by node index (even = leaf, + * odd = parent); `n: null` blanks the node. + * + * Must be an explicit field rather than something the server derives: our + * commits are mls_private_message, so the UpdatePath inside them is + * encrypted and the network cannot read it. + */ + treeDelta: { i: number; n: string | null }[] addedMembers: string[] removedMembers: string[] consumedKeyPackages: { address: string; keyPackage: string }[] @@ -390,6 +410,45 @@ export namespace Tx { } /** Self-removal. Not cryptographically effective until an admin commits a Remove. */ + /** + * Asks to join a group. The requester's own consent to be added, which + * group_commit then requires — nobody can be pulled into a group they did not + * ask for. + * + * Carries NO KeyPackage: the approving commit draws one from the requester's + * published pool. Pinning one here would break if the requester rotated their + * pool while the request was pending, because publishing discards the private + * halves — the Welcome would be undecryptable and they could never join. + */ + export interface GroupJoinRequest extends BaseLiberdusTx { + from: string + groupId: string + /** The group's joinFee at request time, debited now and held on the group. */ + escrow: bigint + message: string + fee: bigint + } + + /** Collects join fees that have finished vesting. */ + export interface GroupFeeClaim extends BaseLiberdusTx { + from: string + groupId: string + fee: bigint + } + + /** Withdraws a join request and returns its escrow. Modelled on reclaim_toll. */ + export interface GroupJoinReclaim extends BaseLiberdusTx { + from: string + groupId: string + fee: bigint + } + + export interface UpdateGroupAddPolicy extends BaseLiberdusTx { + from: string + policy: 'anyone' | 'contacts' | 'nobody' + fee: bigint + } + export interface GroupLeave extends BaseLiberdusTx { from: string groupId: string @@ -741,6 +800,19 @@ export interface UserAccount { mlsLastResortKeyPackage?: string /** Ciphersuite the published KeyPackages were generated for. */ mlsCipherSuite?: number + /** + * Who may add this account to a group WITHOUT it having asked to join. + * + * 'contacts' (default) - only accounts this one is connected to, i.e. has + * waived its chat toll for (toll.required === 0) + * 'anyone' - anybody, i.e. the pre-consent behaviour + * 'nobody' - direct adds refused; join requests only + * + * Being added is not free for the addee: it consumes one of their single-use + * KeyPackages and, under update-on-join, makes them inject a group_commit of + * their own. So the default is restrictive. + */ + groupAddPolicy?: 'anyone' | 'contacts' | 'nobody' } alias: string | null emailHash: string | null @@ -862,6 +934,65 @@ export interface GroupAccount { // --- transcript ----------------------------------------------------------- /** Application messages. Safe to prune: losing them costs history only. */ messages: Tx.GroupMessageRecord[] + /** + * Commits, welcomes, the ratchet tree and the recovery checkpoint all live on + * the GroupTreeAccount instead — see `treeId`. They are needed only by + * group_commit, whereas THIS account is loaded, shipped to the consensus group + * and re-hashed by every single group_message. Keeping them here made the hot + * path carry megabytes it never reads. + */ + treeId: string + + // --- admission ------------------------------------------------------------ + /** + * Price of admission, escrowed by the requester and earned by the approving + * admin. Zero until paid groups ship; the plumbing exists so enabling them is + * a matter of allowing this to be set. + */ + joinFee: bigint + /** Addresses refused admission. The group's analogue of toll.required = 2. */ + blocked: string[] + + // --- misc ----------------------------------------------------------------- + meta: string // client-encrypted group name/avatar; opaque here + maxMembers: number + /** Per-member send throttle, address -> last group_message timestamp. */ + lastMessageAt: { [address: string]: number } + createdBy: string + hasChats: boolean +} + +/** + * The cold half of a group: everything group_commit needs and group_message does + * not. + * + * Split out because `keys()` for a message names only the GroupAccount, so any + * byte stored there is transferred and re-hashed on every message. The ratchet + * tree alone is ~112 kB at 32 members, and `handshakes` grows without bound. + * + * Its id is hash(groupId + 'ratchet-tree') — deterministic so it can be named in + * keys() before the account exists, and domain-separated so it cannot collide + * with a user address (hash(username)) or a group id (hash(creator + nonce)). + */ +export interface GroupTreeAccount { + id: string + type: string + hash: string + timestamp: number + + /** Back-reference, so the pair can be validated as belonging together. */ + groupId: string + + // --- ratchet tree --------------------------------------------------------- + /** + * b64 encodeRatchetTree of the CURRENT tree, maintained by applying each + * commit's `treeDelta`. Public key material only — never group state. + */ + ratchetTree: string + /** Epoch `ratchetTree` corresponds to. Invariant: equals GroupAccount.epoch. */ + treeEpoch: number + + // --- transcript ----------------------------------------------------------- /** * Commits. MUST NOT be pruned on the ordinary retention timer — dropping a * commit a member has not applied locks that member out of the group forever. @@ -870,24 +1001,72 @@ export interface GroupAccount { */ handshakes: Tx.GroupCommitRecord[] - /** Welcome + sealed PQ PSK awaiting collection by each newly added member. */ - pendingWelcomes: { [address: string]: Tx.GroupWelcomeEnvelope } + /** + * Welcome + sealed PQ PSK awaiting collection by each newly added member, + * tagged with who added them so the invitee can be told before deciding. + */ + pendingWelcomes: { [address: string]: Tx.GroupWelcomeEnvelope & { addedBy?: string } } + + /** + * Ratchet tree snapshots, keyed by the epoch they belong to. + * + * A joiner needs the tree matching the GroupContext in its Welcome, but the + * live tree moves on immediately — under update-on-join the joiner's own path + * update is the very next commit. So the server keeps a snapshot, taken at + * zero transaction cost from the tree it already maintains. + * + * Keyed by EPOCH, not by address: every joiner added in one commit shares the + * same tree, and at 100 members a tree is ~354 kB. Storing one per joiner made + * a 10-member add write 3.5 MB. Entries are dropped as soon as no pending + * welcome refers to them. + */ + welcomeTrees: { [epoch: string]: string } + + /** + * Outstanding requests to join, by requester address. + * + * Lives here rather than on the GroupAccount because only group_commit reads + * it — decision 4 exists to keep anything else off the account that every + * group_message transfers and re-hashes. + */ + pendingJoinRequests: { + [address: string]: { + /** Debited from the requester at request time; theirs until approved. */ + escrow: bigint + message: string + timestamp: number + } + } + + /** + * Join fees earned but not yet payable. + * + * An approved fee does NOT land in the admin's balance immediately. It waits + * until `vestingUntil`, so that a member removed before then can be refunded + * rather than having to claw money back from someone who may already have + * spent it. See GROUP_MEMBERSHIP_CONSENT_SPEC §5.2. + */ + vestedFees: { + /** The admin who approved, and is owed this. */ + admin: string + /** The member who paid it, and who gets it back on an early removal. */ + member: string + amount: bigint + vestingUntil: number + }[] // --- recovery ------------------------------------------------------------- + /** + * Recovery pointer for a desynced member. Deliberately carries NO tree: it is + * rewritten on every commit, so its epoch always equals `treeEpoch` and a tree + * here would duplicate `ratchetTree` byte for byte. Read `ratchetTree` when + * `checkpoint.epoch === treeEpoch`. + */ checkpoint: { epoch: number groupInfo: string // b64 GroupInfo with external_pub - ratchetTree: string // b64 timestamp: number } | null - - // --- misc ----------------------------------------------------------------- - meta: string // client-encrypted group name/avatar; opaque here - maxMembers: number - /** Per-member send throttle, address -> last group_message timestamp. */ - lastMessageAt: { [address: string]: number } - createdBy: string - hasChats: boolean } export interface AliasAccount { @@ -1090,7 +1269,8 @@ export type Accounts = NetworkAccount & ChatAccount & DaoProposalsMeta & DaoProposalAccount & - GroupAccount + GroupAccount & + GroupTreeAccount export type AccountVariant = | NetworkAccount @@ -1106,6 +1286,7 @@ export type AccountVariant = | DaoProposalsMeta | DaoProposalAccount | GroupAccount + | GroupTreeAccount /** * ---------------------- NETWORK DATA export interfaceS ---------------------- diff --git a/src/@types/transactionSchemas.ts b/src/@types/transactionSchemas.ts index 4fb00832..e2a15384 100644 --- a/src/@types/transactionSchemas.ts +++ b/src/@types/transactionSchemas.ts @@ -314,9 +314,10 @@ export const schemaGroupCreateTX = { mlsGroupId: { type: 'string', minLength: 1 }, cipherSuite: { type: 'number', minimum: 1 }, meta: { type: 'string' }, + joinFee: { type: 'string' }, maxMembers: { type: 'number', minimum: 1 }, }, - required: [...groupBaseRequired, 'from', 'groupId', 'groupNonce', 'mlsGroupId', 'cipherSuite', 'meta', 'maxMembers'], + required: [...groupBaseRequired, 'from', 'groupId', 'groupNonce', 'mlsGroupId', 'cipherSuite', 'meta', 'maxMembers', 'joinFee'], additionalProperties: false, } @@ -371,6 +372,20 @@ export const schemaGroupCommitTX = { }, groupInfo: { type: 'string' }, ratchetTree: { type: 'string' }, + // Ratchet-tree nodes this commit changed, by node index. `n: null` blanks a + // node, so the type is nullable rather than string. + treeDelta: { + type: 'array', + items: { + type: 'object', + properties: { + i: { type: 'number', minimum: 0 }, + n: { type: ['string', 'null'] }, + }, + required: ['i', 'n'], + additionalProperties: false, + }, + }, addedMembers: { type: 'array', items: { type: 'string' } }, removedMembers: { type: 'array', items: { type: 'string' } }, consumedKeyPackages: { @@ -399,6 +414,7 @@ export const schemaGroupCommitTX = { 'welcomes', 'groupInfo', 'ratchetTree', + 'treeDelta', 'addedMembers', 'removedMembers', 'consumedKeyPackages', @@ -406,6 +422,52 @@ export const schemaGroupCommitTX = { additionalProperties: false, } +export const schemaGroupJoinRequestTX = { + type: 'object', + properties: { + ...groupBaseProperties, + from: { type: 'string' }, + groupId: { type: 'string', minLength: 64, maxLength: 64 }, + escrow: { type: 'string' }, // bigint on the wire + message: { type: 'string' }, + }, + required: [...groupBaseRequired, 'from', 'groupId', 'escrow', 'message'], + additionalProperties: false, +} + +export const schemaGroupFeeClaimTX = { + type: 'object', + properties: { + ...groupBaseProperties, + from: { type: 'string' }, + groupId: { type: 'string', minLength: 64, maxLength: 64 }, + }, + required: [...groupBaseRequired, 'from', 'groupId'], + additionalProperties: false, +} + +export const schemaGroupJoinReclaimTX = { + type: 'object', + properties: { + ...groupBaseProperties, + from: { type: 'string' }, + groupId: { type: 'string', minLength: 64, maxLength: 64 }, + }, + required: [...groupBaseRequired, 'from', 'groupId'], + additionalProperties: false, +} + +export const schemaUpdateGroupAddPolicyTX = { + type: 'object', + properties: { + ...groupBaseProperties, + from: { type: 'string' }, + policy: { type: 'string', enum: ['anyone', 'contacts', 'nobody'] }, + }, + required: [...groupBaseRequired, 'from', 'policy'], + additionalProperties: false, +} + export const schemaGroupLeaveTX = { type: 'object', properties: { @@ -1096,6 +1158,10 @@ function addSchemas(): void { [TXTypes.group_message]: schemaGroupMessageTX, [TXTypes.group_commit]: schemaGroupCommitTX, [TXTypes.group_leave]: schemaGroupLeaveTX, + [TXTypes.update_group_add_policy]: schemaUpdateGroupAddPolicyTX, + [TXTypes.group_join_request]: schemaGroupJoinRequestTX, + [TXTypes.group_join_reclaim]: schemaGroupJoinReclaimTX, + [TXTypes.group_fee_claim]: schemaGroupFeeClaimTX, [TXTypes.read]: schemaReadTX, [TXTypes.reclaim_toll]: schemeReclaimTollTX, [TXTypes.update_chat_toll]: schemaUpdateChatTollTX, diff --git a/src/accounts/groupAccount.ts b/src/accounts/groupAccount.ts index 721315b0..08b0dc58 100644 --- a/src/accounts/groupAccount.ts +++ b/src/accounts/groupAccount.ts @@ -1,4 +1,5 @@ import * as crypto from '../crypto' +import * as utils from '../utils' import { VectorBufferStream } from '@shardus/core' import { Utils } from '@shardus/lib-types' import { SerdeTypeIdent } from '.' @@ -31,10 +32,13 @@ export const groupAccount = (accountId: string, tx: Tx.GroupCreate, timestamp: n memberSince: { [from]: { epoch: 0, timestamp } }, messages: [], - handshakes: [], - pendingWelcomes: {}, - checkpoint: null, + // The tree, commit transcript, welcomes and checkpoint live on the paired + // GroupTreeAccount so that group_message never has to carry them. + treeId: utils.calculateGroupTreeId(accountId), + + joinFee: tx.joinFee ?? BigInt(0), + blocked: [], meta: tx.meta, maxMembers: tx.maxMembers, @@ -80,10 +84,10 @@ export const serializeGroupAccount = (stream: VectorBufferStream, inp: GroupAcco stream.writeString(Utils.safeStringify(inp.memberSince)) stream.writeString(Utils.safeStringify(inp.messages)) - stream.writeString(Utils.safeStringify(inp.handshakes)) - stream.writeString(Utils.safeStringify(inp.pendingWelcomes)) - stream.writeString(Utils.safeStringify(inp.checkpoint)) stream.writeString(Utils.safeStringify(inp.lastMessageAt)) + stream.writeString(inp.treeId) + stream.writeString(inp.joinFee.toString()) + stream.writeString(Utils.safeStringify(inp.blocked)) stream.writeString(inp.meta) stream.writeUInt32(inp.maxMembers) @@ -119,10 +123,10 @@ export const deserializeGroupAccount = (stream: VectorBufferStream, root = false const memberSince = Utils.safeJsonParse(stream.readString()) const messages = Utils.safeJsonParse(stream.readString()) - const handshakes = Utils.safeJsonParse(stream.readString()) - const pendingWelcomes = Utils.safeJsonParse(stream.readString()) - const checkpoint = Utils.safeJsonParse(stream.readString()) const lastMessageAt = Utils.safeJsonParse(stream.readString()) + const treeId = stream.readString() + const joinFee = BigInt(stream.readString()) + const blocked = Utils.safeJsonParse(stream.readString()) const meta = stream.readString() const maxMembers = stream.readUInt32() @@ -141,9 +145,9 @@ export const deserializeGroupAccount = (stream: VectorBufferStream, root = false admins, memberSince, messages, - handshakes, - pendingWelcomes, - checkpoint, + treeId, + joinFee, + blocked, meta, maxMembers, lastMessageAt, diff --git a/src/accounts/groupTreeAccount.ts b/src/accounts/groupTreeAccount.ts new file mode 100644 index 00000000..db7cdb86 --- /dev/null +++ b/src/accounts/groupTreeAccount.ts @@ -0,0 +1,107 @@ +import * as crypto from '../crypto' +import { VectorBufferStream } from '@shardus/core' +import { Utils } from '@shardus/lib-types' +import { SerdeTypeIdent } from '.' +import { GroupTreeAccount } from '../@types' + +/** + * The cold half of a group. + * + * group_message names only the GroupAccount in keys(), so everything stored + * there is transferred to the consensus group and re-hashed on every message. + * The ratchet tree (~112 kB at 32 members) and the unbounded commit transcript + * are needed only by group_commit, so they live here instead. + * + * Created alongside the GroupAccount by group_create, at the deterministic + * address utils.calculateGroupTreeId(groupId). + */ +export const groupTreeAccount = (accountId: string, groupId: string): GroupTreeAccount => { + const account: GroupTreeAccount = { + id: accountId.toLowerCase(), + type: 'GroupTreeAccount', + hash: '', + // Shardus requires a new account to start at timestamp 0; the applying + // transaction sets the real one. + timestamp: 0, + + groupId: groupId.toLowerCase(), + + ratchetTree: '', + treeEpoch: 0, + + handshakes: [], + pendingWelcomes: {}, + welcomeTrees: {}, + pendingJoinRequests: {}, + vestedFees: [], + checkpoint: null, + } + + account.hash = crypto.hashObj(account) + return account +} + +/** + * The variable-length parts are written as JSON rather than field-by-field, for + * the same reason as GroupAccount: they are opaque base64 blobs whose shape is + * driven by the client's MLS stack. + */ +export const serializeGroupTreeAccount = (stream: VectorBufferStream, inp: GroupTreeAccount, root = false): void => { + if (root) { + stream.writeUInt16(SerdeTypeIdent.GroupTreeAccount) + } + + stream.writeString(inp.id) + stream.writeString(inp.type) + stream.writeString(inp.hash) + stream.writeBigUInt64(BigInt(inp.timestamp)) + + stream.writeString(inp.groupId) + stream.writeString(inp.ratchetTree) + stream.writeUInt32(inp.treeEpoch) + + stream.writeString(Utils.safeStringify(inp.handshakes)) + stream.writeString(Utils.safeStringify(inp.pendingWelcomes)) + stream.writeString(Utils.safeStringify(inp.welcomeTrees)) + stream.writeString(Utils.safeStringify(inp.pendingJoinRequests)) + stream.writeString(Utils.safeStringify(inp.vestedFees)) + stream.writeString(Utils.safeStringify(inp.checkpoint)) +} + +export const deserializeGroupTreeAccount = (stream: VectorBufferStream, root = false): GroupTreeAccount => { + if (root && stream.readUInt16() !== SerdeTypeIdent.GroupTreeAccount) { + throw new Error('Unexpected bufferstream for GroupTreeAccount type') + } + + const id = stream.readString() + const type = stream.readString() + const hash = stream.readString() + const timestamp = Number(stream.readBigUInt64()) + + const groupId = stream.readString() + const ratchetTree = stream.readString() + const treeEpoch = stream.readUInt32() + + const handshakes = Utils.safeJsonParse(stream.readString()) + const pendingWelcomes = Utils.safeJsonParse(stream.readString()) + const welcomeTrees = Utils.safeJsonParse(stream.readString()) + const pendingJoinRequests = Utils.safeJsonParse(stream.readString()) + const vestedFees = Utils.safeJsonParse(stream.readString()) + const checkpoint = Utils.safeJsonParse(stream.readString()) + + return { + id, + type, + hash, + timestamp, + groupId, + ratchetTree, + treeEpoch, + handshakes, + pendingWelcomes, + welcomeTrees, + pendingJoinRequests, + vestedFees, + checkpoint, + } +} diff --git a/src/accounts/index.ts b/src/accounts/index.ts index df91089e..efddd2e0 100644 --- a/src/accounts/index.ts +++ b/src/accounts/index.ts @@ -11,6 +11,7 @@ import { deserializeProposalAccount, proposalAccount, serializeProposalAccount } import { daoProposalsMetaAccount, deserializeDaoProposalsMetaAccount, serializeDaoProposalsMetaAccount } from './daoProposalsMetaAccount' import { daoProposalAccount, deserializeDaoProposalAccount, serializeDaoProposalAccount } from './daoProposalAccount' import { deserializeGroupAccount, groupAccount, serializeGroupAccount } from './groupAccount' +import { deserializeGroupTreeAccount, groupTreeAccount, serializeGroupTreeAccount } from './groupTreeAccount' import { VectorBufferStream } from '@shardus/core' import { DeveloperPayment, @@ -28,6 +29,7 @@ import { DaoProposalsMeta, DaoProposalAccount, GroupAccount, + GroupTreeAccount, } from '../@types' import { Utils } from '@shardus/lib-types' @@ -48,6 +50,8 @@ export enum SerdeTypeIdent { DaoProposalsMeta, DaoProposalAccount, GroupAccount, + // Appended, never reordered: these ordinals are on the wire. + GroupTreeAccount, } export const serializeAccounts = (inp: AccountVariant): VectorBufferStream => { @@ -96,6 +100,9 @@ export const serializeAccounts = (inp: AccountVariant): VectorBufferStream => { case 'GroupAccount': serializeGroupAccount(stream, inp as GroupAccount, true) break + case 'GroupTreeAccount': + serializeGroupTreeAccount(stream, inp as GroupTreeAccount, true) + break default: fallbackSerializer(stream, inp, true) break @@ -134,6 +141,8 @@ export const deserializeAccounts = (buffer: Buffer): AccountVariant => { return deserializeDaoProposalAccount(stream) case SerdeTypeIdent.GroupAccount: return deserializeGroupAccount(stream) + case SerdeTypeIdent.GroupTreeAccount: + return deserializeGroupTreeAccount(stream) default: return fallbackDeserializer(stream) } @@ -231,4 +240,5 @@ export default { daoProposalsMetaAccount, daoProposalAccount, groupAccount, + groupTreeAccount, } diff --git a/src/api/group/group.ts b/src/api/group/group.ts index 556b5163..dbd82d37 100644 --- a/src/api/group/group.ts +++ b/src/api/group/group.ts @@ -1,6 +1,7 @@ import { Shardus } from '@shardus/core' -import { GroupAccount, UserAccount } from '../../@types' -import { isGroupAccount, isUserAccount } from '../../@types/accountTypeGuards' +import { GroupAccount, GroupTreeAccount, UserAccount } from '../../@types' +import { isGroupAccount, isGroupTreeAccount, isUserAccount } from '../../@types/accountTypeGuards' +import * as utils from '../../utils' /** * Read endpoints for MLS group chat. @@ -18,6 +19,18 @@ const loadGroup = async (dapp: Shardus, groupId: string): Promise => { + const account = await dapp.getLocalOrRemoteAccount(utils.calculateGroupTreeId(groupId)) + if (!account || !account.data) return null + const tree = account.data as unknown as GroupTreeAccount + return isGroupTreeAccount(tree) ? tree : null +} + /** GET /group/:groupId — metadata only; no transcript. */ export const info = (dapp: Shardus) => @@ -43,9 +56,9 @@ export const info = createdBy: group.createdBy, hasChats: group.hasChats, timestamp: group.timestamp, - checkpointEpoch: group.checkpoint ? group.checkpoint.epoch : null, messageCount: group.messages.length, - handshakeCount: group.handshakes.length, + treeId: group.treeId, + joinFee: group.joinFee.toString(), }, }) } catch (error) { @@ -96,12 +109,17 @@ export const handshakes = res.json({ error: 'No group with the given id' }) return } - const available = group.handshakes.map((h) => h.epoch) + const tree = await loadTree(dapp, groupId) + if (!tree) { + res.json({ handshakes: [], epoch: group.epoch, oldestAvailableEpoch: group.epoch, checkpoint: null }) + return + } + const available = tree.handshakes.map((h) => h.epoch) res.json({ - handshakes: group.handshakes.filter((h) => h.epoch >= fromEpoch), + handshakes: tree.handshakes.filter((h) => h.epoch >= fromEpoch), epoch: group.epoch, oldestAvailableEpoch: available.length > 0 ? Math.min(...available) : group.epoch, - checkpoint: group.checkpoint, + checkpoint: tree.checkpoint, }) } catch (error) { dapp.log(error) @@ -121,12 +139,122 @@ export const welcome = res.json({ error: 'No group with the given id' }) return } - const envelope = group.pendingWelcomes[address] + const tree = await loadTree(dapp, groupId) + const envelope = tree && tree.pendingWelcomes[address] if (!envelope) { res.json({ error: 'No pending welcome for this address' }) return } - res.json({ welcome: envelope, cipherSuite: group.cipherSuite, mlsGroupId: group.mlsGroupId }) + /* + * Attach the tree snapshot for this welcome's epoch. It is stored once per + * epoch rather than per joiner (a tree is ~354 kB at 100 members), and is + * merged in here so the client sees one self-contained envelope. + * + * A missing snapshot means the group moved on before this member joined — + * the joiner cannot use the live tree, because it no longer matches the + * GroupContext in its Welcome, and must be added again. + */ + const ratchetTree = tree.welcomeTrees[String(envelope.epoch)] + if (!ratchetTree) { + res.json({ error: 'The ratchet tree for this welcome is no longer available; ask to be added again' }) + return + } + res.json({ + welcome: { ...envelope, ratchetTree }, + cipherSuite: group.cipherSuite, + mlsGroupId: group.mlsGroupId, + }) + } catch (error) { + dapp.log(error) + res.json({ error }) + } + } + +/** + * GET /group/:groupId/tree — the current ratchet tree. + * + * This is what replaces shipping a full tree inside every Welcome envelope. It + * is public key material only, and a joiner MUST verify it against the + * `tree_hash` in the GroupContext of its Welcome (RFC 9420) rather than trusting + * what the network returns. + * + * `treeEpoch` is the epoch this tree corresponds to. A joiner whose Welcome + * names an earlier epoch must use the snapshot in its own welcome envelope + * instead, since the live tree has already moved on. + */ +export const tree = + (dapp: Shardus) => + async (req, res): Promise => { + try { + const groupId = req.params['groupId'] + const group = await loadGroup(dapp, groupId) + if (!group) { + res.json({ error: 'No group with the given id' }) + return + } + const treeAccount = await loadTree(dapp, groupId) + if (!treeAccount) { + res.json({ error: 'No ratchet tree stored for this group yet' }) + return + } + res.json({ + ratchetTree: treeAccount.ratchetTree, + treeEpoch: treeAccount.treeEpoch, + epoch: group.epoch, + cipherSuite: group.cipherSuite, + }) + } catch (error) { + dapp.log(error) + res.json({ error }) + } + } + +/** + * GET /group/:groupId/requests — outstanding requests to join. + * + * Read by admins to decide who to admit. Public: the roster and member count + * already are, and a would-be member can reasonably check whether their own + * request is still pending before reclaiming it. + */ +export const joinRequests = + (dapp: Shardus) => + async (req, res): Promise => { + try { + const groupId = req.params['groupId'] + const group = await loadGroup(dapp, groupId) + if (!group) { + res.json({ error: 'No group with the given id' }) + return + } + const tree = await loadTree(dapp, groupId) + const requests = tree + ? Object.entries(tree.pendingJoinRequests).map(([address, r]) => ({ + address, + message: r.message, + escrow: r.escrow.toString(), + timestamp: r.timestamp, + })) + : [] + /* + * Vested fees are reported alongside, so an admin's client can show what + * is claimable without a second round trip. `matured` is relative to the + * caller's clock only for display; the claim transaction recomputes it + * from the transaction timestamp so validators agree. + */ + const now = Date.now() + const fees = tree ? tree.vestedFees || [] : [] + res.json({ + requests, + joinFee: group.joinFee.toString(), + epoch: group.epoch, + vestedFees: fees.map((v) => ({ + admin: v.admin, + member: v.member, + amount: v.amount.toString(), + vestingUntil: v.vestingUntil, + matured: v.vestingUntil <= now, + })), + }) } catch (error) { dapp.log(error) res.json({ error }) @@ -144,11 +272,12 @@ export const checkpoint = res.json({ error: 'No group with the given id' }) return } - if (!group.checkpoint) { + const tree = await loadTree(dapp, groupId) + if (!tree || !tree.checkpoint) { res.json({ error: 'No checkpoint yet for this group' }) return } - res.json({ checkpoint: group.checkpoint, epoch: group.epoch, cipherSuite: group.cipherSuite }) + res.json({ checkpoint: tree.checkpoint, epoch: group.epoch, cipherSuite: group.cipherSuite }) } catch (error) { dapp.log(error) res.json({ error }) diff --git a/src/api/group/index.ts b/src/api/group/index.ts index 137f7a89..aacbf159 100644 --- a/src/api/group/index.ts +++ b/src/api/group/index.ts @@ -1,10 +1,12 @@ -import { info, messages, handshakes, welcome, checkpoint, keyPackages, accountGroups } from './group' +import { info, messages, handshakes, welcome, tree, joinRequests, checkpoint, keyPackages, accountGroups } from './group' export default { info, messages, handshakes, welcome, + tree, + joinRequests, checkpoint, keyPackages, accountGroups, diff --git a/src/api/index.ts b/src/api/index.ts index 3efbd502..a5606af3 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -57,6 +57,8 @@ export default (dapp: Shardus): void => { dapp.registerExternalGet('group/:groupId', group.info(dapp)) dapp.registerExternalGet('group/:groupId/checkpoint', group.checkpoint(dapp)) dapp.registerExternalGet('group/:groupId/messages/:timestamp', group.messages(dapp)) + dapp.registerExternalGet('group/:groupId/tree', group.tree(dapp)) + dapp.registerExternalGet('group/:groupId/requests', group.joinRequests(dapp)) dapp.registerExternalGet('group/:groupId/handshakes/:epoch', group.handshakes(dapp)) dapp.registerExternalGet('group/:groupId/welcome/:address', group.welcome(dapp)) dapp.registerExternalGet('account/:id/keypackages', group.keyPackages(dapp)) diff --git a/src/config/index.ts b/src/config/index.ts index 37e76de5..cd7f06b0 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -24,6 +24,22 @@ const daoVotingDurationMs = process.env.DAO_VOTING_DURATION_MS ? Number(process. const daoGraceDurationMs = process.env.DAO_GRACE_DURATION_MS ? Number(process.env.DAO_GRACE_DURATION_MS) : 7 * ONE_DAY const daoClaimDurationMs = process.env.DAO_CLAIM_DURATION_MS ? Number(process.env.DAO_CLAIM_DURATION_MS) : 30 * ONE_DAY +/* + * Group chat durations, overridable via env for local/E2E testing, exactly as + * the DAO phases above are. Without this a tester cannot exercise fee + * collection at all: a 7-day vest means the Collect button reads "nothing to + * collect yet" for a week. + * + * GROUP_JOIN_FEE_VESTING_MS=60000 # 1 minute + * GROUP_JOIN_REQUEST_TIMEOUT_MS=60000 + */ +const groupJoinFeeVestingMs = process.env.GROUP_JOIN_FEE_VESTING_MS + ? Number(process.env.GROUP_JOIN_FEE_VESTING_MS) + : 7 * ONE_DAY +const groupJoinRequestTimeoutMs = process.env.GROUP_JOIN_REQUEST_TIMEOUT_MS + ? Number(process.env.GROUP_JOIN_REQUEST_TIMEOUT_MS) + : 7 * ONE_DAY + // MIGHT BE USEFUL TO HAVE TIME CONSTANTS IN THE FORM OF CYCLES export const cycleDuration = process.env.CYCLE_DURATION ? Number(process.env.CYCLE_DURATION) : 60 const reduceTimeFromTxTimestamp = cycleDuration * ONE_SECOND @@ -289,6 +305,13 @@ interface LiberdusFlags { groupMessageMinIntervalMs: number /** Max unconsumed KeyPackages an account may hold at once. */ groupMaxKeyPackagesPerAccount: number + groupMaxHandshakes: number + groupDefaultAddPolicy: 'anyone' | 'contacts' | 'nobody' + groupMaxPendingJoinRequests: number + groupJoinRequestTimeoutMs: number + groupMaxJoinRequestMessageLength: number + groupJoinFeeVestingMs: number + groupMaxJoinFeeUsdStr: string versionFlags: { replierNoToll: boolean allowZeroToll: boolean @@ -344,11 +367,53 @@ export const LiberdusFlags: LiberdusFlags = { enableGroupChat: true, groupMaxMembers: 50, groupMaxMembersPerCommit: 10, - groupMessageSizeLimit: 64, // 64 kB; an X-Wing commit is ~5.5 kB + // 64 kB. Measured, not estimated: the commit blob is ~3.9 kB on an add and + // ~0.4 kB on a remove, and does NOT grow with the group. What grows is the + // ratchet tree carried in each welcome — ~1.8 kB per existing member — so in + // practice this limit caps how large a group can be when adding a member. + groupMessageSizeLimit: 64, groupMessageMaxLength: 500, groupMessageRetentionDays: 7, groupMessageMinIntervalMs: 1000, groupMaxKeyPackagesPerAccount: 10, // X-Wing KeyPackages are ~2.6 kB each + /* + * How many commits the group transcript keeps. + * + * Pruning a commit locks out any member that has not applied it — they must + * reset and be re-added — so this is a deliberate trade of history for a + * bounded account. Measured at 100 members: a handshake record averages ~9 kB + * and the ratchet tree is ~354 kB, so 50 records keeps the GroupTreeAccount + * near 800 kB in steady state. Clients see `oldestAvailableEpoch` on the + * handshakes endpoint and surface a reset when they fall behind it. + */ + groupMaxHandshakes: 50, + /* + * Network default for accounts that have not set `groupAddPolicy`. + * + * 'contacts' is the safe production value: being added costs the addee a + * KeyPackage and, under update-on-join, a transaction of their own. Set + * 'anyone' for local testing where accounts are not mutual contacts. + */ + groupDefaultAddPolicy: 'contacts' as 'anyone' | 'contacts' | 'nobody', + /* + * Join requests are the one place a stranger can write to a group's account. + * With a joinFee of zero nothing is escrowed, so escrow deters nothing and + * this cap plus the timeout are the only defence against a group being + * flooded. They are load-bearing, not tuning knobs. + */ + groupMaxPendingJoinRequests: 100, + groupJoinRequestTimeoutMs, + groupMaxJoinRequestMessageLength: 200, + /* + * How long an approved join fee waits before the admin can collect it. + * + * The window in which removing the member refunds them instead. It does not + * stop an admin who simply waits it out — that part is irreducibly + * reputational — but it removes the cheap "take the fee, remove them" scam. + */ + groupJoinFeeVestingMs, + /** Upper bound on what a group may charge, as a sanity check on typos. */ + groupMaxJoinFeeUsdStr: '1000.0', versionFlags: { replierNoToll: true, // turn on by 2.3.5 allowZeroToll: true, // turn on by 2.3.6 diff --git a/src/transactions/group_commit.ts b/src/transactions/group_commit.ts index e0f6f057..24fa5428 100644 --- a/src/transactions/group_commit.ts +++ b/src/transactions/group_commit.ts @@ -2,10 +2,11 @@ import * as crypto from '../crypto' import { Shardus, ShardusTypes } from '@shardus/core' import * as utils from '../utils' import * as config from '../config' -import { UserAccount, GroupAccount, WrappedStates, Tx, AppReceiptData, TXTypes } from '../@types' +import { UserAccount, GroupAccount, GroupTreeAccount, ChatAccount, WrappedStates, Tx, AppReceiptData, TXTypes } from '../@types' import { SafeBigIntMath } from '../utils/safeBigIntMath' import * as AccountsStorage from '../storage/accountStorage' -import { isUserAccount, isGroupAccount } from '../@types/accountTypeGuards' +import { isUserAccount, isGroupAccount, isGroupTreeAccount } from '../@types/accountTypeGuards' +import create from '../accounts' /** * An MLS membership change: proposals + commit, plus the Welcomes for anyone @@ -53,6 +54,33 @@ export const validate_fields = (tx: Tx.GroupCommit, response: ShardusTypes.Incom response.reason = 'tx "groupInfo" and "ratchetTree" must be strings.' return response } + /* + * The ratchet tree is published as a DELTA: only the nodes this commit + * changed, addressed by ratchet-tree node index. The full tree is ~1.8 kB per + * member, whereas a delta is one node on an add and O(log N) on a rekey, so + * this is what keeps a commit's size independent of group size. + * + * `ratchetTree` is now a BASELINE, sent only on a group's first commit or on + * migration; every other commit leaves it empty and sends a delta instead. + */ + if (!Array.isArray(tx.treeDelta)) { + response.reason = 'tx "treeDelta" must be an array.' + return response + } + for (const entry of tx.treeDelta) { + if (!entry || typeof entry.i !== 'number' || !Number.isInteger(entry.i) || entry.i < 0) { + response.reason = 'tx "treeDelta" contains an entry with an invalid node index.' + return response + } + if (entry.n !== null && typeof entry.n !== 'string') { + response.reason = 'tx "treeDelta" entries must carry a base64 node or null to blank it.' + return response + } + } + if (tx.ratchetTree.length === 0 && tx.treeDelta.length === 0) { + response.reason = 'tx must carry either a baseline "ratchetTree" or a non-empty "treeDelta".' + return response + } if (!Array.isArray(tx.addedMembers) || !Array.isArray(tx.removedMembers)) { response.reason = 'tx "addedMembers" and "removedMembers" must be arrays.' return response @@ -125,13 +153,41 @@ export const validate_fields = (tx: Tx.GroupCommit, response: ShardusTypes.Incom } } + /* + * The welcomes are counted here, not just the commit. + * + * Each welcome envelope carries a full ratchet tree (~1.8 kB per group member) + * plus a sealed post-quantum PSK, so on an add they are by far the largest + * part of the transaction — the commit itself is a few kB and does not grow + * with the group. Measuring only commit + groupInfo + ratchetTree let a + * multi-hundred-kB transaction pass a 64 kB limit, which made the check worse + * than useless: it reported a number nobody could act on while the real + * payload went unbounded. + */ + const welcomeBytes = tx.welcomes.reduce((sum, w) => { + const env = w.envelope + // sealedPsk is an object ({cipherText, nonce}), so sum its fields rather + // than stringifying it — String(obj) would score every PSK as 15 bytes. + return ( + sum + + Buffer.byteLength(String(env.welcome), 'utf8') + + Buffer.byteLength(String(env.ratchetTree || ''), 'utf8') + + Buffer.byteLength(String(env.sealedPsk.cipherText || ''), 'utf8') + + Buffer.byteLength(String(env.sealedPsk.nonce || ''), 'utf8') + ) + }, 0) + const totalBytes = Buffer.byteLength(tx.commit, 'utf8') + Buffer.byteLength(tx.groupInfo, 'utf8') + Buffer.byteLength(tx.ratchetTree, 'utf8') + - tx.proposals.reduce((sum, p) => sum + Buffer.byteLength(String(p), 'utf8'), 0) + tx.proposals.reduce((sum, p) => sum + Buffer.byteLength(String(p), 'utf8'), 0) + + tx.treeDelta.reduce((sum, d) => sum + Buffer.byteLength(String(d.n || ''), 'utf8') + 16, 0) + + welcomeBytes if (totalBytes / 1024 > config.LiberdusFlags.groupMessageSizeLimit) { - response.reason = `commit payload exceeds ${config.LiberdusFlags.groupMessageSizeLimit} kB.` + response.reason = + `commit payload exceeds ${config.LiberdusFlags.groupMessageSizeLimit} kB ` + + `(${Math.ceil(totalBytes / 1024)} kB, of which ${Math.ceil(welcomeBytes / 1024)} kB is welcomes).` return response } @@ -159,6 +215,8 @@ export const validate = ( ): ShardusTypes.IncomingTransactionResult => { const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data const group: GroupAccount = wrappedStates[tx.groupId] && wrappedStates[tx.groupId].data + const treeForValidate: GroupTreeAccount = + wrappedStates[utils.calculateGroupTreeId(tx.groupId)] && wrappedStates[utils.calculateGroupTreeId(tx.groupId)].data if (typeof from === 'undefined' || from === null) { response.reason = '"from" account does not exist.' @@ -203,6 +261,84 @@ export const validate = ( response.reason = `added member ${address} does not have a UserAccount.` return response } + + /* + * CONSENT TO BE ADDED, route 1: they asked. + * + * A pending join request IS the consent, and it is per-group and explicit, + * so it overrides whatever the addee's blanket add policy says. + */ + const requested = !!(treeForValidate && treeForValidate.pendingJoinRequests[address]) + + /* + * CONSENT TO BE ADDED, route 2. + * + * Being added is not free for the addee: it consumes one of their single-use + * KeyPackages, and under update-on-join it makes them inject a group_commit + * of their own. So an add that nobody asked for spends someone else's money + * and their key material. The addee's account is already loaded here — it is + * read just above — so this check costs nothing. + */ + if (!requested) { + const policy = addee.data.groupAddPolicy ?? config.LiberdusFlags.groupDefaultAddPolicy + if (policy === 'nobody') { + response.reason = `${address} does not accept group invitations; they must request to join.` + return response + } + if (policy === 'contacts') { + /* + * "Connected" is expressed through the toll, not a friends list. + * + * A new ChatAccount starts at required: [1, 1] — both sides charging — + * so `required === 0` is a deliberate act by the ADDEE waiving the toll + * for this specific account, which is exactly the relationship that used + * to be a friend entry. `required === 2` is an explicit block. + * + * Read from the addee's own slot: toll.required[i] is what party i + * demands of the other, so the adder's setting says nothing about + * whether the addee wants to hear from them. + */ + const chatId = utils.calculateChatId(address, tx.from) + const chat: ChatAccount = wrappedStates[chatId] && wrappedStates[chatId].data + const [addr1] = utils.sortAddresses(address, tx.from) + const addeeIndex = addr1 === address ? 0 : 1 + const addeeRequires = chat && Array.isArray(chat.toll?.required) ? chat.toll.required[addeeIndex] : 1 + + if (addeeRequires === 2) { + response.reason = `${address} has blocked ${tx.from}.` + return response + } + if (addeeRequires !== 0) { + response.reason = `${address} only accepts group invitations from accounts they are connected to.` + return response + } + } + } + + /* + * The declared KeyPackage must actually be in the addee's pool. + * + * apply() removes it by value, so a string that was never there burns + * nothing and leaves the real package reusable — and single-use KeyPackages + * are precisely what stop one init key being used for two adds. Without this + * the declaration is unenforced and the forward-secrecy property it exists + * to provide is not actually held. + */ + const declared = tx.consumedKeyPackages.find((c) => c.address === address) + if (!declared) { + // validate_fields already requires one per added member, but validate must + // not depend on that having run — a throw here would fail the whole + // transaction opaquely instead of rejecting it with a reason. + response.reason = `no key package declared for ${address}.` + return response + } + const pool = Array.isArray(addee.data.mlsKeyPackages) ? addee.data.mlsKeyPackages : [] + const isLastResort = + !!addee.data.mlsLastResortKeyPackage && declared.keyPackage === addee.data.mlsLastResortKeyPackage + if (!pool.includes(declared.keyPackage) && !isLastResort) { + response.reason = `the key package declared for ${address} is not in their published pool.` + return response + } } for (const address of tx.removedMembers) { if (!group.members.includes(address)) { @@ -261,6 +397,50 @@ export const validate = ( return response } +/** + * How far behind the current epoch an uncollected welcome may fall before it is + * dropped. Generous: the only cost of keeping one is storage, but discarding one + * an invitee could still have used would silently strand them. + */ +const WELCOME_RETENTION_EPOCHS = 50 + +/** + * Applies a ratchet-tree delta to the stored tree. + * + * The tree is stored as base64 of ts-mls `encodeRatchetTree`, which is a + * length-prefixed list of optional nodes. Rather than re-implement that codec + * here, the account holds a JSON array of per-node base64 blobs (`null` = blank) + * that the client assembles and the server only indexes into. The server never + * parses a node; it moves opaque strings by index. + * + * Deterministic, so every validator produces the same bytes — a requirement for + * consensus. + */ +const applyTreeDelta = (stored: string, delta: { i: number; n: string | null }[]): string => { + let nodes: (string | null)[] = [] + if (stored.length > 0) { + try { + const parsed = JSON.parse(stored) + if (Array.isArray(parsed)) nodes = parsed + } catch { + // Unreadable stored tree: rebuild from the delta rather than throwing and + // wedging the group. The joiner's tree-hash check is what actually + // protects correctness here. + nodes = [] + } + } + for (const entry of delta) { + // Grow with explicit blanks so indices stay meaningful; a sparse JS array + // would serialise as nulls anyway but with undefined holes in between. + while (nodes.length <= entry.i) nodes.push(null) + nodes[entry.i] = entry.n + } + // Trailing blanks carry no information and would grow the account forever as + // members are removed from the right-hand side of the tree. + while (nodes.length > 0 && nodes[nodes.length - 1] === null) nodes.pop() + return JSON.stringify(nodes) +} + export const apply = ( tx: Tx.GroupCommit, txTimestamp: number, @@ -271,6 +451,12 @@ export const apply = ( ): void => { const from: UserAccount = wrappedStates[tx.from].data const group: GroupAccount = wrappedStates[tx.groupId].data + const treeId = utils.calculateGroupTreeId(tx.groupId) + const tree: GroupTreeAccount = wrappedStates[treeId] && wrappedStates[treeId].data + + if (!tree) { + throw Error('getRelevantAccount must create the GroupTreeAccount before apply') + } const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) @@ -294,7 +480,21 @@ export const apply = ( timestamp: txTimestamp, sign: tx.sign, } - group.handshakes.push(commitRecord) + tree.handshakes.push(commitRecord) + /* + * Bound the transcript. + * + * Pruning a commit locks out any member that has not applied it — MLS state + * cannot be rebuilt from the public transcript, so they must reset and be + * re-added. That is the trade: an unbounded transcript grows ~9 kB per commit + * forever (measured at 100 members) and is transferred and re-hashed on every + * later commit. Clients read `oldestAvailableEpoch` from the handshakes + * endpoint and surface a reset when they fall behind it. + */ + const maxHandshakes = config.LiberdusFlags.groupMaxHandshakes + if (maxHandshakes > 0 && tree.handshakes.length > maxHandshakes) { + tree.handshakes = tree.handshakes.slice(-maxHandshakes) + } group.epoch = previousEpoch + 1 @@ -304,31 +504,156 @@ export const apply = ( for (const address of tx.addedMembers) { group.memberSince[address] = { epoch: group.epoch, timestamp: txTimestamp } + + /* + * Approving a request consumes it, and its escrow is earned by the admin who + * did the approving — `from`, not the group and not whoever created it. + * + * joinFee is zero until paid groups ship, so this moves nothing today. When + * it does, §5.2 of the consent spec adds a vesting delay here so that + * "take the fee, remove the member" can be refunded rather than clawed back. + */ + const request = tree.pendingJoinRequests[address] + if (request) { + /* + * The fee does NOT land in the admin's balance yet. + * + * Paying immediately would make "take the fee, remove the member" a scam + * that has to be undone by clawing money back from someone who may + * already have spent it. Vesting inverts that: until `vestingUntil` the + * money is still recoverable, so removing the member simply returns it + * (see the removal loop below). Only after the window does the admin + * become able to claim it. Same shape as a 1:1 toll, which is likewise + * conditional rather than immediate. + */ + if (request.escrow > BigInt(0)) { + tree.vestedFees.push({ + admin: tx.from, + member: address, + amount: request.escrow, + vestingUntil: txTimestamp + config.LiberdusFlags.groupJoinFeeVestingMs, + }) + } + delete tree.pendingJoinRequests[address] + } } for (const address of tx.removedMembers) { delete group.memberSince[address] delete group.lastMessageAt[address] - delete group.pendingWelcomes[address] + delete tree.pendingWelcomes[address] + + /* + * Removed before their join fee vested: give it back. + * + * This is what makes the vesting window meaningful rather than decorative — + * the money is returned automatically, from an account nobody has been paid + * from yet, instead of having to be recovered from the admin afterwards. + * A member who LEAVES is not refunded (group_leave does not run this), or + * every paid group would be a free trial. + */ + const stillVesting = tree.vestedFees.filter((v) => v.member === address && v.vestingUntil > txTimestamp) + if (stillVesting.length > 0) { + const refundee: UserAccount = wrappedStates[address] && wrappedStates[address].data + if (refundee && isUserAccount(refundee)) { + for (const v of stillVesting) { + refundee.data.balance = SafeBigIntMath.add(refundee.data.balance, v.amount) + } + refundee.timestamp = txTimestamp + } + tree.vestedFees = tree.vestedFees.filter((v) => !(v.member === address && v.vestingUntil > txTimestamp)) + } + } + + /* + * Advance the stored ratchet tree. + * + * A baseline replaces it wholesale; otherwise the delta is applied by node + * index. This is pure data movement — the network does no MLS cryptography and + * cannot check that the delta is honest. It does not need to: RFC 9420 + * requires a joiner to verify the tree against the `tree_hash` in the + * GroupContext carried in its Welcome, so a committer who publishes a corrupt + * tree is caught by the joiner and cannot forge one that hashes correctly. + */ + if (tx.ratchetTree.length > 0) { + tree.ratchetTree = tx.ratchetTree + } else { + tree.ratchetTree = applyTreeDelta(tree.ratchetTree, tx.treeDelta) + } + tree.treeEpoch = group.epoch + + console.log("thant: tree", tree) + console.log("thant: tree.ratchetTree", tree.ratchetTree) + + /* + * Drop the sender's own pending welcome. + * + * Welcomes are collected over a GET, which cannot mutate consensus state, so + * nothing else ever removed them — they accumulated for the life of the group, + * and each one carries a full tree snapshot. A commit signed by that member is + * proof they joined, and under the update-on-join rule every joiner sends one + * almost immediately. + */ + delete tree.pendingWelcomes[tx.from] + + /* + * Backstop for an invitee that never joins and never commits: their welcome is + * addressed to an epoch whose keys have long rotated, so it is useless to them + * and merely expensive to keep. Bounded by the roster either way, but this + * keeps a group that repeatedly invites no-shows from carrying them forever. + */ + for (const [address, envelope] of Object.entries(tree.pendingWelcomes)) { + if (group.epoch - (envelope.epoch || 0) > WELCOME_RETENTION_EPOCHS) { + delete tree.pendingWelcomes[address] + } } - // Park each Welcome (and its sealed PQ PSK) for collection by the new member. + /* + * Park each Welcome for collection, and take ONE tree snapshot for the epoch. + * + * The joiner needs the tree matching the GroupContext in its Welcome, but the + * live tree moves on immediately — under update-on-join the very next commit is + * the joiner's own path update. The snapshot costs zero transaction bytes, + * since the assembled tree is already in hand. + * + * Keyed by epoch rather than stored per joiner: everyone added in one commit + * shares the same tree, and at 100 members that tree is ~354 kB — a per-joiner + * copy made a 10-member add write 3.5 MB. + */ for (const welcome of tx.welcomes) { - group.pendingWelcomes[welcome.address] = { + tree.pendingWelcomes[welcome.address] = { ...welcome.envelope, + // Who did the adding. The invitee's client needs this to say "X added you" + // when asking whether to accept, and nothing else records it — the commit + // transcript is pruned, and the roster does not say who admitted whom. + addedBy: tx.from, epoch: group.epoch, timestamp: txTimestamp, } } + if (tx.welcomes.length > 0) { + tree.welcomeTrees[String(group.epoch)] = tree.ratchetTree + } + + // Drop snapshots no pending welcome refers to any more. + const neededEpochs = new Set(Object.values(tree.pendingWelcomes).map((w) => String(w.epoch))) + for (const epoch of Object.keys(tree.welcomeTrees)) { + if (!neededEpochs.has(epoch)) delete tree.welcomeTrees[epoch] + } /* * Checkpoint every commit. A member who was offline while older commits were * pruned can rejoin from this GroupInfo via an external commit; without it * they would be locked out permanently. */ - group.checkpoint = { + /* + * No tree copy here. The checkpoint is rewritten on every commit, so its epoch + * always equals treeEpoch and its tree would be byte-identical to + * `tree.ratchetTree` — it was storing the whole thing twice. Consumers read + * `tree.ratchetTree` when `checkpoint.epoch === tree.treeEpoch`. + */ + tree.checkpoint = { epoch: group.epoch, groupInfo: tx.groupInfo, - ratchetTree: tx.ratchetTree, timestamp: txTimestamp, } @@ -371,6 +696,7 @@ export const apply = ( } group.timestamp = txTimestamp + tree.timestamp = txTimestamp from.timestamp = txTimestamp const appReceiptData: AppReceiptData = { @@ -439,28 +765,68 @@ export const createFailedAppReceiptData = ( */ export const keys = (tx: Tx.GroupCommit, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { result.sourceKeys = [tx.from] - result.targetKeys = [tx.groupId, ...tx.addedMembers, ...tx.removedMembers] + // The tree account is a separate address, so it must be named here or apply() + // cannot write it. group_message deliberately does NOT name it. + /* + * The chat account for each (adder, addee) pair carries the toll setting that + * says whether the addee accepts invitations from this account, so it has to + * be loaded for validate() to read. Read-only — see memoryPattern. + */ + const connectionChats = tx.addedMembers.map((address) => utils.calculateChatId(address, tx.from)) + result.targetKeys = [ + tx.groupId, + utils.calculateGroupTreeId(tx.groupId), + ...tx.addedMembers, + ...tx.removedMembers, + ...connectionChats, + ] result.allKeys = [...result.sourceKeys, ...result.targetKeys] return result } export const memoryPattern = (tx: Tx.GroupCommit, result: ShardusTypes.TransactionKeys): ShardusTypes.ShardusMemoryPatternsInput => { return { - rw: [tx.from, tx.groupId, ...tx.addedMembers, ...tx.removedMembers], + rw: [tx.from, tx.groupId, utils.calculateGroupTreeId(tx.groupId), ...tx.addedMembers, ...tx.removedMembers], wo: [], on: [], - ri: [], + // Connection chats are only read, never written, by a commit. + ri: tx.addedMembers.map((address) => utils.calculateChatId(address, tx.from)), ro: [], } } export const createRelevantAccount = ( dapp: Shardus, - account: UserAccount | GroupAccount, + account: UserAccount | GroupAccount | GroupTreeAccount | ChatAccount, accountId: string, tx: Tx.GroupCommit, accountCreated = false, ): ShardusTypes.WrappedResponse => { + /* + * The tree account is created lazily rather than by group_create, so groups + * that predate the split get one on their first commit. That commit is also + * the one that publishes the baseline `ratchetTree` (see apply), so the pair + * becomes consistent in a single step. + */ + if (!account && accountId === utils.calculateGroupTreeId(tx.groupId)) { + account = create.groupTreeAccount(accountId, tx.groupId) + accountCreated = true + } + /* + * Two accounts that have never chatted have no chat account, and therefore no + * toll setting for validate() to read — the transaction would otherwise fail + * opaquely on a missing account rather than with a reason. + * + * The default construction waives only the INITIATOR's toll (chatAccount sets + * required[senderIndex] = 0 so a replier is not charged), leaving the addee's + * at 1. validate() reads the addee's slot, so this materialises exactly the + * "not connected" state and the add is refused with a clear message. + */ + if (!account && tx.addedMembers.some((address) => utils.calculateChatId(address, tx.from) === accountId)) { + const addee = tx.addedMembers.find((address) => utils.calculateChatId(address, tx.from) === accountId) + account = create.chatAccount(accountId, { from: tx.from, to: addee } as Tx.Message) + accountCreated = true + } if (!account) { throw Error('Account must exist in order to commit to a group') } diff --git a/src/transactions/group_create.ts b/src/transactions/group_create.ts index 48b76616..2f57864f 100644 --- a/src/transactions/group_create.ts +++ b/src/transactions/group_create.ts @@ -59,6 +59,21 @@ export const validate_fields = (tx: Tx.GroupCreate, response: ShardusTypes.Incom response.reason = `tx "maxMembers" must be an integer between 1 and ${config.LiberdusFlags.groupMaxMembers}.` return response } + /* + * Price of admission. Zero means an open group; a positive value is escrowed + * by each requester and earned by the admin who approves them, after a + * vesting delay (see group_commit). Capped as a guard against a typo turning + * a group into one nobody can afford to join. + */ + if (typeof tx.joinFee !== 'bigint' || tx.joinFee < BigInt(0)) { + response.reason = 'tx "joinFee" must be a non-negative bigint.' + return response + } + const maxJoinFee = utils.usdStrToWei(config.LiberdusFlags.groupMaxJoinFeeUsdStr, AccountsStorage.cachedNetworkAccount) + if (maxJoinFee > BigInt(0) && tx.joinFee > maxJoinFee) { + response.reason = `tx "joinFee" exceeds the maximum allowed (${maxJoinFee}).` + return response + } if (typeof tx.fee !== 'bigint') { response.reason = 'tx "fee" must be a bigint.' return response diff --git a/src/transactions/group_fee_claim.ts b/src/transactions/group_fee_claim.ts new file mode 100644 index 00000000..34c1af0f --- /dev/null +++ b/src/transactions/group_fee_claim.ts @@ -0,0 +1,198 @@ +import * as crypto from '../crypto' +import { Shardus, ShardusTypes } from '@shardus/core' +import * as utils from '../utils' +import { UserAccount, GroupTreeAccount, WrappedStates, Tx, AppReceiptData } from '../@types' +import { SafeBigIntMath } from '../utils/safeBigIntMath' +import * as AccountsStorage from '../storage/accountStorage' +import { isUserAccount, isGroupTreeAccount } from '../@types/accountTypeGuards' + +/** Fees this admin has earned and that have finished vesting. */ +const matured = (tree: GroupTreeAccount, admin: string, now: number): bigint => + (tree.vestedFees || []) + .filter((v) => v.admin === admin && v.vestingUntil <= now) + .reduce((sum, v) => SafeBigIntMath.add(sum, v.amount), BigInt(0)) + +/** + * Collects join fees that have finished vesting. + * + * An approved join fee is not paid out immediately: until `vestingUntil` it can + * still be returned to the member if they are removed, which is what stops + * "take the fee, remove them" from being a cheap scam. This is the other half — + * once the window has passed, the admin who did the approving collects. + * + * Deliberately a separate transaction rather than a sweep during group_commit: + * an admin should be able to collect without having to make a membership change + * to trigger it, and a group with no activity should not strand its fees. + */ +export const validate_fields = ( + tx: Tx.GroupFeeClaim, + response: ShardusTypes.IncomingTransactionResult, +): ShardusTypes.IncomingTransactionResult => { + if (utils.isValidAddress(tx.from) === false) { + response.reason = 'tx "from" is not a valid address.' + return response + } + if (utils.isValidAddress(tx.groupId) === false) { + response.reason = 'tx "groupId" is not a valid address.' + return response + } + if (typeof tx.fee !== 'bigint') { + response.reason = 'tx "fee" must be a bigint.' + return response + } + if (!tx.sign || !tx.sign.owner || !tx.sign.sig || tx.sign.owner !== tx.from) { + response.reason = 'not signed by from account' + return response + } + if (crypto.verifyObj(tx) === false) { + response.reason = 'incorrect signing' + return response + } + response.success = true + return response +} + +export const validate = ( + tx: Tx.GroupFeeClaim, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + const treeId = utils.calculateGroupTreeId(tx.groupId) + const tree: GroupTreeAccount = wrappedStates[treeId] && wrappedStates[treeId].data + + if (typeof from === 'undefined' || from === null || !isUserAccount(from)) { + response.reason = '"from" account does not exist.' + return response + } + if (!tree || !isGroupTreeAccount(tree)) { + response.reason = 'this group has no fees to claim.' + return response + } + /* + * Timestamps come from the transaction, not Date.now(), so every validator + * reaches the same answer about what has matured. + */ + if (matured(tree, tx.from, tx.timestamp) <= BigInt(0)) { + response.reason = 'no join fees have finished vesting for this account.' + return response + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.success = false + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < transactionFee) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + +export const apply = ( + tx: Tx.GroupFeeClaim, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, +): void => { + const from: UserAccount = wrappedStates[tx.from].data + const treeId = utils.calculateGroupTreeId(tx.groupId) + const tree: GroupTreeAccount = wrappedStates[treeId].data + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + + const amount = matured(tree, tx.from, txTimestamp) + from.data.balance = SafeBigIntMath.add(from.data.balance, amount) + tree.vestedFees = (tree.vestedFees || []).filter((v) => !(v.admin === tx.from && v.vestingUntil <= txTimestamp)) + + tree.timestamp = txTimestamp + from.timestamp = txTimestamp + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: true, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { groupId: tx.groupId, claimed: amount.toString() }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) + dapp.log('Applied group_fee_claim tx', tx.groupId, tx.from, amount) +} + +export const createFailedAppReceiptData = ( + tx: Tx.GroupFeeClaim, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, + reason: string, +): void => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + let transactionFee = BigInt(0) + if (from !== undefined && from !== null) { + const networkFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (from.data.balance >= networkFee) { + transactionFee = networkFee + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } else { + transactionFee = from.data.balance + from.data.balance = BigInt(0) + } + from.timestamp = txTimestamp + } + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: false, + reason, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { groupId: tx.groupId }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) +} + +export const keys = (tx: Tx.GroupFeeClaim, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { + result.sourceKeys = [tx.from] + result.targetKeys = [utils.calculateGroupTreeId(tx.groupId)] + result.allKeys = [...result.sourceKeys, ...result.targetKeys] + return result +} + +export const memoryPattern = ( + tx: Tx.GroupFeeClaim, + result: ShardusTypes.TransactionKeys, +): ShardusTypes.ShardusMemoryPatternsInput => { + return { rw: [tx.from, utils.calculateGroupTreeId(tx.groupId)], wo: [], on: [], ri: [], ro: [] } +} + +export const createRelevantAccount = ( + dapp: Shardus, + account: UserAccount | GroupTreeAccount, + accountId: string, + tx: Tx.GroupFeeClaim, + accountCreated = false, +): ShardusTypes.WrappedResponse => { + if (!account) { + throw Error('Account must exist in order to claim group fees') + } + return dapp.createWrappedResponse(accountId, accountCreated, account.hash, account.timestamp, account) +} diff --git a/src/transactions/group_join_reclaim.ts b/src/transactions/group_join_reclaim.ts new file mode 100644 index 00000000..6855a64b --- /dev/null +++ b/src/transactions/group_join_reclaim.ts @@ -0,0 +1,196 @@ +import * as crypto from '../crypto' +import { Shardus, ShardusTypes } from '@shardus/core' +import * as utils from '../utils' +import { UserAccount, GroupTreeAccount, WrappedStates, Tx, AppReceiptData } from '../@types' +import { SafeBigIntMath } from '../utils/safeBigIntMath' +import * as AccountsStorage from '../storage/accountStorage' +import { isUserAccount, isGroupTreeAccount } from '../@types/accountTypeGuards' + +/** + * Withdraws a pending join request and returns its escrow. + * + * The counterpart to reclaim_toll: money the other side never earned comes back + * to the person who put it up. An admin who ignores a request is not refusing + * it — silence is how a group declines, because making the group pay a fee to + * say "no" to a spammer would be backwards — so the requester needs a way to + * take their money and their request back. + * + * Unlike reclaim_toll there is no timeout to wait out. A pending request has no + * counterparty mid-transaction to protect: nobody has done work on the strength + * of it, so trapping the requester's funds would serve no one. Withdrawing + * before approval is always allowed. + */ +export const validate_fields = ( + tx: Tx.GroupJoinReclaim, + response: ShardusTypes.IncomingTransactionResult, +): ShardusTypes.IncomingTransactionResult => { + if (utils.isValidAddress(tx.from) === false) { + response.reason = 'tx "from" is not a valid address.' + return response + } + if (utils.isValidAddress(tx.groupId) === false) { + response.reason = 'tx "groupId" is not a valid address.' + return response + } + if (typeof tx.fee !== 'bigint') { + response.reason = 'tx "fee" must be a bigint.' + return response + } + if (!tx.sign || !tx.sign.owner || !tx.sign.sig || tx.sign.owner !== tx.from) { + response.reason = 'not signed by from account' + return response + } + if (crypto.verifyObj(tx) === false) { + response.reason = 'incorrect signing' + return response + } + response.success = true + return response +} + +export const validate = ( + tx: Tx.GroupJoinReclaim, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + const treeId = utils.calculateGroupTreeId(tx.groupId) + const tree: GroupTreeAccount = wrappedStates[treeId] && wrappedStates[treeId].data + + if (typeof from === 'undefined' || from === null || !isUserAccount(from)) { + response.reason = '"from" account does not exist.' + return response + } + if (!tree || !isGroupTreeAccount(tree)) { + response.reason = 'this group has no pending requests.' + return response + } + /* + * Only the requester may move this money. The request is keyed by address and + * the transaction is signed by tx.from, so there is no way to reclaim on + * someone else's behalf. + */ + if (!tree.pendingJoinRequests[tx.from]) { + response.reason = 'no pending join request from this account.' + return response + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.success = false + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < transactionFee) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + +export const apply = ( + tx: Tx.GroupJoinReclaim, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, +): void => { + const from: UserAccount = wrappedStates[tx.from].data + const treeId = utils.calculateGroupTreeId(tx.groupId) + const tree: GroupTreeAccount = wrappedStates[treeId].data + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + + const request = tree.pendingJoinRequests[tx.from] + const refund = request ? request.escrow : BigInt(0) + from.data.balance = SafeBigIntMath.add(from.data.balance, refund) + delete tree.pendingJoinRequests[tx.from] + + tree.timestamp = txTimestamp + from.timestamp = txTimestamp + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: true, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { groupId: tx.groupId, refunded: refund.toString() }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) + dapp.log('Applied group_join_reclaim tx', tx.groupId, tx.from, refund) +} + +export const createFailedAppReceiptData = ( + tx: Tx.GroupJoinReclaim, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, + reason: string, +): void => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + let transactionFee = BigInt(0) + if (from !== undefined && from !== null) { + const networkFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (from.data.balance >= networkFee) { + transactionFee = networkFee + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } else { + transactionFee = from.data.balance + from.data.balance = BigInt(0) + } + from.timestamp = txTimestamp + } + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: false, + reason, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { groupId: tx.groupId }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) +} + +export const keys = (tx: Tx.GroupJoinReclaim, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { + result.sourceKeys = [tx.from] + result.targetKeys = [utils.calculateGroupTreeId(tx.groupId)] + result.allKeys = [...result.sourceKeys, ...result.targetKeys] + return result +} + +export const memoryPattern = ( + tx: Tx.GroupJoinReclaim, + result: ShardusTypes.TransactionKeys, +): ShardusTypes.ShardusMemoryPatternsInput => { + return { rw: [tx.from, utils.calculateGroupTreeId(tx.groupId)], wo: [], on: [], ri: [], ro: [] } +} + +export const createRelevantAccount = ( + dapp: Shardus, + account: UserAccount | GroupTreeAccount, + accountId: string, + tx: Tx.GroupJoinReclaim, + accountCreated = false, +): ShardusTypes.WrappedResponse => { + if (!account) { + throw Error('Account must exist in order to reclaim a join request') + } + return dapp.createWrappedResponse(accountId, accountCreated, account.hash, account.timestamp, account) +} diff --git a/src/transactions/group_join_request.ts b/src/transactions/group_join_request.ts new file mode 100644 index 00000000..6edd760b --- /dev/null +++ b/src/transactions/group_join_request.ts @@ -0,0 +1,259 @@ +import * as crypto from '../crypto' +import { Shardus, ShardusTypes } from '@shardus/core' +import * as utils from '../utils' +import * as config from '../config' +import create from '../accounts' +import { UserAccount, GroupAccount, GroupTreeAccount, WrappedStates, Tx, AppReceiptData } from '../@types' +import { SafeBigIntMath } from '../utils/safeBigIntMath' +import * as AccountsStorage from '../storage/accountStorage' +import { isUserAccount, isGroupAccount } from '../@types/accountTypeGuards' + +/** + * Asks to join a group. + * + * This is the consent artefact. group_commit requires a matching request before + * it will admit anyone (unless the addee separately allows direct adds), which + * is what stops an account being pulled into a group it never asked for — + * spending its KeyPackages and, under update-on-join, its money. + * + * Carries NO KeyPackage. The approving commit draws one from the requester's + * published pool instead: pinning a package here would break if the requester + * rotated their pool while the request was pending, because publishing discards + * the private halves and the Welcome would become undecryptable. + * + * The escrow follows the toll model in `message`: the requester's balance is + * debited now and the amount recorded as a claim on the group, earned by the + * approving admin or reclaimed by the requester (see group_join_reclaim). + * joinFee is zero until paid groups ship, so today this is a no-op that keeps + * the shape correct. + */ +export const validate_fields = ( + tx: Tx.GroupJoinRequest, + response: ShardusTypes.IncomingTransactionResult, +): ShardusTypes.IncomingTransactionResult => { + if (utils.isValidAddress(tx.from) === false) { + response.reason = 'tx "from" is not a valid address.' + return response + } + if (utils.isValidAddress(tx.groupId) === false) { + response.reason = 'tx "groupId" is not a valid address.' + return response + } + if (typeof tx.escrow !== 'bigint' || tx.escrow < BigInt(0)) { + response.reason = 'tx "escrow" must be a non-negative bigint.' + return response + } + if (typeof tx.message !== 'string') { + response.reason = 'tx "message" must be a string.' + return response + } + if (tx.message.length > config.LiberdusFlags.groupMaxJoinRequestMessageLength) { + response.reason = `tx "message" must be at most ${config.LiberdusFlags.groupMaxJoinRequestMessageLength} characters.` + return response + } + if (typeof tx.fee !== 'bigint') { + response.reason = 'tx "fee" must be a bigint.' + return response + } + if (!tx.sign || !tx.sign.owner || !tx.sign.sig || tx.sign.owner !== tx.from) { + response.reason = 'not signed by from account' + return response + } + if (crypto.verifyObj(tx) === false) { + response.reason = 'incorrect signing' + return response + } + response.success = true + return response +} + +export const validate = ( + tx: Tx.GroupJoinRequest, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId] && wrappedStates[tx.groupId].data + const treeId = utils.calculateGroupTreeId(tx.groupId) + const tree: GroupTreeAccount = wrappedStates[treeId] && wrappedStates[treeId].data + + if (typeof from === 'undefined' || from === null) { + response.reason = '"from" account does not exist.' + return response + } + if (!isUserAccount(from)) { + response.reason = 'from account is not a UserAccount' + return response + } + if (typeof group === 'undefined' || group === null || !isGroupAccount(group)) { + response.reason = '"groupId" account does not exist.' + return response + } + if (group.members.includes(tx.from)) { + response.reason = 'already a member of this group.' + return response + } + if (Array.isArray(group.blocked) && group.blocked.includes(tx.from)) { + response.reason = 'this group is not accepting requests from you.' + return response + } + if (group.members.length >= group.maxMembers) { + response.reason = 'the group is full.' + return response + } + if (tree && tree.pendingJoinRequests[tx.from]) { + response.reason = 'a join request from this account is already pending.' + return response + } + if (tree && Object.keys(tree.pendingJoinRequests).length >= config.LiberdusFlags.groupMaxPendingJoinRequests) { + response.reason = 'this group has too many pending join requests; try again later.' + return response + } + /* + * The escrow must match the advertised price exactly. Because the escrowed + * amount IS the consent, an admin who raises joinFee afterwards can only ever + * be paid what was agreed here — no separate "max fee" rule is needed. + */ + if (tx.escrow !== group.joinFee) { + response.reason = `tx "escrow" (${tx.escrow}) must equal the group's join fee (${group.joinFee}).` + return response + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.success = false + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < transactionFee + tx.escrow) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the join fee (${tx.escrow}) + transaction fee (${transactionFee}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + +export const apply = ( + tx: Tx.GroupJoinRequest, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, +): void => { + const from: UserAccount = wrappedStates[tx.from].data + const treeId = utils.calculateGroupTreeId(tx.groupId) + const tree: GroupTreeAccount = wrappedStates[treeId] && wrappedStates[treeId].data + + if (!tree) { + throw Error('getRelevantAccount must create the GroupTreeAccount before apply') + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + // Held, not spent: reclaimable by the requester until an admin approves. + from.data.balance = SafeBigIntMath.subtract(from.data.balance, tx.escrow) + + tree.pendingJoinRequests[tx.from] = { + escrow: tx.escrow, + message: tx.message, + timestamp: txTimestamp, + } + + tree.timestamp = txTimestamp + from.timestamp = txTimestamp + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: true, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { groupId: tx.groupId, escrow: tx.escrow.toString() }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) + dapp.log('Applied group_join_request tx', tx.groupId, tx.from) +} + +export const createFailedAppReceiptData = ( + tx: Tx.GroupJoinRequest, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, + reason: string, +): void => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + let transactionFee = BigInt(0) + if (from !== undefined && from !== null) { + const networkFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (from.data.balance >= networkFee) { + transactionFee = networkFee + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } else { + transactionFee = from.data.balance + from.data.balance = BigInt(0) + } + from.timestamp = txTimestamp + } + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: false, + reason, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { groupId: tx.groupId }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) +} + +export const keys = (tx: Tx.GroupJoinRequest, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { + result.sourceKeys = [tx.from] + // The group account is read (roster, blocked, joinFee); the tree account is + // written (the request itself). + result.targetKeys = [tx.groupId, utils.calculateGroupTreeId(tx.groupId)] + result.allKeys = [...result.sourceKeys, ...result.targetKeys] + return result +} + +export const memoryPattern = ( + tx: Tx.GroupJoinRequest, + result: ShardusTypes.TransactionKeys, +): ShardusTypes.ShardusMemoryPatternsInput => { + return { + rw: [tx.from, utils.calculateGroupTreeId(tx.groupId)], + wo: [], + on: [], + ri: [tx.groupId], + ro: [], + } +} + +export const createRelevantAccount = ( + dapp: Shardus, + account: UserAccount | GroupAccount | GroupTreeAccount, + accountId: string, + tx: Tx.GroupJoinRequest, + accountCreated = false, +): ShardusTypes.WrappedResponse => { + if (!account && accountId === utils.calculateGroupTreeId(tx.groupId)) { + account = create.groupTreeAccount(accountId, tx.groupId) + accountCreated = true + } + if (!account) { + throw Error('Account must exist in order to request to join a group') + } + return dapp.createWrappedResponse(accountId, accountCreated, account.hash, account.timestamp, account) +} diff --git a/src/transactions/group_leave.ts b/src/transactions/group_leave.ts index 76e0ebef..e1eb4003 100644 --- a/src/transactions/group_leave.ts +++ b/src/transactions/group_leave.ts @@ -115,7 +115,13 @@ export const apply = ( group.admins = group.admins.filter((a) => a !== tx.from) delete group.memberSince[tx.from] delete group.lastMessageAt[tx.from] - delete group.pendingWelcomes[tx.from] + /* + * Any uncollected Welcome for this account lives on the GroupTreeAccount, + * which group_leave deliberately does NOT name in keys() — leaving is a + * roster change and should not drag the tree into consensus. A stale welcome + * is harmless: it is addressed to key material the leaver has abandoned, and + * the next commit overwrites or removes it. + */ // If the last admin walked out, promote the longest-standing remaining member // so the group can still be managed. diff --git a/src/transactions/index.ts b/src/transactions/index.ts index 7c444d05..9d3ffb33 100644 --- a/src/transactions/index.ts +++ b/src/transactions/index.ts @@ -61,6 +61,10 @@ import * as group_create from './group_create' import * as group_keypackage_publish from './group_keypackage_publish' import * as group_message from './group_message' import * as group_commit from './group_commit' +import * as update_group_add_policy from './update_group_add_policy' +import * as group_join_request from './group_join_request' +import * as group_join_reclaim from './group_join_reclaim' +import * as group_fee_claim from './group_fee_claim' import * as group_leave from './group_leave' export default { @@ -127,5 +131,9 @@ export default { group_keypackage_publish, group_message, group_commit, + update_group_add_policy, + group_join_request, + group_join_reclaim, + group_fee_claim, group_leave, } diff --git a/src/transactions/update_group_add_policy.ts b/src/transactions/update_group_add_policy.ts new file mode 100644 index 00000000..57a09a2d --- /dev/null +++ b/src/transactions/update_group_add_policy.ts @@ -0,0 +1,178 @@ +import * as crypto from '../crypto' +import { Shardus, ShardusTypes } from '@shardus/core' +import * as utils from '../utils' +import { UserAccount, WrappedStates, Tx, AppReceiptData } from '../@types' +import { SafeBigIntMath } from '../utils/safeBigIntMath' +import * as AccountsStorage from '../storage/accountStorage' +import { isUserAccount } from '../@types/accountTypeGuards' + +const POLICIES = ['anyone', 'contacts', 'nobody'] + +/** + * Sets who may add this account to a group without it having asked to join. + * + * 'contacts' means accounts this one is connected to — those it has waived its + * chat toll for (toll.required === 0), which is the relationship that replaced + * the deprecated friends list. + * + * The network default is restrictive, because being added is not free for the + * addee: it consumes one of their single-use KeyPackages and, under + * update-on-join, makes them inject a group_commit of their own. This lets an + * account opt into being addable by anyone, or refuse direct adds entirely and + * accept members only through join requests. + */ +export const validate_fields = ( + tx: Tx.UpdateGroupAddPolicy, + response: ShardusTypes.IncomingTransactionResult, +): ShardusTypes.IncomingTransactionResult => { + if (utils.isValidAddress(tx.from) === false) { + response.reason = 'tx "from" is not a valid address.' + return response + } + if (typeof tx.policy !== 'string' || !POLICIES.includes(tx.policy)) { + response.reason = `tx "policy" must be one of ${POLICIES.join(', ')}.` + return response + } + if (typeof tx.fee !== 'bigint') { + response.reason = 'tx "fee" must be a bigint.' + return response + } + if (!tx.sign || !tx.sign.owner || !tx.sign.sig || tx.sign.owner !== tx.from) { + response.reason = 'not signed by from account' + return response + } + if (crypto.verifyObj(tx) === false) { + response.reason = 'incorrect signing' + return response + } + response.success = true + return response +} + +export const validate = ( + tx: Tx.UpdateGroupAddPolicy, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + + if (typeof from === 'undefined' || from === null) { + response.reason = '"from" account does not exist.' + return response + } + if (!isUserAccount(from)) { + response.reason = 'from account is not a UserAccount' + return response + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < transactionFee) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + +export const apply = ( + tx: Tx.UpdateGroupAddPolicy, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, +): void => { + const from: UserAccount = wrappedStates[tx.from].data + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + + from.data.groupAddPolicy = tx.policy + from.timestamp = txTimestamp + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: true, + from: tx.from, + to: tx.from, + type: tx.type, + transactionFee, + additionalInfo: { policy: tx.policy }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) + dapp.log('Applied update_group_add_policy tx', tx.from, tx.policy) +} + +export const createFailedAppReceiptData = ( + tx: Tx.UpdateGroupAddPolicy, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, + reason: string, +): void => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + let transactionFee = BigInt(0) + if (from !== undefined && from !== null) { + const networkFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (from.data.balance >= networkFee) { + transactionFee = networkFee + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } else { + transactionFee = from.data.balance + from.data.balance = BigInt(0) + } + from.timestamp = txTimestamp + } + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: false, + reason, + from: tx.from, + to: tx.from, + type: tx.type, + transactionFee, + additionalInfo: {}, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) +} + +export const keys = (tx: Tx.UpdateGroupAddPolicy, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { + result.sourceKeys = [tx.from] + result.targetKeys = [] + result.allKeys = [...result.sourceKeys, ...result.targetKeys] + return result +} + +export const memoryPattern = ( + tx: Tx.UpdateGroupAddPolicy, + result: ShardusTypes.TransactionKeys, +): ShardusTypes.ShardusMemoryPatternsInput => { + return { rw: [tx.from], wo: [], on: [], ri: [], ro: [] } +} + +export const createRelevantAccount = ( + dapp: Shardus, + account: UserAccount, + accountId: string, + tx: Tx.UpdateGroupAddPolicy, + accountCreated = false, +): ShardusTypes.WrappedResponse => { + if (!account) { + throw Error('Account must exist in order to set a group add policy') + } + return dapp.createWrappedResponse(accountId, accountCreated, account.hash, account.timestamp, account) +} diff --git a/src/utils/index.ts b/src/utils/index.ts index 9c033a2b..e35f0620 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -624,6 +624,18 @@ export function calculateGroupId(creator: string, groupNonce: string): string { return crypto.hash(`${creator.toLowerCase()}${groupNonce.toLowerCase()}`) } +/** + * Address of the GroupTreeAccount paired with a group. + * + * Deterministic so it can be named in keys() before the account exists, and + * domain-separated by a literal that no other address derivation uses: a user + * address is hash(username) and a group id is hash(creator + nonce), so neither + * can be steered into colliding with this. + */ +export function calculateGroupTreeId(groupId: string): string { + return crypto.hash(`${groupId.toLowerCase()}ratchet-tree`) +} + export function validateTxTimestamp(txnTimestamp: number): { success: boolean; reason: string } { const validationResult = { success: false, reason: '' } try { From c11ba5ed1a9c207bf60e614761f287298912ca4c Mon Sep 17 00:00:00 2001 From: Thant Sin Toe Date: Mon, 31 Aug 2026 14:19:19 +0700 Subject: [PATCH 3/7] Screen group commits before they reach the queue A group_commit names the epoch it was built against and the network accepts one commit per epoch, so members lose ordinary races. That check needs the GroupAccount, so it only ran in validate() -- which apply() reaches after consensus, where a rejection still takes a full fee from the loser. The client grew a leaf-index stagger and a backoff to make losing rarer, which is treating the symptom. Group types were absent from preCrackableTxTypes, so every commit went straight to the queue. They are in it now, with a branch that loads the GroupAccount and the sender, and validatePreCrack carrying the checks those two accounts can answer. The GroupTreeAccount is deliberately not fetched: the fence needs group.epoch alone, and the ratchet tree is ~112 kB at 32 members. group_leave needed no new code -- its validate already reads only those two. This is an optimization for honest senders and not a security boundary. It runs only on the node a client injects to; a transaction spread by gossip reaches handleSharedTX, which calls app.validate and never precracks. The comments say so wherever someone might later be tempted to lean on it. Also bounds treeDelta's node index, which was checked for non-negativity and nothing else. applyTreeDelta grows an array to reach it while the size cap measures only the node payload, so {i: 50000000, n: "AA"} was ~18 bytes that cost fifty million array slots. The bound is in validate_fields because that is the only check a gossiped transaction has to clear. And drops two debug logs that printed the whole ratchet tree on every commit. Co-Authored-By: Claude Opus 5 --- src/index.ts | 67 +++++++++++++- src/transactions/group_commit.ts | 104 +++++++++++++++++++++- test/groupCommitPreCrack.test.ts | 139 ++++++++++++++++++++++++++++++ test/groupCommitTreeDelta.test.ts | 77 +++++++++++++++++ 4 files changed, 383 insertions(+), 4 deletions(-) create mode 100644 test/groupCommitPreCrack.test.ts create mode 100644 test/groupCommitTreeDelta.test.ts diff --git a/src/index.ts b/src/index.ts index 241ecd9b..22763c5b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -86,6 +86,26 @@ const groupChatTxTypes = new Set([ TXTypes.group_leave, ]) +/** + * Group transactions that are worth screening before they enter the queue. + * + * A group_commit names the epoch it was built against, and the network accepts + * exactly one commit per epoch. Members therefore lose ordinary races -- and + * because that check needs the GroupAccount, it can only run in validate(), + * which apply() reaches AFTER consensus, where a rejection still costs the + * sender a full fee. Screening here turns the loser's fee into a pre-queue + * rejection that costs nothing. + * + * This is an optimization for honest senders, NOT a security boundary. It runs + * only on the node a client injects to: a transaction arriving by gossip goes + * through handleSharedTX, which calls app.validate and never precracks. Nothing + * downstream may assume a hostile node ran this. + * + * group_message is deliberately absent. It is by far the highest-volume group + * transaction and precracking it would add an account fetch to every message. + */ +const groupPreCrackTxTypes = new Set([TXTypes.group_commit, TXTypes.group_leave]) + let isReadyToJoinLatestValue = false let mustUseAdminCert = false @@ -906,6 +926,7 @@ const shardusSetup = (): void => { TXTypes.claim_reward, TXTypes.apply_penalty, ...(LiberdusFlags.enableNewDAOTransactions ? daoPreCrackTxTypes : []), + ...(LiberdusFlags.enableGroupChat ? groupPreCrackTxTypes : []), ] if (preCrackableTxTypes.includes(tx.type) === false) { return { status: true, reason: 'Tx PreCrack Skipped' } @@ -922,12 +943,43 @@ const shardusSetup = (): void => { let from = tx.from let to = tx.to const isDaoPreCrack = daoPreCrackTxTypes.has(tx.type) + const isGroupPreCrack = groupPreCrackTxTypes.has(tx.type) if (isDaoPreCrack) { from = undefined to = undefined } + /* + * Group commits need the GroupAccount for the epoch fence, and the + * sender's UserAccount for membership and the fee checks -- `from` + * above already fetches the latter. + * + * The GroupTreeAccount is deliberately NOT fetched. The fence needs + * group.epoch and nothing else, and the ratchet tree runs to ~112 kB at + * 32 members; pulling it through getLocalOrRemoteAccount on every + * commit injection would cost far more than the fee it saves. That is + * why these types use validatePreCrack rather than validate() below: + * validate() dereferences the tree account and would throw here. + */ + if (isGroupPreCrack && tx.groupId) { + promises.push( + dapp.getLocalOrRemoteAccount(tx.groupId).then((queuedWrappedState) => { + // A missing group is left absent so validatePreCrack can report + // it in its own words, rather than surfacing as an exception. + if (!queuedWrappedState) return + wrappedStates[tx.groupId] = { + accountId: queuedWrappedState.accountId, + stateId: queuedWrappedState.stateId, + data: queuedWrappedState.data as LiberdusTypes.Accounts, + timestamp: queuedWrappedState.timestamp, + accountCreated: false, + isPartial: false, + } + }), + ) + } + if ( tx.type === TXTypes.deposit_stake || tx.type === TXTypes.withdraw_stake || @@ -1072,7 +1124,20 @@ const shardusSetup = (): void => { console.log('Running txPreCrackData', tx, wrappedStates) - const res = transactions[tx.type].validate(tx, wrappedStates, { success: false, reason: 'Tx Validation Fails' }, dapp) + /* + * Some transaction types validate against a smaller account set here + * than apply() uses, and expose validatePreCrack to say so -- see + * group_commit, which must not pull the ratchet tree just to check an + * epoch. The module-namespace union cannot express "present on some + * members", hence the cast. + */ + const txModule = transactions[tx.type] as unknown as { + validate: typeof transactions[TXTypes.transfer]['validate'] + validatePreCrack?: typeof transactions[TXTypes.transfer]['validate'] + } + const validateForPreCrack = txModule.validatePreCrack ?? txModule.validate + + const res = validateForPreCrack(tx, wrappedStates, { success: false, reason: 'Tx Validation Fails' }, dapp) if (res.success === false) { return { status: false, reason: res.reason } } else { diff --git a/src/transactions/group_commit.ts b/src/transactions/group_commit.ts index 24fa5428..99ed3c5f 100644 --- a/src/transactions/group_commit.ts +++ b/src/transactions/group_commit.ts @@ -67,11 +67,34 @@ export const validate_fields = (tx: Tx.GroupCommit, response: ShardusTypes.Incom response.reason = 'tx "treeDelta" must be an array.' return response } + /* + * The index has to be bounded, not merely non-negative. + * + * applyTreeDelta grows the node array until it reaches entry.i, and the size + * check further down measures only the NODE PAYLOAD -- String(d.n) + 16 -- + * so it never sees the index at all. `{i: 50000000, n: "AA"}` scores about + * 18 bytes, clears any groupMessageSizeLimit, and then costs fifty million + * array slots that get serialised into the account. + * + * This belongs in validate_fields rather than validate(): handleSharedTX + * calls app.validate and nothing else, so this is the only check a + * transaction arriving by gossip has to pass. A bound placed here holds even + * against a node that skips its own precrack and spreads the transaction + * directly to the group. + * + * A tree of N leaves uses node indices up to 2N-2, so 2 * groupMaxMembers is + * the ceiling for any group the network will admit. + */ + const maxNodeIndex = 2 * config.LiberdusFlags.groupMaxMembers for (const entry of tx.treeDelta) { if (!entry || typeof entry.i !== 'number' || !Number.isInteger(entry.i) || entry.i < 0) { response.reason = 'tx "treeDelta" contains an entry with an invalid node index.' return response } + if (entry.i > maxNodeIndex) { + response.reason = `tx "treeDelta" node index ${entry.i} exceeds the maximum ${maxNodeIndex} for a group of up to ${config.LiberdusFlags.groupMaxMembers} members.` + return response + } if (entry.n !== null && typeof entry.n !== 'string') { response.reason = 'tx "treeDelta" entries must carry a base64 node or null to blank it.' return response @@ -207,6 +230,84 @@ export const validate_fields = (tx: Tx.GroupCommit, response: ShardusTypes.Incom return response } +/** + * The subset of validate() that needs only the GroupAccount and the sender. + * + * Called from txPreCrackData, before the transaction enters the queue, where a + * rejection costs nothing. validate() itself cannot be reused there: it + * dereferences the GroupTreeAccount, and fetching a ~112 kB ratchet tree on + * every commit injection would cost more than the fee this saves. + * + * Every check here is copied from validate(), same order and same wording, so + * a transaction that clears this one fails later only for a reason that genuinely + * needs the tree -- or because the epoch moved underneath it between here and + * consensus, which is the race this cannot eliminate. + * + * Deliberately NOT a security boundary. It runs only on the node a client + * injects to; a transaction spread by gossip reaches handleSharedTX, which + * calls app.validate and never precracks. validate() inside apply() remains the + * check that actually decides anything. + */ +export const validatePreCrack = ( + tx: Tx.GroupCommit, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId] && wrappedStates[tx.groupId].data + + if (typeof from === 'undefined' || from === null) { + response.reason = '"from" account does not exist.' + return response + } + if (!isUserAccount(from)) { + response.reason = 'from account is not a UserAccount' + return response + } + if (typeof group === 'undefined' || group === null) { + response.reason = '"groupId" account does not exist.' + return response + } + if (!isGroupAccount(group)) { + response.reason = 'groupId account is not a GroupAccount' + return response + } + if (!group.members.includes(tx.from)) { + response.reason = 'sender is not a member of this group.' + return response + } + + // THE FENCE, screened early. This is the check the whole hook exists for: + // without it, every member that loses an ordinary epoch race pays a full fee + // to be told it lost. + if (tx.epoch !== group.epoch) { + response.reason = `stale epoch: commit targets epoch ${tx.epoch} but the group is at ${group.epoch}. Apply the latest commit and retry.` + return response + } + + const changesMembership = tx.addedMembers.length > 0 || tx.removedMembers.length > 0 + if (changesMembership && !group.admins.includes(tx.from)) { + response.reason = 'only an admin may add or remove members.' + return response + } + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < transactionFee) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + export const validate = ( tx: Tx.GroupCommit, wrappedStates: WrappedStates, @@ -581,9 +682,6 @@ export const apply = ( } tree.treeEpoch = group.epoch - console.log("thant: tree", tree) - console.log("thant: tree.ratchetTree", tree.ratchetTree) - /* * Drop the sender's own pending welcome. * diff --git a/test/groupCommitPreCrack.test.ts b/test/groupCommitPreCrack.test.ts new file mode 100644 index 00000000..46985072 --- /dev/null +++ b/test/groupCommitPreCrack.test.ts @@ -0,0 +1,139 @@ +import { validatePreCrack } from '../src/transactions/group_commit' +import * as AccountsStorage from '../src/storage/accountStorage' +import { ShardusTypes } from '@shardus/core' + +/** + * validatePreCrack is the subset of validate() that txPreCrackData can run + * before a transaction enters the queue, where a rejection is free. + * + * The property that matters most is the one asserted last: it must reach a + * verdict WITHOUT the GroupTreeAccount. If it ever starts needing the tree, + * precrack has to fetch ~112 kB per commit injection and the whole point of the + * hook is lost -- so that is pinned by a test rather than left to a comment. + */ + +const FEE = 10n ** 16n +const ADMIN = '1'.repeat(64) +const MEMBER = '2'.repeat(64) +const OUTSIDER = '3'.repeat(64) +const GROUP = '9'.repeat(64) + +beforeAll(() => { + // getTransactionFeeWei reads the cached network account; the pre-2.4.2 branch + // returns current.transactionFee directly, which keeps this fixture small. + ;(AccountsStorage as any).cachedNetworkAccount = { + current: { activeVersion: '1.0.0', transactionFee: FEE }, + } +}) + +const userAccount = (id: string, balance = FEE * 100n): any => ({ + id, + type: 'UserAccount', + data: { balance }, +}) + +const groupAccount = (epoch: number): any => ({ + id: GROUP, + type: 'GroupAccount', + members: [ADMIN, MEMBER], + admins: [ADMIN], + epoch, +}) + +const wrap = (accounts: Record): any => { + const out: Record = {} + for (const [id, data] of Object.entries(accounts)) { + out[id] = { accountId: id, stateId: '', data, timestamp: 0, accountCreated: false, isPartial: false } + } + return out +} + +const commitTx = (over: Record = {}): any => ({ + type: 'group_commit', + from: MEMBER, + groupId: GROUP, + epoch: 4, + addedMembers: [], + removedMembers: [], + fee: FEE, + ...over, +}) + +const run = (tx: any, states: any): ShardusTypes.IncomingTransactionResult => + validatePreCrack( + tx, + states, + { success: false, reason: 'Tx Validation Fails', status: 400 } as unknown as ShardusTypes.IncomingTransactionResult, + undefined as never, + ) + +const bothAccounts = (epoch: number, balance?: bigint): any => + wrap({ [MEMBER]: userAccount(MEMBER, balance), [GROUP]: groupAccount(epoch) }) + +describe('group_commit validatePreCrack', () => { + test('a commit naming the current epoch passes', () => { + const res = run(commitTx(), bothAccounts(4)) + expect(res.success).toBe(true) + }) + + test('a stale epoch is rejected before the queue', () => { + // The whole reason the hook exists: this is what a lost race looks like, + // and it used to cost the loser a full fee. + const res = run(commitTx({ epoch: 4 }), bothAccounts(5)) + expect(res.success).toBe(false) + expect(res.reason).toContain('stale epoch') + }) + + test('an epoch from the future is rejected too', () => { + const res = run(commitTx({ epoch: 9 }), bothAccounts(5)) + expect(res.success).toBe(false) + expect(res.reason).toContain('stale epoch') + }) + + test('a non-member is rejected', () => { + const states = wrap({ [OUTSIDER]: userAccount(OUTSIDER), [GROUP]: groupAccount(4) }) + const res = run(commitTx({ from: OUTSIDER }), states) + expect(res.success).toBe(false) + expect(res.reason).toContain('not a member') + }) + + test('a non-admin cannot add or remove members', () => { + const res = run(commitTx({ addedMembers: [OUTSIDER] }), bothAccounts(4)) + expect(res.success).toBe(false) + expect(res.reason).toContain('only an admin') + }) + + test('an admin changing membership passes', () => { + const states = wrap({ [ADMIN]: userAccount(ADMIN), [GROUP]: groupAccount(4) }) + const res = run(commitTx({ from: ADMIN, addedMembers: [OUTSIDER] }), states) + expect(res.success).toBe(true) + }) + + test('a missing group is reported in words, not as an exception', () => { + const res = run(commitTx(), wrap({ [MEMBER]: userAccount(MEMBER) })) + expect(res.success).toBe(false) + expect(res.reason).toContain('does not exist') + }) + + test('a fee below the network fee is rejected', () => { + const res = run(commitTx({ fee: FEE - 1n }), bothAccounts(4)) + expect(res.success).toBe(false) + expect(res.reason).toContain('network transaction fee') + }) + + test('a sender who cannot cover the fee is rejected', () => { + const res = run(commitTx(), bothAccounts(4, FEE - 1n)) + expect(res.success).toBe(false) + expect(res.reason).toContain('sufficient funds') + }) + + test('it reaches a verdict without the GroupTreeAccount', () => { + // No tree account is present in any fixture above, and this asserts that is + // load-bearing rather than incidental: precrack must never need to fetch it. + const states = bothAccounts(4) + const treeKeys = Object.keys(states).filter((k) => k !== MEMBER && k !== GROUP) + expect(treeKeys).toHaveLength(0) + expect(run(commitTx(), states).success).toBe(true) + expect(run(commitTx({ epoch: 99 }), states).success).toBe(false) + }) +}) diff --git a/test/groupCommitTreeDelta.test.ts b/test/groupCommitTreeDelta.test.ts new file mode 100644 index 00000000..9c61daa1 --- /dev/null +++ b/test/groupCommitTreeDelta.test.ts @@ -0,0 +1,77 @@ +import { validate_fields } from '../src/transactions/group_commit' +import * as config from '../src/config' +import { ShardusTypes } from '@shardus/core' + +/** + * The treeDelta node index must be bounded, not merely non-negative. + * + * applyTreeDelta grows the node array to reach entry.i, while the payload size + * cap measures only String(entry.n) + 16 -- so an enormous index is a tiny + * transaction that costs the account an enormous array. These assert the bound + * that stops it. + * + * validate_fields is the right place for it: handleSharedTX calls app.validate + * and nothing else, so this is the only check a transaction arriving by gossip + * has to clear. + */ + +const MAX_INDEX = 2 * config.LiberdusFlags.groupMaxMembers + +// A field-valid commit, up to the point validate_fields inspects treeDelta. +const baseTx = (treeDelta: { i: number; n: string | null }[]): any => ({ + type: 'group_commit', + from: '1'.repeat(64), + groupId: '2'.repeat(64), + epoch: 0, + commit: 'AAAA', + proposals: [], + pskId: '', + pskNonce: '', + groupInfo: '', + ratchetTree: '', + treeDelta, +}) + +const run = (tx: any): ShardusTypes.IncomingTransactionResult => { + const response = { + success: false, + reason: 'Invalid transaction', + status: 400, + } as unknown as ShardusTypes.IncomingTransactionResult + return validate_fields(tx as any, response) +} + +describe('group_commit treeDelta node index bound', () => { + test('the amplification payload is rejected', () => { + // ~18 bytes on the size cap, fifty million array slots once applied. + const res = run(baseTx([{ i: 50_000_000, n: 'AA' }])) + expect(res.reason).toContain('exceeds the maximum') + }) + + test('an index just past the ceiling is rejected', () => { + const res = run(baseTx([{ i: MAX_INDEX + 1, n: 'AA' }])) + expect(res.reason).toContain('exceeds the maximum') + }) + + test('a blanking entry cannot smuggle a large index either', () => { + const res = run(baseTx([{ i: 50_000_000, n: null }])) + expect(res.reason).toContain('exceeds the maximum') + }) + + test('the largest legitimate index is accepted', () => { + // A full tree of groupMaxMembers leaves reaches node index 2N-2, so the + // ceiling itself must still pass or a legitimate commit would be refused. + const res = run(baseTx([{ i: MAX_INDEX, n: 'AA' }])) + expect(res.reason).not.toContain('exceeds the maximum') + }) + + test('ordinary small indices are unaffected', () => { + const res = run(baseTx([{ i: 0, n: 'AA' }, { i: 3, n: null }])) + expect(res.reason).not.toContain('node index') + }) + + test('a negative index is still rejected by the existing check', () => { + const res = run(baseTx([{ i: -1, n: 'AA' }])) + expect(res.reason).toContain('invalid node index') + }) +}) From 2132045d14eea26ad1729cb1f5f9dce5edb65881 Mon Sep 17 00:00:00 2001 From: Thant Sin Toe Date: Mon, 31 Aug 2026 14:56:44 +0700 Subject: [PATCH 4/7] Give a group a balance to pay for its own tree repair Removing a member blanks its ancestors, and someone has to spend a commit to fill them in. That someone is whichever member happens to sit nearby, so today a bystander pays for the group's upkeep. This adds the balance that will pay instead; spending it comes next. Funded per added member, by the admin doing the adding, so the cost of a member's eventual departure is paid by whoever chose to admit them. A commit that adds nobody owes nothing -- a rekey or a removal draws the balance down, it does not top it up. Held in LIB rather than as a count of prepaid repairs. What a deposit needs to buy is one future repair commit, and the price of that is whatever the fee is on the day it happens, so the deposit is a multiple of the live fee and solvency gets judged against the fee current when it is read. A count fixed at add time goes wrong the moment the fee moves. repairDepositOwed is shared by validate, validatePreCrack and apply on purpose: if they disagreed about the amount, a commit could pass validation and then underflow the admin's balance in apply. The serializer carries no version tag, so appending a field would make every existing group unloadable -- readString past the end of the buffer. The read is guarded on isAtOrPastEnd instead, and an older buffer decodes as an unfunded group. A test builds an old-layout buffer field by field to hold that compatibility in place. Co-Authored-By: Claude Opus 5 --- src/@types/index.ts | 26 ++++ src/accounts/groupAccount.ts | 18 +++ src/config/index.ts | 11 ++ src/transactions/group_commit.ts | 46 ++++++- test/groupMaintenanceBalance.test.ts | 186 +++++++++++++++++++++++++++ 5 files changed, 283 insertions(+), 4 deletions(-) create mode 100644 test/groupMaintenanceBalance.test.ts diff --git a/src/@types/index.ts b/src/@types/index.ts index 768c8a2d..5e3467f9 100644 --- a/src/@types/index.ts +++ b/src/@types/index.ts @@ -953,6 +953,32 @@ export interface GroupAccount { /** Addresses refused admission. The group's analogue of toll.required = 2. */ blocked: string[] + // --- maintenance ---------------------------------------------------------- + /** + * Pays the transaction fee for repairing this group's ratchet tree. + * + * Removing a member blanks its ancestors, and someone has to spend a commit + * to fill them back in. That someone is an ordinary member who happened to + * sit nearby, so charging them makes a bystander pay for the group's own + * upkeep. This balance pays instead. + * + * Funded per added member at add time, by the admin doing the adding: each + * member prepays for the cleanup their eventual departure causes. + * + * Held in LIB rather than as a count of prepaid repairs, deliberately. A + * count fixed at add time goes wrong the moment the network fee moves; + * solvency is judged against the fee current at the time it is read. + * + * NOT withdrawable. It leaves only as a burned repair fee, which is what + * makes it uninteresting to steal. It also never pays for a FAILED + * transaction -- see group_commit -- because that would hand anyone who can + * inject transactions a way to drain it. + * + * Optional on the wire: groups created before this field existed deserialize + * with zero rather than failing. + */ + maintenanceBalance: bigint + // --- misc ----------------------------------------------------------------- meta: string // client-encrypted group name/avatar; opaque here maxMembers: number diff --git a/src/accounts/groupAccount.ts b/src/accounts/groupAccount.ts index 08b0dc58..2d051359 100644 --- a/src/accounts/groupAccount.ts +++ b/src/accounts/groupAccount.ts @@ -40,6 +40,10 @@ export const groupAccount = (accountId: string, tx: Tx.GroupCreate, timestamp: n joinFee: tx.joinFee ?? BigInt(0), blocked: [], + // Empty at creation: the founder is the only member, and deposits arrive + // one per added member as the group grows. + maintenanceBalance: BigInt(0), + meta: tx.meta, maxMembers: tx.maxMembers, lastMessageAt: {}, @@ -93,6 +97,10 @@ export const serializeGroupAccount = (stream: VectorBufferStream, inp: GroupAcco stream.writeUInt32(inp.maxMembers) stream.writeString(inp.createdBy) stream.writeUInt8(inp.hasChats ? 1 : 0) + + // Appended last, and read back defensively, so a group serialized before this + // field existed still deserializes. See the read side. + stream.writeString((inp.maintenanceBalance ?? BigInt(0)).toString()) } export const deserializeGroupAccount = (stream: VectorBufferStream, root = false): GroupAccount => { @@ -133,6 +141,15 @@ export const deserializeGroupAccount = (stream: VectorBufferStream, root = false const createdBy = stream.readString() const hasChats = stream.readUInt8() === 1 + /* + * maintenanceBalance was added after groups were already being serialized, + * and these serializers carry no version tag. Reading it only when bytes + * remain lets an older buffer decode as an unfunded group instead of + * throwing, which is the difference between "this group has no deposits yet" + * and "this group can no longer be loaded". + */ + const maintenanceBalance = stream.isAtOrPastEnd() ? BigInt(0) : BigInt(stream.readString()) + return { id, type, @@ -148,6 +165,7 @@ export const deserializeGroupAccount = (stream: VectorBufferStream, root = false treeId, joinFee, blocked, + maintenanceBalance, meta, maxMembers, lastMessageAt, diff --git a/src/config/index.ts b/src/config/index.ts index cd7f06b0..c688f176 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -293,6 +293,16 @@ interface LiberdusFlags { enableGroupChat: boolean /** Hard cap on members per group. Bounds commit size and account hotspotting. */ groupMaxMembers: number + /** + * Repair deposit charged per added member, as a multiple of the current + * transaction fee, and paid into the group's maintenanceBalance. + * + * A multiple of the live fee rather than a fixed amount of LIB: what the + * deposit needs to buy is one future repair commit, and the price of that is + * whatever the fee is on the day it happens. Two covers a repair with + * headroom. + */ + groupRepairDepositMultiplier: number /** Members addable/removable in one commit (bounds the transaction key set). */ groupMaxMembersPerCommit: number /** Max size of one MLS application message, in kB. */ @@ -367,6 +377,7 @@ export const LiberdusFlags: LiberdusFlags = { enableGroupChat: true, groupMaxMembers: 50, groupMaxMembersPerCommit: 10, + groupRepairDepositMultiplier: 2, // 64 kB. Measured, not estimated: the commit blob is ~3.9 kB on an add and // ~0.4 kB on a remove, and does NOT grow with the group. What grows is the // ratchet tree carried in each welcome — ~1.8 kB per existing member — so in diff --git a/src/transactions/group_commit.ts b/src/transactions/group_commit.ts index 99ed3c5f..b3f11185 100644 --- a/src/transactions/group_commit.ts +++ b/src/transactions/group_commit.ts @@ -230,6 +230,25 @@ export const validate_fields = (tx: Tx.GroupCommit, response: ShardusTypes.Incom return response } +/** + * What this commit owes the group's maintenance balance: one repair deposit per + * added member, priced at the fee current right now. + * + * Charged to the admin doing the adding, so the cost of a member's eventual + * departure is paid by whoever chose to admit them, at the moment they choose + * it. Nothing is owed by a commit that adds nobody -- a plain rekey or a + * removal takes money OUT of the balance, it does not put more in. + * + * Shared by validate, validatePreCrack and apply so the three cannot disagree + * about the amount; a mismatch would let a commit pass validation and then + * underflow the admin's balance in apply. + */ +const repairDepositOwed = (tx: Tx.GroupCommit, transactionFee: bigint): bigint => { + if (tx.addedMembers.length === 0) return BigInt(0) + const perMember = SafeBigIntMath.multiply(transactionFee, BigInt(config.LiberdusFlags.groupRepairDepositMultiplier)) + return SafeBigIntMath.multiply(perMember, BigInt(tx.addedMembers.length)) +} + /** * The subset of validate() that needs only the GroupAccount and the sender. * @@ -298,8 +317,9 @@ export const validatePreCrack = ( response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` return response } - if (from.data.balance < transactionFee) { - response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + const depositOwed = repairDepositOwed(tx, transactionFee) + if (from.data.balance < SafeBigIntMath.add(transactionFee, depositOwed)) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}) and the repair deposit (${depositOwed}).` return response } @@ -488,8 +508,9 @@ export const validate = ( response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` return response } - if (from.data.balance < transactionFee) { - response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}).` + const depositOwed = repairDepositOwed(tx, transactionFee) + if (from.data.balance < SafeBigIntMath.add(transactionFee, depositOwed)) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}) and the repair deposit (${depositOwed}).` return response } @@ -562,6 +583,23 @@ export const apply = ( const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + /* + * Collect the repair deposit for anyone this commit admits. + * + * Unlike the fee, this is not burned: it moves from the admin into the + * group, where it can only ever be spent paying the fee on a future repair + * commit. validate() has already checked the admin can cover fee + deposit, + * so this cannot underflow. + * + * `?? 0` because a group serialized before maintenanceBalance existed + * deserializes without it. + */ + const depositOwed = repairDepositOwed(tx, transactionFee) + if (depositOwed > BigInt(0)) { + from.data.balance = SafeBigIntMath.subtract(from.data.balance, depositOwed) + group.maintenanceBalance = SafeBigIntMath.add(group.maintenanceBalance ?? BigInt(0), depositOwed) + } + const previousEpoch = group.epoch // Record the commit before mutating membership, so the transcript reads as diff --git a/test/groupMaintenanceBalance.test.ts b/test/groupMaintenanceBalance.test.ts new file mode 100644 index 00000000..70c403ee --- /dev/null +++ b/test/groupMaintenanceBalance.test.ts @@ -0,0 +1,186 @@ +import { VectorBufferStream } from '@shardus/core' +import * as crypto from '../src/crypto' +import { Utils } from '@shardus/lib-types' +import { groupAccount, serializeGroupAccount, deserializeGroupAccount } from '../src/accounts/groupAccount' +import { SerdeTypeIdent } from '../src/accounts' +import { validatePreCrack } from '../src/transactions/group_commit' +import * as AccountsStorage from '../src/storage/accountStorage' +import * as config from '../src/config' +import { GroupAccount } from '../src/@types' +import { ShardusTypes } from '@shardus/core' + +/** + * The group's maintenance balance: what funds it, and that it survives a + * serialization round trip -- including from a buffer written before the field + * existed, which must decode as an unfunded group rather than throwing. + */ + +const FEE = 10n ** 16n +const MULT = BigInt(config.LiberdusFlags.groupRepairDepositMultiplier) +const ADMIN = '1'.repeat(64) +const MEMBER = '2'.repeat(64) +const JOINER = '3'.repeat(64) +const JOINER2 = '4'.repeat(64) +const GROUP = '9'.repeat(64) + +beforeAll(() => { + // groupAccount() hashes itself and derives the tree id, both of which need + // the crypto module keyed. Same key the app and the other suites use. + crypto.init('69fa4195670576c0160d660c3be36556ff8d504725be8a59b5a96509e0c994bc') + // hashObj walks the account, which holds bigints; the app installs the same + // BigInt-aware stringifier at startup (src/index.ts). + crypto.setCustomStringifier(Utils.safeStringify, 'shardus_safeStringify') + ;(AccountsStorage as any).cachedNetworkAccount = { + current: { activeVersion: '1.0.0', transactionFee: FEE }, + } +}) + +const makeGroup = (): GroupAccount => + groupAccount( + GROUP, + { + from: ADMIN, + groupId: GROUP, + groupNonce: 'ab'.repeat(16), + mlsGroupId: 'ff'.repeat(8), + cipherSuite: 84, + meta: 'encrypted-meta', + maxMembers: 20, + joinFee: BigInt(0), + fee: FEE, + } as any, + 1_700_000_000_000, + ) + +const wrap = (accounts: Record): any => { + const out: Record = {} + for (const [id, data] of Object.entries(accounts)) { + out[id] = { accountId: id, stateId: '', data, timestamp: 0, accountCreated: false, isPartial: false } + } + return out +} + +const runPreCrack = (tx: any, states: any): ShardusTypes.IncomingTransactionResult => + validatePreCrack( + tx, + states, + { success: false, reason: 'Tx Validation Fails', status: 400 } as unknown as ShardusTypes.IncomingTransactionResult, + undefined as never, + ) + +const commitTx = (over: Record = {}): any => ({ + type: 'group_commit', + from: ADMIN, + groupId: GROUP, + epoch: 0, + addedMembers: [], + removedMembers: [], + fee: FEE, + ...over, +}) + +const groupAt = (epoch: number, admins = [ADMIN]): any => ({ + id: GROUP, + type: 'GroupAccount', + members: [ADMIN, MEMBER], + admins, + epoch, +}) + +const user = (id: string, balance: bigint): any => ({ id, type: 'UserAccount', data: { balance } }) + +describe('a new group starts unfunded', () => { + test('maintenanceBalance is zero at creation', () => { + expect(makeGroup().maintenanceBalance).toBe(BigInt(0)) + }) +}) + +describe('the repair deposit is charged for added members', () => { + const states = (balance: bigint): any => wrap({ [ADMIN]: user(ADMIN, balance), [GROUP]: groupAt(0) }) + + test('an admin who can cover fee plus one deposit passes', () => { + const needed = FEE + FEE * MULT + expect(runPreCrack(commitTx({ addedMembers: [JOINER] }), states(needed)).success).toBe(true) + }) + + test('an admin one wei short of fee plus deposit is rejected', () => { + const needed = FEE + FEE * MULT + const res = runPreCrack(commitTx({ addedMembers: [JOINER] }), states(needed - 1n)) + expect(res.success).toBe(false) + expect(res.reason).toContain('repair deposit') + }) + + test('the deposit scales with the number of members added', () => { + // Enough for one joiner, not for two. + const oneJoiner = FEE + FEE * MULT + const res = runPreCrack(commitTx({ addedMembers: [JOINER, JOINER2] }), states(oneJoiner)) + expect(res.success).toBe(false) + expect(res.reason).toContain('repair deposit') + + const twoJoiners = FEE + FEE * MULT * 2n + expect(runPreCrack(commitTx({ addedMembers: [JOINER, JOINER2] }), states(twoJoiners)).success).toBe(true) + }) + + test('a commit that adds nobody owes no deposit', () => { + // A plain rekey or a removal only needs the fee itself. + const res = runPreCrack(commitTx({ from: MEMBER }), wrap({ [MEMBER]: user(MEMBER, FEE), [GROUP]: groupAt(0) })) + expect(res.success).toBe(true) + }) +}) + +describe('serialization', () => { + test('a funded balance survives a round trip', () => { + const group = makeGroup() + group.maintenanceBalance = FEE * 7n + + const out = new VectorBufferStream(0) + serializeGroupAccount(out, group, true) + const back = deserializeGroupAccount(VectorBufferStream.fromBuffer(out.getBuffer()), true) + + expect(back.maintenanceBalance).toBe(FEE * 7n) + // and nothing ahead of the new field shifted + expect(back.id).toBe(group.id) + expect(back.epoch).toBe(group.epoch) + expect(back.members).toEqual(group.members) + expect(back.joinFee).toBe(group.joinFee) + expect(back.createdBy).toBe(group.createdBy) + expect(back.hasChats).toBe(group.hasChats) + }) + + test('a buffer written before the field existed decodes as unfunded', () => { + // The old layout, field for field, stopping where the old serializer did. + // This is the case that would otherwise read past the end and throw, + // making existing groups unloadable rather than merely unfunded. + const group = makeGroup() + const old = new VectorBufferStream(0) + old.writeUInt16(SerdeTypeIdent.GroupAccount) + old.writeString(group.id) + old.writeString(group.type) + old.writeString(group.hash) + old.writeBigUInt64(BigInt(group.timestamp)) + old.writeString(group.mlsGroupId) + old.writeUInt16(group.cipherSuite) + old.writeUInt32(group.epoch) + old.writeUInt32(group.members.length) + for (const m of group.members) old.writeString(m) + old.writeUInt32(group.admins.length) + for (const a of group.admins) old.writeString(a) + old.writeString(Utils.safeStringify(group.memberSince)) + old.writeString(Utils.safeStringify(group.messages)) + old.writeString(Utils.safeStringify(group.lastMessageAt)) + old.writeString(group.treeId) + old.writeString(group.joinFee.toString()) + old.writeString(Utils.safeStringify(group.blocked)) + old.writeString(group.meta) + old.writeUInt32(group.maxMembers) + old.writeString(group.createdBy) + old.writeUInt8(group.hasChats ? 1 : 0) + // ...and no maintenanceBalance. + + const back = deserializeGroupAccount(VectorBufferStream.fromBuffer(old.getBuffer()), true) + expect(back.maintenanceBalance).toBe(BigInt(0)) + expect(back.id).toBe(group.id) + expect(back.createdBy).toBe(group.createdBy) + expect(back.maxMembers).toBe(group.maxMembers) + }) +}) From db774af9b1bdf4d05e96944bc11f41e1f96c0ac3 Mon Sep 17 00:00:00 2001 From: Thant Sin Toe Date: Mon, 31 Aug 2026 15:02:27 +0700 Subject: [PATCH 5/7] Charge tree repair to the group, not to the bystander The group's balance now pays the fee on a commit that repairs the tree, and the member who happened to perform it is left alone. That member was never the cause of the work: a removal blanks the departing leaf's ancestors, and whoever sits nearby ends up committing the path update that fills them in. Eligibility is derived, not declared. The tree is stored as a JSON array of node-or-null, so "was that node blank" is an array lookup -- no MLS parsing, and nothing the sender can assert. It deliberately asks whether the tree was damaged and this commit repairs it, rather than whether the sender is the member who ought to have done it; the server holds members[] but not leaf indices, so it could not answer the second question. Two calls worth knowing. An index at or past the end of the array counts as blank, because trailing blanks are trimmed on write and a removal on the right of the tree leaves its ancestors -- root included -- simply absent; refusing those would deny the subsidy to exactly the repairs it exists to fund. And a delta entry that blanks a node never counts, since blanking is damage. The sender's balance requirement is lifted for a repair the group will cover, or the member with an empty wallet still could not submit one. validate() can check that exactly. validatePreCrack cannot -- it has no tree, by design -- so it approximates with the questions the GroupAccount can answer and errs towards letting work through, because a pre-queue hook wrongly refusing an honest repair is the failure that would actually hurt. The balance still never pays for a FAILED commit. Anyone who can inject transactions can generate those at will, since precrack only screens the node a client injects to, so paying for them would be an open drain. Absent tx arrays are read as empty throughout. validate_fields rejects non-arrays before any of this runs, so the only way to arrive without them is a caller that skipped it, and "nobody was added" is the right reading. Co-Authored-By: Claude Opus 5 --- src/transactions/group_commit.ts | 138 ++++++++++++++++- test/groupRepairPaysFromBalance.test.ts | 194 ++++++++++++++++++++++++ 2 files changed, 327 insertions(+), 5 deletions(-) create mode 100644 test/groupRepairPaysFromBalance.test.ts diff --git a/src/transactions/group_commit.ts b/src/transactions/group_commit.ts index b3f11185..f5f754ea 100644 --- a/src/transactions/group_commit.ts +++ b/src/transactions/group_commit.ts @@ -230,6 +230,65 @@ export const validate_fields = (tx: Tx.GroupCommit, response: ShardusTypes.Incom return response } +/** + * Does this commit repair the tree, rather than change it? + * + * A removal blanks every ancestor of the departing leaf, and the group stays + * degraded until some member commits a path update that fills them back in. + * That is the work the maintenance balance exists to pay for, and this is how + * the network recognises it -- from state it already holds, with nothing + * asserted by the sender. + * + * The tree is stored as a JSON array of node-or-null (see applyTreeDelta), so + * "was that node blank" is an array lookup. No MLS parsing is involved, and no + * leaf is attributed to anyone: the question is whether the tree was damaged + * and this commit repairs it, NOT whether the sender is the member who ought to + * have done it. The server stores members[] but not leaf indices, so it could + * not answer the second question anyway. + * + * Two judgement calls worth knowing about: + * + * - An index at or past the end of the array counts as blank. Trailing blanks + * are trimmed on write, so a removal on the right-hand side of the tree + * leaves its ancestors -- up to and including the root -- simply absent. + * Refusing to count those would deny the subsidy to exactly the repairs it + * is meant to fund. + * - A delta entry that blanks a node (n === null) never counts. Blanking is + * damage, not repair. + * + * MUST be evaluated before applyTreeDelta runs, while the stored tree is still + * the pre-commit one. + */ +const fillsABlankNode = (storedTree: string, delta: { i: number; n: string | null }[]): boolean => { + let nodes: (string | null)[] = [] + if (storedTree.length > 0) { + try { + const parsed = JSON.parse(storedTree) + if (Array.isArray(parsed)) nodes = parsed + } catch { + // An unreadable tree is not something to hand out a subsidy for. + return false + } + } + for (const entry of delta) { + if (entry.n === null) continue + if (entry.i >= nodes.length || nodes[entry.i] === null) return true + } + return false +} + +/** + * A commit that changes no membership -- the shape a path update takes. + * + * Absent arrays are read as empty. validate_fields has already rejected any + * that are present but not arrays, so the only way to get here with one missing + * is a caller that skipped it, and "nobody was added" is the right reading. + */ +const isMembershipNeutral = (tx: Tx.GroupCommit): boolean => + (tx.addedMembers?.length ?? 0) === 0 && + (tx.removedMembers?.length ?? 0) === 0 && + (tx.welcomes?.length ?? 0) === 0 + /** * What this commit owes the group's maintenance balance: one repair deposit per * added member, priced at the fee current right now. @@ -244,9 +303,10 @@ export const validate_fields = (tx: Tx.GroupCommit, response: ShardusTypes.Incom * underflow the admin's balance in apply. */ const repairDepositOwed = (tx: Tx.GroupCommit, transactionFee: bigint): bigint => { - if (tx.addedMembers.length === 0) return BigInt(0) + const added = tx.addedMembers?.length ?? 0 + if (added === 0) return BigInt(0) const perMember = SafeBigIntMath.multiply(transactionFee, BigInt(config.LiberdusFlags.groupRepairDepositMultiplier)) - return SafeBigIntMath.multiply(perMember, BigInt(tx.addedMembers.length)) + return SafeBigIntMath.multiply(perMember, BigInt(added)) } /** @@ -318,7 +378,28 @@ export const validatePreCrack = ( return response } const depositOwed = repairDepositOwed(tx, transactionFee) - if (from.data.balance < SafeBigIntMath.add(transactionFee, depositOwed)) { + + /* + * The same excusal validate() makes for a group-funded repair, approximated. + * + * Whether a commit really fills a blank cannot be answered here: that needs + * the ratchet tree, and fetching it per injection is the cost this hook + * exists to avoid. So this asks the two questions the GroupAccount can + * answer -- is the commit membership-neutral, and is the balance able to pay + * -- plus a cheap shape check that the delta writes at least one node. + * + * The approximation is deliberately loose in the safe direction. It can let + * through a broke sender whose commit turns out not to be a repair, and + * validate() then rejects it inside apply(); it will never reject a genuine + * repair that validate() would have excused. A pre-queue hook wrongly + * refusing honest work is the failure that would actually hurt. + */ + const writesANode = (tx.treeDelta ?? []).some((entry) => entry && entry.n !== null) + const groupMightPayTheFee = + isMembershipNeutral(tx) && writesANode && (group.maintenanceBalance ?? BigInt(0)) >= transactionFee + + const senderOwes = groupMightPayTheFee ? depositOwed : SafeBigIntMath.add(transactionFee, depositOwed) + if (from.data.balance < senderOwes) { response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}) and the repair deposit (${depositOwed}).` return response } @@ -509,7 +590,24 @@ export const validate = ( return response } const depositOwed = repairDepositOwed(tx, transactionFee) - if (from.data.balance < SafeBigIntMath.add(transactionFee, depositOwed)) { + + /* + * A repair the group's balance will cover does not need the sender to hold + * anything. Without this the member with an empty wallet -- exactly the + * person this whole mechanism is meant to stop charging -- still could not + * submit the repair. + * + * The condition matches apply() exactly, including the tree lookup, so a + * commit excused here is the same commit the balance goes on to pay for. + */ + const groupWillPayTheFee = + isMembershipNeutral(tx) && + !!treeForValidate && + fillsABlankNode(treeForValidate.ratchetTree, tx.treeDelta) && + (group.maintenanceBalance ?? BigInt(0)) >= transactionFee + + const senderOwes = groupWillPayTheFee ? depositOwed : SafeBigIntMath.add(transactionFee, depositOwed) + if (from.data.balance < senderOwes) { response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}) and the repair deposit (${depositOwed}).` return response } @@ -581,7 +679,37 @@ export const apply = ( } const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) - from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + + /* + * Who pays the fee. + * + * A repair commit is the group's own upkeep, so the group's balance pays it + * and the member who happened to perform it is left alone. Anything else -- + * and any repair the balance cannot cover -- is charged to the sender as + * before. + * + * Evaluated here, ahead of applyTreeDelta, because fillsABlankNode has to see + * the tree as it was before this commit. + * + * The fee is burned either way: it is subtracted from an account and reported + * in the receipt, with no credit anywhere. This decides which account it + * comes out of, and changes nothing about supply. + * + * Note what is NOT here: a failed commit never reaches apply(), and the + * balance never pays for one. Anyone able to inject transactions can generate + * failures at will -- precrack only screens the node a client injects to -- + * so paying for them would be an open drain. + */ + const groupPaysTheFee = + isMembershipNeutral(tx) && + fillsABlankNode(tree.ratchetTree, tx.treeDelta) && + (group.maintenanceBalance ?? BigInt(0)) >= transactionFee + + if (groupPaysTheFee) { + group.maintenanceBalance = SafeBigIntMath.subtract(group.maintenanceBalance ?? BigInt(0), transactionFee) + } else { + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } /* * Collect the repair deposit for anyone this commit admits. diff --git a/test/groupRepairPaysFromBalance.test.ts b/test/groupRepairPaysFromBalance.test.ts new file mode 100644 index 00000000..fda176ba --- /dev/null +++ b/test/groupRepairPaysFromBalance.test.ts @@ -0,0 +1,194 @@ +import { VectorBufferStream } from '@shardus/core' +import { Utils } from '@shardus/lib-types' +import * as crypto from '../src/crypto' +import { groupAccount } from '../src/accounts/groupAccount' +import { groupTreeAccount } from '../src/accounts/groupTreeAccount' +import { apply } from '../src/transactions/group_commit' +import * as AccountsStorage from '../src/storage/accountStorage' +import * as utils from '../src/utils' +import { GroupAccount, GroupTreeAccount, UserAccount } from '../src/@types' + +/** + * Who pays the fee for a commit. + * + * The rule the group's balance is there to express: a commit that repairs the + * tree is the group's own upkeep and the group pays for it; everything else is + * charged to whoever sent it. "Repairs the tree" is decided from the stored + * node array, so these drive apply() with real accounts and watch the two + * balances move. + */ + +const FEE = 10n ** 16n +const ADMIN = '1'.repeat(64) +const MEMBER = '2'.repeat(64) +const GROUP = '9'.repeat(64) + +beforeAll(() => { + crypto.init('69fa4195670576c0160d660c3be36556ff8d504725be8a59b5a96509e0c994bc') + crypto.setCustomStringifier(Utils.safeStringify, 'shardus_safeStringify') + ;(AccountsStorage as any).cachedNetworkAccount = { + current: { activeVersion: '1.0.0', transactionFee: FEE }, + } +}) + +const dapp: any = { log: () => undefined, applyResponseAddReceiptData: () => undefined } + +const makeGroup = (over: Partial = {}): GroupAccount => { + const g = groupAccount( + GROUP, + { + from: ADMIN, + groupId: GROUP, + groupNonce: 'ab'.repeat(16), + mlsGroupId: 'ff'.repeat(8), + cipherSuite: 84, + meta: 'meta', + maxMembers: 20, + joinFee: BigInt(0), + fee: FEE, + } as any, + 1_700_000_000_000, + ) + g.members = [ADMIN, MEMBER] + g.admins = [ADMIN] + return Object.assign(g, over) +} + +/** A stored tree: `null` is a blank node, a string is an occupied one. */ +const makeTree = (nodes: (string | null)[]): GroupTreeAccount => { + const t = groupTreeAccount(utils.calculateGroupTreeId(GROUP), GROUP) + t.ratchetTree = JSON.stringify(nodes) + return t +} + +const user = (id: string, balance: bigint): UserAccount => + ({ id, type: 'UserAccount', hash: '', timestamp: 0, data: { balance, chats: {} } } as any) + +const commitTx = (over: Record = {}): any => ({ + type: 'group_commit', + from: MEMBER, + groupId: GROUP, + epoch: 0, + commit: 'Y29tbWl0', + proposals: [], + pskId: '', + pskNonce: '', + groupInfo: '', + ratchetTree: '', + treeDelta: [], + addedMembers: [], + removedMembers: [], + welcomes: [], + consumedKeyPackages: [], + fee: FEE, + timestamp: 1_700_000_100_000, + sign: { owner: MEMBER, sig: '00' }, + ...over, +}) + +/** Runs apply() and reports where the fee came from. */ +const applyCommit = ( + tx: any, + opts: { treeNodes: (string | null)[]; balance: bigint; senderBalance?: bigint }, +): { group: GroupAccount; sender: UserAccount } => { + const group = makeGroup({ maintenanceBalance: opts.balance }) + const tree = makeTree(opts.treeNodes) + const sender = user(tx.from, opts.senderBalance ?? FEE * 100n) + const wrappedStates: any = { + [tx.from]: { data: sender }, + [GROUP]: { data: group }, + [utils.calculateGroupTreeId(GROUP)]: { data: tree }, + } + // apply() writes a chat entry onto each added member, so they have to be here. + for (const address of tx.addedMembers) wrappedStates[address] = { data: user(address, BigInt(0)) } + apply(tx, 1_700_000_200_000, 'tx-id', wrappedStates, dapp, {} as any) + return { group, sender } +} + +describe('a repair commit is paid for by the group', () => { + test('filling a blank node draws on the maintenance balance, not the sender', () => { + const startingSender = FEE * 100n + const { group, sender } = applyCommit(commitTx({ treeDelta: [{ i: 1, n: 'bm9kZQ==' }] }), { + treeNodes: ['a', null, 'c'], // node 1 is blank + balance: FEE * 5n, + senderBalance: startingSender, + }) + expect(group.maintenanceBalance).toBe(FEE * 4n) + expect(sender.data.balance).toBe(startingSender) + }) + + test('a member with an empty wallet can still perform a funded repair', () => { + // The person this whole mechanism exists to stop charging. + const { group, sender } = applyCommit(commitTx({ treeDelta: [{ i: 1, n: 'bm9kZQ==' }] }), { + treeNodes: ['a', null, 'c'], + balance: FEE * 3n, + senderBalance: BigInt(0), + }) + expect(group.maintenanceBalance).toBe(FEE * 2n) + expect(sender.data.balance).toBe(BigInt(0)) + }) + + test('a node past the end of the trimmed array counts as blank', () => { + // Trailing blanks are trimmed on write, so a removal on the right of the + // tree leaves its ancestors simply absent rather than present-and-null. + const { group } = applyCommit(commitTx({ treeDelta: [{ i: 6, n: 'bm9kZQ==' }] }), { + treeNodes: ['a', 'b', 'c'], + balance: FEE * 5n, + }) + expect(group.maintenanceBalance).toBe(FEE * 4n) + }) +}) + +describe('everything else is charged to the sender', () => { + test('a commit that overwrites occupied nodes is not a repair', () => { + const startingSender = FEE * 100n + const { group, sender } = applyCommit(commitTx({ treeDelta: [{ i: 0, n: 'bm9kZQ==' }] }), { + treeNodes: ['a', 'b', 'c'], // nothing blank + balance: FEE * 5n, + senderBalance: startingSender, + }) + expect(group.maintenanceBalance).toBe(FEE * 5n) + expect(sender.data.balance).toBe(startingSender - FEE) + }) + + test('blanking a node is damage, not repair', () => { + const startingSender = FEE * 100n + const { group, sender } = applyCommit(commitTx({ treeDelta: [{ i: 1, n: null }] }), { + treeNodes: ['a', 'b', 'c'], + balance: FEE * 5n, + senderBalance: startingSender, + }) + expect(group.maintenanceBalance).toBe(FEE * 5n) + expect(sender.data.balance).toBe(startingSender - FEE) + }) + + test('an underfunded group cannot pay, so the sender does', () => { + const startingSender = FEE * 100n + const { group, sender } = applyCommit(commitTx({ treeDelta: [{ i: 1, n: 'bm9kZQ==' }] }), { + treeNodes: ['a', null, 'c'], + balance: FEE - 1n, // one wei short + senderBalance: startingSender, + }) + expect(group.maintenanceBalance).toBe(FEE - 1n) + expect(sender.data.balance).toBe(startingSender - FEE) + }) + + test('a commit that adds a member is never a repair, and collects a deposit', () => { + const startingSender = FEE * 100n + const tx = commitTx({ + from: ADMIN, + addedMembers: ['7'.repeat(64)], + welcomes: [], + treeDelta: [{ i: 1, n: 'bm9kZQ==' }], // fills a blank, but membership changed + }) + const { group, sender } = applyCommit(tx, { + treeNodes: ['a', null, 'c'], + balance: FEE * 5n, + senderBalance: startingSender, + }) + const deposit = FEE * 2n // groupRepairDepositMultiplier + // Balance went UP by the deposit and was not spent on the fee. + expect(group.maintenanceBalance).toBe(FEE * 5n + deposit) + expect(sender.data.balance).toBe(startingSender - FEE - deposit) + }) +}) From f7d0bcdfb068c9e16228ad662c40d0462b6d5122 Mon Sep 17 00:00:00 2001 From: Thant Sin Toe Date: Mon, 31 Aug 2026 16:31:01 +0700 Subject: [PATCH 6/7] Let anyone top up a group's maintenance balance group_commit collects a deposit per added member and spends it repairing the tree, but between those two there was no way to put anything in. This adds group_maintenance_fund, and exposes the balance on GET /group/:groupId so a client can see when upkeep is running out. Funding is open to anyone, member or not. The balance is spendable on exactly one thing -- burning the fee on a repair commit -- and there is no withdrawal transaction anywhere in the system, so a contribution cannot be redirected or taken back out. That is what makes a stranger's contribution safe to accept and the balance uninteresting to steal. A test walks src/transactions and fails if anything other than group_commit ever subtracts from it, so the argument stays true rather than merely documented. The balance is exposed over the API because the server cannot raise the alarm itself: the transcript is ciphertext it holds no key for, so it can never write into a group chat. Warning about low upkeep has to be the client's job, and the client needs the number. No end-of-life payout, contrary to the plan. That was designed around the last member leaving and taking the remainder, but group_leave refuses to let the last member go at all -- so a group always keeps at least one, and a one-member group has no copath and can never need a repair. The balance is never stranded, only idle until the group grows again, at which point the adds that grow it top it up anyway. Adding an exit would have weakened the no-withdrawal invariant to solve a problem that does not exist. Co-Authored-By: Claude Opus 5 --- src/@types/index.ts | 32 ++- src/api/group/group.ts | 9 + src/index.ts | 1 + src/transactions/group_maintenance_fund.ts | 214 +++++++++++++++++++++ src/transactions/index.ts | 2 + test/groupMaintenanceFund.test.ts | 162 ++++++++++++++++ 6 files changed, 416 insertions(+), 4 deletions(-) create mode 100644 src/transactions/group_maintenance_fund.ts create mode 100644 test/groupMaintenanceFund.test.ts diff --git a/src/@types/index.ts b/src/@types/index.ts index 5e3467f9..1d671589 100644 --- a/src/@types/index.ts +++ b/src/@types/index.ts @@ -164,6 +164,7 @@ export enum TXTypes { group_join_request = 'group_join_request', group_join_reclaim = 'group_join_reclaim', group_fee_claim = 'group_fee_claim', + group_maintenance_fund = 'group_maintenance_fund', } export interface BaseLiberdusTx { @@ -436,6 +437,23 @@ export namespace Tx { fee: bigint } + /** + * Tops up a group's maintenanceBalance, which pays the fee on commits that + * repair the ratchet tree. + * + * Open to anyone, member or not: the balance can only ever be spent burning a + * repair fee, so a contribution cannot be redirected and there is nothing to + * gain by restricting who may make one. There is deliberately no matching + * withdrawal transaction -- see GroupAccount.maintenanceBalance. + */ + export interface GroupMaintenanceFund extends BaseLiberdusTx { + from: string + groupId: string + /** Amount to add to the balance, on top of this transaction's own fee. */ + amount: bigint + fee: bigint + } + /** Withdraws a join request and returns its escrow. Modelled on reclaim_toll. */ export interface GroupJoinReclaim extends BaseLiberdusTx { from: string @@ -969,10 +987,16 @@ export interface GroupAccount { * count fixed at add time goes wrong the moment the network fee moves; * solvency is judged against the fee current at the time it is read. * - * NOT withdrawable. It leaves only as a burned repair fee, which is what - * makes it uninteresting to steal. It also never pays for a FAILED - * transaction -- see group_commit -- because that would hand anyone who can - * inject transactions a way to drain it. + * NOT withdrawable, and deliberately without an end-of-life payout. It leaves + * only as a burned repair fee, which is what makes it uninteresting to steal. + * It also never pays for a FAILED transaction -- see group_commit -- because + * that would hand anyone who can inject transactions a way to drain it. + * + * There is no case where the balance is stranded, so nothing is owed an exit. + * group_leave refuses to let the last member go, so a group always keeps at + * least one, and a one-member group has no copath and so can never need a + * repair. The balance simply idles until the group grows again -- at which + * point the adds that grow it top it up anyway. * * Optional on the wire: groups created before this field existed deserialize * with zero rather than failing. diff --git a/src/api/group/group.ts b/src/api/group/group.ts index dbd82d37..03fb7fbf 100644 --- a/src/api/group/group.ts +++ b/src/api/group/group.ts @@ -59,6 +59,15 @@ export const info = messageCount: group.messages.length, treeId: group.treeId, joinFee: group.joinFee.toString(), + /* + * What the group has left to pay for repairing its own tree. + * + * Public like everything else here, and needed client-side: the + * server cannot warn anyone that maintenance is running dry, because + * the transcript is ciphertext it has no key for. A string, as with + * joinFee, since JSON has no bigint. + */ + maintenanceBalance: (group.maintenanceBalance ?? BigInt(0)).toString(), }, }) } catch (error) { diff --git a/src/index.ts b/src/index.ts index 22763c5b..79ed43e0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -84,6 +84,7 @@ const groupChatTxTypes = new Set([ TXTypes.group_message, TXTypes.group_commit, TXTypes.group_leave, + TXTypes.group_maintenance_fund, ]) /** diff --git a/src/transactions/group_maintenance_fund.ts b/src/transactions/group_maintenance_fund.ts new file mode 100644 index 00000000..5fca83ea --- /dev/null +++ b/src/transactions/group_maintenance_fund.ts @@ -0,0 +1,214 @@ +import * as crypto from '../crypto' +import { Shardus, ShardusTypes } from '@shardus/core' +import * as utils from '../utils' +import { UserAccount, GroupAccount, WrappedStates, Tx, AppReceiptData } from '../@types' +import { SafeBigIntMath } from '../utils/safeBigIntMath' +import * as AccountsStorage from '../storage/accountStorage' +import { isUserAccount, isGroupAccount } from '../@types/accountTypeGuards' + +/** + * Tops up the balance a group spends repairing its own ratchet tree. + * + * Removing a member blanks the departing leaf's ancestors, and some member has + * to commit a path update to fill them in. group_commit pays that fee out of + * GroupAccount.maintenanceBalance rather than charging whoever happened to + * perform it. This is how the balance gets refilled between the deposits that + * group_commit collects when members are added. + * + * Open to anyone. The balance is spendable on exactly one thing -- burning the + * fee on a repair commit -- and there is no withdrawal transaction anywhere in + * the system, so a contribution cannot be redirected or taken back out. That is + * what makes it safe to let a stranger pay, and uninteresting to steal. + */ +export const validate_fields = ( + tx: Tx.GroupMaintenanceFund, + response: ShardusTypes.IncomingTransactionResult, +): ShardusTypes.IncomingTransactionResult => { + if (utils.isValidAddress(tx.from) === false) { + response.reason = 'tx "from" is not a valid address.' + return response + } + if (utils.isValidAddress(tx.groupId) === false) { + response.reason = 'tx "groupId" is not a valid address.' + return response + } + if (typeof tx.amount !== 'bigint') { + response.reason = 'tx "amount" must be a bigint.' + return response + } + if (tx.amount <= BigInt(0)) { + response.reason = 'tx "amount" must be greater than zero.' + return response + } + if (typeof tx.fee !== 'bigint') { + response.reason = 'tx "fee" must be a bigint.' + return response + } + if (!tx.sign || !tx.sign.owner || !tx.sign.sig || tx.sign.owner !== tx.from) { + response.reason = 'not signed by from account' + return response + } + if (crypto.verifyObj(tx) === false) { + response.reason = 'incorrect signing' + return response + } + response.success = true + return response +} + +export const validate = ( + tx: Tx.GroupMaintenanceFund, + wrappedStates: WrappedStates, + response: ShardusTypes.IncomingTransactionResult, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + dapp: Shardus, +): ShardusTypes.IncomingTransactionResult => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId] && wrappedStates[tx.groupId].data + + if (typeof from === 'undefined' || from === null) { + response.reason = '"from" account does not exist.' + return response + } + if (!isUserAccount(from)) { + response.reason = 'from account is not a UserAccount' + return response + } + if (typeof group === 'undefined' || group === null) { + response.reason = '"groupId" account does not exist.' + return response + } + if (!isGroupAccount(group)) { + response.reason = 'groupId account is not a GroupAccount' + return response + } + + // Membership is deliberately not required; see the note at the top. + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (transactionFee > tx.fee) { + response.reason = `The network transaction fee (${transactionFee}) is greater than the transaction fee provided (${tx.fee}).` + return response + } + if (from.data.balance < SafeBigIntMath.add(transactionFee, tx.amount)) { + response.reason = `from account does not have sufficient funds ${from.data.balance} to cover the transaction fee (${transactionFee}) and the contribution (${tx.amount}).` + return response + } + + response.success = true + response.reason = 'This transaction is valid!' + return response +} + +export const apply = ( + tx: Tx.GroupMaintenanceFund, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, +): void => { + const from: UserAccount = wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId].data + + const transactionFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + + // The fee is burned; the contribution is not. It moves into the group, where + // its only exit is the fee on a future repair commit. + from.data.balance = SafeBigIntMath.subtract(from.data.balance, tx.amount) + // `?? 0` covers a group serialized before maintenanceBalance existed. + group.maintenanceBalance = SafeBigIntMath.add(group.maintenanceBalance ?? BigInt(0), tx.amount) + + group.timestamp = txTimestamp + from.timestamp = txTimestamp + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: true, + from: tx.from, + to: tx.groupId, + type: tx.type, + transactionFee, + additionalInfo: { + groupId: tx.groupId, + contributed: tx.amount.toString(), + maintenanceBalance: group.maintenanceBalance.toString(), + }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) + dapp.log('Applied group_maintenance_fund tx', tx.groupId, tx.from, tx.amount) +} + +export const createFailedAppReceiptData = ( + tx: Tx.GroupMaintenanceFund, + txTimestamp: number, + txId: string, + wrappedStates: WrappedStates, + dapp: Shardus, + applyResponse: ShardusTypes.ApplyResponse, + reason: string, +): void => { + const from: UserAccount = wrappedStates[tx.from] && wrappedStates[tx.from].data + let transactionFee = BigInt(0) + if (from !== undefined && from !== null) { + const networkFee = utils.getTransactionFeeWei(AccountsStorage.cachedNetworkAccount) + if (from.data.balance >= networkFee) { + transactionFee = networkFee + from.data.balance = SafeBigIntMath.subtract(from.data.balance, transactionFee) + } else { + transactionFee = from.data.balance + from.data.balance = BigInt(0) + } + from.timestamp = txTimestamp + } + + const appReceiptData: AppReceiptData = { + txId, + timestamp: txTimestamp, + success: false, + reason, + from: tx.from, + type: tx.type, + to: tx.groupId, + transactionFee, + additionalInfo: { groupId: tx.groupId }, + } + const appReceiptDataHash = crypto.hashObj(appReceiptData) + dapp.applyResponseAddReceiptData(applyResponse, appReceiptData, appReceiptDataHash) +} + +export const keys = ( + tx: Tx.GroupMaintenanceFund, + result: ShardusTypes.TransactionKeys, +): ShardusTypes.TransactionKeys => { + result.sourceKeys = [tx.from] + // The GroupAccount only. maintenanceBalance lives there, and nothing here + // touches the ratchet tree. + result.targetKeys = [tx.groupId] + result.allKeys = [...result.sourceKeys, ...result.targetKeys] + return result +} + +export const memoryPattern = ( + tx: Tx.GroupMaintenanceFund, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + result: ShardusTypes.TransactionKeys, +): ShardusTypes.ShardusMemoryPatternsInput => { + return { rw: [tx.from, tx.groupId], wo: [], on: [], ri: [], ro: [] } +} + +export const createRelevantAccount = ( + dapp: Shardus, + account: UserAccount | GroupAccount, + accountId: string, + tx: Tx.GroupMaintenanceFund, + accountCreated = false, +): ShardusTypes.WrappedResponse => { + if (!account) { + throw Error('Account must exist in order to fund group maintenance') + } + return dapp.createWrappedResponse(accountId, accountCreated, account.hash, account.timestamp, account) +} diff --git a/src/transactions/index.ts b/src/transactions/index.ts index 9d3ffb33..4e8faf49 100644 --- a/src/transactions/index.ts +++ b/src/transactions/index.ts @@ -65,6 +65,7 @@ import * as update_group_add_policy from './update_group_add_policy' import * as group_join_request from './group_join_request' import * as group_join_reclaim from './group_join_reclaim' import * as group_fee_claim from './group_fee_claim' +import * as group_maintenance_fund from './group_maintenance_fund' import * as group_leave from './group_leave' export default { @@ -135,5 +136,6 @@ export default { group_join_request, group_join_reclaim, group_fee_claim, + group_maintenance_fund, group_leave, } diff --git a/test/groupMaintenanceFund.test.ts b/test/groupMaintenanceFund.test.ts new file mode 100644 index 00000000..3b3afcfa --- /dev/null +++ b/test/groupMaintenanceFund.test.ts @@ -0,0 +1,162 @@ +import * as fs from 'fs' +import * as path from 'path' +import { Utils } from '@shardus/lib-types' +import * as crypto from '../src/crypto' +import { groupAccount } from '../src/accounts/groupAccount' +import { validate, validate_fields, apply, keys } from '../src/transactions/group_maintenance_fund' +import * as AccountsStorage from '../src/storage/accountStorage' +import { GroupAccount, UserAccount } from '../src/@types' +import { ShardusTypes } from '@shardus/core' + +/** + * group_maintenance_fund tops up the balance that pays for tree repair. + * + * The property worth defending here is what the transaction does NOT do: it + * moves value in and provides no way back out. There is no withdrawal + * transaction anywhere, which is what makes it safe to let anyone contribute. + */ + +const FEE = 10n ** 16n +const ADMIN = '1'.repeat(64) +const STRANGER = '5'.repeat(64) +const GROUP = '9'.repeat(64) + +beforeAll(() => { + crypto.init('69fa4195670576c0160d660c3be36556ff8d504725be8a59b5a96509e0c994bc') + crypto.setCustomStringifier(Utils.safeStringify, 'shardus_safeStringify') + ;(AccountsStorage as any).cachedNetworkAccount = { + current: { activeVersion: '1.0.0', transactionFee: FEE }, + } +}) + +const dapp: any = { log: () => undefined, applyResponseAddReceiptData: () => undefined } + +const makeGroup = (balance = BigInt(0)): GroupAccount => { + const g = groupAccount( + GROUP, + { + from: ADMIN, + groupId: GROUP, + groupNonce: 'ab'.repeat(16), + mlsGroupId: 'ff'.repeat(8), + cipherSuite: 84, + meta: 'meta', + maxMembers: 20, + joinFee: BigInt(0), + fee: FEE, + } as any, + 1_700_000_000_000, + ) + g.maintenanceBalance = balance + return g +} + +const user = (id: string, balance: bigint): UserAccount => + ({ id, type: 'UserAccount', hash: '', timestamp: 0, data: { balance, chats: {} } } as any) + +const fundTx = (over: Record = {}): any => ({ + type: 'group_maintenance_fund', + from: ADMIN, + groupId: GROUP, + amount: FEE * 3n, + fee: FEE, + timestamp: 1_700_000_100_000, + sign: { owner: ADMIN, sig: '00' }, + ...over, +}) + +const blank = (): ShardusTypes.IncomingTransactionResult => + ({ success: false, reason: 'Invalid transaction', status: 400 } as unknown as ShardusTypes.IncomingTransactionResult) + +const runValidate = (tx: any, states: any) => validate(tx, states, blank(), undefined as never) + +const wrap = (accounts: Record): any => { + const out: Record = {} + for (const [id, data] of Object.entries(accounts)) { + out[id] = { accountId: id, stateId: '', data, timestamp: 0, accountCreated: false, isPartial: false } + } + return out +} + +describe('field validation', () => { + test('a zero contribution is refused', () => { + expect(validate_fields(fundTx({ amount: BigInt(0) }), blank()).reason).toContain('greater than zero') + }) + + test('a negative contribution is refused', () => { + expect(validate_fields(fundTx({ amount: -1n }), blank()).reason).toContain('greater than zero') + }) + + test('a non-bigint amount is refused', () => { + expect(validate_fields(fundTx({ amount: 5 }), blank()).reason).toContain('must be a bigint') + }) +}) + +describe('funding the balance', () => { + test('a contribution moves from the sender into the group', () => { + const group = makeGroup(FEE * 2n) + const sender = user(ADMIN, FEE * 100n) + const tx = fundTx({ amount: FEE * 3n }) + apply(tx, 1_700_000_200_000, 'tx-id', wrap({ [ADMIN]: sender, [GROUP]: group }), dapp, {} as any) + + expect(group.maintenanceBalance).toBe(FEE * 5n) + // the contribution AND the fee both leave the sender + expect(sender.data.balance).toBe(FEE * 100n - FEE * 3n - FEE) + }) + + test('anyone may contribute, member or not', () => { + // The balance can only ever burn a repair fee, so a stranger's contribution + // cannot be redirected and there is nothing to gain by restricting this. + const group = makeGroup() + const states = wrap({ [STRANGER]: user(STRANGER, FEE * 100n), [GROUP]: group }) + expect(runValidate(fundTx({ from: STRANGER }), states).success).toBe(true) + }) + + test('a sender who cannot cover fee plus contribution is refused', () => { + const group = makeGroup() + const needed = FEE + FEE * 3n + const states = wrap({ [ADMIN]: user(ADMIN, needed - 1n), [GROUP]: group }) + const res = runValidate(fundTx({ amount: FEE * 3n }), states) + expect(res.success).toBe(false) + expect(res.reason).toContain('sufficient funds') + }) + + test('a missing group is refused', () => { + const states = wrap({ [ADMIN]: user(ADMIN, FEE * 100n) }) + expect(runValidate(fundTx(), states).reason).toContain('does not exist') + }) + + test('it funds a group serialized before the field existed', () => { + const group = makeGroup() + delete (group as any).maintenanceBalance + const sender = user(ADMIN, FEE * 100n) + apply(fundTx({ amount: FEE }), 1_700_000_200_000, 'tx-id', wrap({ [ADMIN]: sender, [GROUP]: group }), dapp, {} as any) + expect(group.maintenanceBalance).toBe(FEE) + }) +}) + +describe('the balance has no way out', () => { + test('nothing subtracts from the balance except the repair fee', () => { + // The "nothing to steal" argument for letting anyone fund this rests + // entirely on there being no exit other than a burned repair fee. Rather + // than trust a comment, walk the source: every subtraction from + // maintenanceBalance must live in group_commit, which is where the fee for + // a repair commit is taken. + const txDir = path.join(__dirname, '..', 'src', 'transactions') + const offenders: string[] = [] + for (const file of fs.readdirSync(txDir).filter((f) => f.endsWith('.ts'))) { + const src = fs.readFileSync(path.join(txDir, file), 'utf8') + const subtracts = /SafeBigIntMath\.subtract\(\s*(?:group|account)\.maintenanceBalance/.test(src) + if (subtracts && file !== 'group_commit.ts') offenders.push(file) + } + expect(offenders).toEqual([]) + }) + + test('funding touches the group account and the sender, and nothing else', () => { + const result: any = { sourceKeys: [], targetKeys: [], allKeys: [] } + const k = keys(fundTx(), result) + expect(k.sourceKeys).toEqual([ADMIN]) + // Notably not the GroupTreeAccount: nothing here reads the ratchet tree. + expect(k.targetKeys).toEqual([GROUP]) + }) +}) From 6b7fedc242cc9f4a94051a0166f2e7629a164c9d Mon Sep 17 00:00:00 2001 From: Thant Sin Toe Date: Mon, 31 Aug 2026 17:18:29 +0700 Subject: [PATCH 7/7] Publish how many people are waiting to join a group An admin sitting on the Group info page does not see a join request arrive. The polling and the re-render both already exist -- syncGroup fetches /group/:groupId every tick, and onGroupUpdated re-renders an open Group info for that group -- but nothing in that chain notices a request, so the signal never fires and the page only updates when it is closed and reopened. The requests are on the cold tree account, and loading a ~112 kB ratchet tree on every poll to discover an integer would undo the reason the accounts are split. So the COUNT is mirrored onto the GroupAccount, which is polled already, and rides along on a response the client is fetching regardless. It is recomputed from the map at every site that touches it, never incremented. All three already hold the tree, so deriving it is free, and a derived number cannot drift the way a hand-maintained counter eventually does -- a test corrupts it and shows the next change heals it. group_join_reclaim had to grow its key set to name the group account. It only claimed the tree before, which is correct for a withdrawal and wrong once the count lives elsewhere: a withdrawn request would have stayed counted forever and admins would see a badge for someone no longer asking. Serialized last and read back on isAtOrPastEnd, so groups written before this field still load, same as maintenanceBalance. Co-Authored-By: Claude Opus 5 --- src/@types/index.ts | 18 +++ src/accounts/groupAccount.ts | 5 + src/api/group/group.ts | 10 ++ src/transactions/group_commit.ts | 2 + src/transactions/group_join_reclaim.ts | 15 ++- src/transactions/group_join_request.ts | 8 ++ src/utils/index.ts | 20 ++++ test/groupPendingJoinCount.test.ts | 152 +++++++++++++++++++++++++ 8 files changed, 227 insertions(+), 3 deletions(-) create mode 100644 test/groupPendingJoinCount.test.ts diff --git a/src/@types/index.ts b/src/@types/index.ts index 1d671589..052a01a5 100644 --- a/src/@types/index.ts +++ b/src/@types/index.ts @@ -970,6 +970,24 @@ export interface GroupAccount { joinFee: bigint /** Addresses refused admission. The group's analogue of toll.required = 2. */ blocked: string[] + /** + * How many requests to join are outstanding. + * + * A mirror of `Object.keys(GroupTreeAccount.pendingJoinRequests).length`, + * kept here so a client can notice a new request without loading the tree. + * The requests themselves are cold data, but their COUNT is polled: it is how + * an admin's open Group info page learns that someone just asked to join, and + * pulling ~112 kB of ratchet tree on every poll to discover an integer is + * exactly what splitting the accounts was meant to avoid. + * + * Always RECOMPUTED from the map, never incremented. Every site that touches + * pendingJoinRequests already holds the tree, so deriving it costs nothing + * and cannot drift the way a hand-maintained counter eventually does. + * + * Optional on the wire: groups serialized before this field existed + * deserialize with zero. + */ + pendingJoinCount: number // --- maintenance ---------------------------------------------------------- /** diff --git a/src/accounts/groupAccount.ts b/src/accounts/groupAccount.ts index 2d051359..6208452d 100644 --- a/src/accounts/groupAccount.ts +++ b/src/accounts/groupAccount.ts @@ -39,6 +39,8 @@ export const groupAccount = (accountId: string, tx: Tx.GroupCreate, timestamp: n joinFee: tx.joinFee ?? BigInt(0), blocked: [], + // Nobody can have asked to join a group that has just been created. + pendingJoinCount: 0, // Empty at creation: the founder is the only member, and deposits arrive // one per added member as the group grows. @@ -101,6 +103,7 @@ export const serializeGroupAccount = (stream: VectorBufferStream, inp: GroupAcco // Appended last, and read back defensively, so a group serialized before this // field existed still deserializes. See the read side. stream.writeString((inp.maintenanceBalance ?? BigInt(0)).toString()) + stream.writeUInt32(inp.pendingJoinCount ?? 0) } export const deserializeGroupAccount = (stream: VectorBufferStream, root = false): GroupAccount => { @@ -149,6 +152,7 @@ export const deserializeGroupAccount = (stream: VectorBufferStream, root = false * and "this group can no longer be loaded". */ const maintenanceBalance = stream.isAtOrPastEnd() ? BigInt(0) : BigInt(stream.readString()) + const pendingJoinCount = stream.isAtOrPastEnd() ? 0 : stream.readUInt32() return { id, @@ -166,6 +170,7 @@ export const deserializeGroupAccount = (stream: VectorBufferStream, root = false joinFee, blocked, maintenanceBalance, + pendingJoinCount, meta, maxMembers, lastMessageAt, diff --git a/src/api/group/group.ts b/src/api/group/group.ts index 03fb7fbf..0724ab53 100644 --- a/src/api/group/group.ts +++ b/src/api/group/group.ts @@ -68,6 +68,16 @@ export const info = * joinFee, since JSON has no bigint. */ maintenanceBalance: (group.maintenanceBalance ?? BigInt(0)).toString(), + /* + * How many people are waiting to be let in. + * + * The requests themselves are on the cold tree account and are read + * through /requests, but the count rides here because this endpoint + * is polled: it is how an admin looking at Group info finds out that + * someone just asked to join, without anyone loading a ratchet tree + * to discover an integer. + */ + pendingJoinCount: group.pendingJoinCount ?? 0, }, }) } catch (error) { diff --git a/src/transactions/group_commit.ts b/src/transactions/group_commit.ts index f5f754ea..8075bd7a 100644 --- a/src/transactions/group_commit.ts +++ b/src/transactions/group_commit.ts @@ -804,6 +804,8 @@ export const apply = ( delete tree.pendingJoinRequests[address] } } + // Approving members clears their requests, so the mirrored count moves too. + utils.syncPendingJoinCount(group, tree) for (const address of tx.removedMembers) { delete group.memberSince[address] delete group.lastMessageAt[address] diff --git a/src/transactions/group_join_reclaim.ts b/src/transactions/group_join_reclaim.ts index 6855a64b..1ed97c84 100644 --- a/src/transactions/group_join_reclaim.ts +++ b/src/transactions/group_join_reclaim.ts @@ -1,7 +1,7 @@ import * as crypto from '../crypto' import { Shardus, ShardusTypes } from '@shardus/core' import * as utils from '../utils' -import { UserAccount, GroupTreeAccount, WrappedStates, Tx, AppReceiptData } from '../@types' +import { UserAccount, GroupAccount, GroupTreeAccount, WrappedStates, Tx, AppReceiptData } from '../@types' import { SafeBigIntMath } from '../utils/safeBigIntMath' import * as AccountsStorage from '../storage/accountStorage' import { isUserAccount, isGroupTreeAccount } from '../@types/accountTypeGuards' @@ -101,6 +101,7 @@ export const apply = ( applyResponse: ShardusTypes.ApplyResponse, ): void => { const from: UserAccount = wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId] && wrappedStates[tx.groupId].data const treeId = utils.calculateGroupTreeId(tx.groupId) const tree: GroupTreeAccount = wrappedStates[treeId].data @@ -111,6 +112,8 @@ export const apply = ( const refund = request ? request.escrow : BigInt(0) from.data.balance = SafeBigIntMath.add(from.data.balance, refund) delete tree.pendingJoinRequests[tx.from] + utils.syncPendingJoinCount(group, tree) + if (group) group.timestamp = txTimestamp tree.timestamp = txTimestamp from.timestamp = txTimestamp @@ -170,7 +173,13 @@ export const createFailedAppReceiptData = ( export const keys = (tx: Tx.GroupJoinReclaim, result: ShardusTypes.TransactionKeys): ShardusTypes.TransactionKeys => { result.sourceKeys = [tx.from] - result.targetKeys = [utils.calculateGroupTreeId(tx.groupId)] + /* + * The group account is named because withdrawing a request changes + * pendingJoinCount, which is mirrored there for admins to poll. Without it a + * withdrawn request would stay counted forever, and admins would see a badge + * for someone who is no longer asking. + */ + result.targetKeys = [tx.groupId, utils.calculateGroupTreeId(tx.groupId)] result.allKeys = [...result.sourceKeys, ...result.targetKeys] return result } @@ -179,7 +188,7 @@ export const memoryPattern = ( tx: Tx.GroupJoinReclaim, result: ShardusTypes.TransactionKeys, ): ShardusTypes.ShardusMemoryPatternsInput => { - return { rw: [tx.from, utils.calculateGroupTreeId(tx.groupId)], wo: [], on: [], ri: [], ro: [] } + return { rw: [tx.from, tx.groupId, utils.calculateGroupTreeId(tx.groupId)], wo: [], on: [], ri: [], ro: [] } } export const createRelevantAccount = ( diff --git a/src/transactions/group_join_request.ts b/src/transactions/group_join_request.ts index 6edd760b..9c30afa0 100644 --- a/src/transactions/group_join_request.ts +++ b/src/transactions/group_join_request.ts @@ -145,6 +145,7 @@ export const apply = ( applyResponse: ShardusTypes.ApplyResponse, ): void => { const from: UserAccount = wrappedStates[tx.from].data + const group: GroupAccount = wrappedStates[tx.groupId] && wrappedStates[tx.groupId].data const treeId = utils.calculateGroupTreeId(tx.groupId) const tree: GroupTreeAccount = wrappedStates[treeId] && wrappedStates[treeId].data @@ -162,6 +163,13 @@ export const apply = ( message: tx.message, timestamp: txTimestamp, } + /* + * Mirror the count onto the group account. That is the account an admin's + * client already polls, so this is what lets an open Group info page notice + * the request without anyone loading the ratchet tree. + */ + utils.syncPendingJoinCount(group, tree) + if (group) group.timestamp = txTimestamp tree.timestamp = txTimestamp from.timestamp = txTimestamp diff --git a/src/utils/index.ts b/src/utils/index.ts index e35f0620..a06185d8 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -13,6 +13,8 @@ import { WrappedStates, GoldenTicketRequest, Accounts, + GroupAccount, + GroupTreeAccount, } from '../@types' import { AdminCert } from '../transactions/admin_certificate' import * as crypto from '../crypto' @@ -903,3 +905,21 @@ export const rollbackWrappedStates = (wrappedStates: WrappedStates, originalWrap } } } + +/** + * Mirrors the number of outstanding join requests onto the GroupAccount. + * + * The requests live on the cold GroupTreeAccount, but their count is polled: + * it is how an admin's open Group info page learns someone just asked to join. + * Loading the ratchet tree on every poll to discover an integer would undo the + * reason the accounts are split at all. + * + * Recomputed from the map rather than incremented. Every caller already holds + * the tree, so deriving it is free, and a derived number cannot drift out of + * step the way a hand-maintained counter eventually does. Call it after any + * change to pendingJoinRequests. + */ +export function syncPendingJoinCount(group: GroupAccount, tree: GroupTreeAccount): void { + if (!group || !tree) return + group.pendingJoinCount = Object.keys(tree.pendingJoinRequests || {}).length +} diff --git a/test/groupPendingJoinCount.test.ts b/test/groupPendingJoinCount.test.ts new file mode 100644 index 00000000..a3f2622e --- /dev/null +++ b/test/groupPendingJoinCount.test.ts @@ -0,0 +1,152 @@ +import { Utils } from '@shardus/lib-types' +import { VectorBufferStream } from '@shardus/core' +import * as crypto from '../src/crypto' +import { groupAccount, serializeGroupAccount, deserializeGroupAccount } from '../src/accounts/groupAccount' +import { groupTreeAccount } from '../src/accounts/groupTreeAccount' +import * as utils from '../src/utils' +import { keys as reclaimKeys } from '../src/transactions/group_join_reclaim' +import { keys as requestKeys } from '../src/transactions/group_join_request' +import { GroupAccount, GroupTreeAccount } from '../src/@types' + +/** + * pendingJoinCount mirrors the number of outstanding join requests onto the + * GroupAccount, so an admin's client can notice a new one from an endpoint it + * already polls rather than loading the ratchet tree to count an integer. + * + * The property that keeps it honest is that it is RECOMPUTED from the map and + * never incremented -- so it cannot drift, whatever order things happen in. + */ + +const ADMIN = '1'.repeat(64) +const GROUP = '9'.repeat(64) + +beforeAll(() => { + crypto.init('69fa4195670576c0160d660c3be36556ff8d504725be8a59b5a96509e0c994bc') + crypto.setCustomStringifier(Utils.safeStringify, 'shardus_safeStringify') +}) + +const makeGroup = (): GroupAccount => + groupAccount( + GROUP, + { + from: ADMIN, + groupId: GROUP, + groupNonce: 'ab'.repeat(16), + mlsGroupId: 'ff'.repeat(8), + cipherSuite: 84, + meta: 'meta', + maxMembers: 20, + joinFee: BigInt(0), + fee: 10n ** 16n, + } as any, + 1_700_000_000_000, + ) + +const makeTree = (addresses: string[] = []): GroupTreeAccount => { + const t = groupTreeAccount(utils.calculateGroupTreeId(GROUP), GROUP) + for (const a of addresses) { + t.pendingJoinRequests[a] = { escrow: BigInt(0), message: '', timestamp: 0 } as any + } + return t +} + +describe('the mirrored count', () => { + test('a new group has none', () => { + expect(makeGroup().pendingJoinCount).toBe(0) + }) + + test('it follows the map up and down', () => { + const group = makeGroup() + const tree = makeTree() + + tree.pendingJoinRequests['a'] = { escrow: BigInt(0), message: '', timestamp: 0 } as any + utils.syncPendingJoinCount(group, tree) + expect(group.pendingJoinCount).toBe(1) + + tree.pendingJoinRequests['b'] = { escrow: BigInt(0), message: '', timestamp: 0 } as any + utils.syncPendingJoinCount(group, tree) + expect(group.pendingJoinCount).toBe(2) + + delete tree.pendingJoinRequests['a'] + utils.syncPendingJoinCount(group, tree) + expect(group.pendingJoinCount).toBe(1) + }) + + test('it is recomputed, so it cannot drift', () => { + // Corrupt the count and mutate the map: a counter that incremented would + // stay wrong forever, a derived one self-heals on the next change. + const group = makeGroup() + const tree = makeTree(['a', 'b', 'c']) + group.pendingJoinCount = 99 + utils.syncPendingJoinCount(group, tree) + expect(group.pendingJoinCount).toBe(3) + }) + + test('clearing every request returns it to zero', () => { + const group = makeGroup() + const tree = makeTree(['a', 'b']) + utils.syncPendingJoinCount(group, tree) + expect(group.pendingJoinCount).toBe(2) + tree.pendingJoinRequests = {} + utils.syncPendingJoinCount(group, tree) + expect(group.pendingJoinCount).toBe(0) + }) + + test('a missing account is a no-op rather than a throw', () => { + expect(() => utils.syncPendingJoinCount(undefined as any, makeTree())).not.toThrow() + expect(() => utils.syncPendingJoinCount(makeGroup(), undefined as any)).not.toThrow() + }) +}) + +describe('the accounts a transaction must claim', () => { + test('withdrawing a request names the group account', () => { + // Without this the count would stay high after a withdrawal and admins + // would see a request that no longer exists. This is the reason + // group_join_reclaim's key set had to grow. + const k = reclaimKeys({ from: ADMIN, groupId: GROUP } as any, { + sourceKeys: [], + targetKeys: [], + allKeys: [], + } as any) + expect(k.targetKeys).toContain(GROUP) + expect(k.targetKeys).toContain(utils.calculateGroupTreeId(GROUP)) + }) + + test('making a request already named it', () => { + const k = requestKeys({ from: ADMIN, groupId: GROUP } as any, { + sourceKeys: [], + targetKeys: [], + allKeys: [], + } as any) + expect(k.targetKeys).toContain(GROUP) + }) +}) + +describe('serialization', () => { + test('the count survives a round trip', () => { + const group = makeGroup() + group.pendingJoinCount = 4 + const out = new VectorBufferStream(0) + serializeGroupAccount(out, group, true) + const back = deserializeGroupAccount(VectorBufferStream.fromBuffer(out.getBuffer()), true) + expect(back.pendingJoinCount).toBe(4) + // the field ahead of it is still intact + expect(back.maintenanceBalance).toBe(group.maintenanceBalance) + }) + + test('a buffer written before the field existed decodes as zero', () => { + // Same tolerance as maintenanceBalance: an older group must stay loadable. + const group = makeGroup() + group.pendingJoinCount = 7 + const out = new VectorBufferStream(0) + serializeGroupAccount(out, group, true) + + // Chop the trailing uint32 to reproduce the pre-field layout exactly. + const full = out.getBuffer() + const truncated = full.slice(0, full.length - 4) + const back = deserializeGroupAccount(VectorBufferStream.fromBuffer(truncated), true) + expect(back.pendingJoinCount).toBe(0) + expect(back.maintenanceBalance).toBe(group.maintenanceBalance) + expect(back.createdBy).toBe(group.createdBy) + }) +})