From 16a7a7b76bc6e969bbd6ecc395740919d7c6a8bf Mon Sep 17 00:00:00 2001 From: siddh34 Date: Sun, 2 Aug 2026 13:05:29 +0530 Subject: [PATCH] fix: memory over allocation due to last partial block group --- .../ContainerizationEXT4/EXT4+Formatter.swift | 138 +++++++++++------- .../TestEXT4Format.swift | 116 ++++++++++++++- 2 files changed, 197 insertions(+), 57 deletions(-) diff --git a/Sources/ContainerizationEXT4/EXT4+Formatter.swift b/Sources/ContainerizationEXT4/EXT4+Formatter.swift index 6cca38859..dcc4e9206 100644 --- a/Sources/ContainerizationEXT4/EXT4+Formatter.swift +++ b/Sources/ContainerizationEXT4/EXT4+Formatter.swift @@ -38,6 +38,8 @@ extension EXT4 { blockSize * 8 // limited by inode bitmap } + private let minimumInodesPerGroup: UInt32 = 896 + private var groupsPerDescriptorBlock: UInt32 { blockSize / groupDescriptorSize } @@ -55,6 +57,11 @@ extension EXT4 { ((groupCount - 1) / groupsPerDescriptorBlock + 1) * 32 } + private var blocksInLastGroup: UInt32 { + let remainder = blockCount % blocksPerGroup + return remainder == 0 ? blocksPerGroup : remainder + } + /// Initializes an ext4 filesystem formatter. /// /// This constructor creates an instance of the ext4 formatter designed to format a block device @@ -685,11 +692,6 @@ extension EXT4 { if newSize < contentRequiredSize { newSize = contentRequiredSize } - // number of blocks needed for group descriptors - let groupDescriptorBlockCount: UInt32 = (blockGroupSize.blockGroups - 1) / self.groupsPerDescriptorBlock + 1 - guard groupDescriptorBlockCount <= self.groupDescriptorBlocks else { - throw Error.insufficientSpaceForGroupDescriptorBlocks - } var totalBlocks: UInt32 = 0 var totalInodes: UInt32 = 0 @@ -700,20 +702,33 @@ extension EXT4 { if newSize < minGroups * blocksPerGroup * blockSize { newSize = UInt64(minGroups * blocksPerGroup * blockSize) } - let totalGroups = (((newSize / UInt64(self.blockSize)) - 1) / UInt64(self.blocksPerGroup)) + 1 - // If the provided disk size is not aligned to a blockgroup boundary, it needs to - // be expanded to the next blockgroup boundary. - // Example: - // Provided disk size: 2 GB + 100MB: 2148 MB - // BlockSize: 4096 - // Blockgroup size: 32768 blocks: 128MB - // Number of blocks: 549888 - // Number of blockgroups = 549888 / 32768 = 16.78125 - // Aligned disk size = 557056 blocks = 17 blockgroups: 2176 MB - if newSize < totalGroups * blocksPerGroup * blockSize { - newSize = UInt64(totalGroups * blocksPerGroup * blockSize) + // Preserve the requested filesystem size exactly when possible. + // Any trailing partial group is kept as-is; we do not round up to a full + // block-group boundary just to place that group's metadata. + // + // For groups beyond blockGroupSize.blockGroups, metadata is packed into a + // reserved region starting at dataBlocks: + // - inode table: inodeTableSizePerGroup blocks + // - block bitmap: 1 block + // - inode bitmap: 1 block + // + // This keeps descriptor pointers in-bounds even when the last group is tiny + // (for example, 128 MiB + 4 KiB), while still preserving exact-size images + // for larger partial tails (for example, 160 MiB). + + let fsBlocks: UInt64 = (newSize + UInt64(self.blockSize) - 1) / UInt64(self.blockSize) // round up to block boundary + let totalGroups = ((fsBlocks - 1) / UInt64(self.blocksPerGroup)) + 1 // round up to group boundary + let groupDescriptorBlockCount: UInt32 = (UInt32(totalGroups) - 1) / self.groupsPerDescriptorBlock + 1 // round up to descriptor block boundary + guard groupDescriptorBlockCount <= self.groupDescriptorBlocks else { + throw Error.insufficientSpaceForGroupDescriptorBlocks } + let extraGroupCount = UInt64(UInt32(totalGroups) - blockGroupSize.blockGroups) // count of groups beyond blockGroupSize.blockGroups that require packed metadata layout + let packedMetadataStart = UInt64(dataBlocks) // start block (inclusive) of packed metadata region for extra groups + let packedMetadataBlocks = extraGroupCount * UInt64(inodeTableSizePerGroup + 2) // each extra group has inodeTableSizePerGroup blocks for the inode table, plus 1 block for the block bitmap and 1 block for the inode bitmap + let packedMetadataEnd = UInt32(packedMetadataStart + packedMetadataBlocks) // end block (exclusive) of packed metadata region for extra groups + let reservedDataBlocks = max(dataBlocks, packedMetadataEnd) // exclusive upper bound of reserved blocks (data/metadata), used for bitmap marking + // Snapshot groupDescriptorBlocks before self.size potentially changes: the bitmap // loop uses this to identify which GDT slots were physically reserved at init time, // so it can mark any unused slots as free without accidentally freeing content blocks @@ -739,13 +754,13 @@ extension EXT4 { var blocks: UInt32 = 0 // blocks bitmap var bitmap: [UInt8] = .init(repeating: 0, count: self.blockSize * 2) // 1 for blocks, 1 for inodes - if (group + 1) * UInt32(self.blocksPerGroup) <= dataBlocks { // fully allocated group + if (group + 1) * UInt32(self.blocksPerGroup) <= reservedDataBlocks { // fully allocated group for i in 0..<(self.blockSize) { bitmap[Int(i)] = 0xff // mark as allocated } blocks = UInt32(self.blocksPerGroup) - } else if group * UInt32(self.blocksPerGroup) < dataBlocks { // partially allocated group - for i in 0.. packedStart { + let localStart = UInt32(packedStart - groupStart) + let localEnd = UInt32(packedEnd - groupStart) + + for i in localStart...init(repeating: 0, count: 1024)) - let computedInodes = totalGroups * blockGroupSize.inodesPerGroup - var blocksCount = totalGroups * self.blocksPerGroup - while blocksCount < totalBlocks { + var blocksCount = (newSize + UInt64(self.blockSize) - 1) / UInt64(self.blockSize) + if blocksCount < totalBlocks { blocksCount = UInt64(totalBlocks) } let totalFreeBlocks: UInt64 @@ -1053,15 +1092,16 @@ extension EXT4 { var groups: UInt32 = UInt32.max var inodesPerGroup: UInt32 = 0 - let inc = Int(self.blockSize * 512) / Int(EXT4.InodeSize) // inodesPerGroup - // minimizes the number of blockGroups needed to its lowest value - for ipg in stride(from: inc, through: Int(self.maxInodesPerGroup), by: inc) { + let start = Int(self.minimumInodesPerGroup) // inodesPerGroup + let step = Int(self.minimumInodesPerGroup) + for ipg in stride(from: start, through: Int(self.maxInodesPerGroup), by: step) { let g = groupCount(blocks, inodes, UInt32(ipg)) - if g < groups { + if g < groups || (g == groups && UInt32(ipg) < inodesPerGroup) { groups = g inodesPerGroup = UInt32(ipg) } } + return (groups, inodesPerGroup) } diff --git a/Tests/ContainerizationEXT4Tests/TestEXT4Format.swift b/Tests/ContainerizationEXT4Tests/TestEXT4Format.swift index f0505ce28..810df5525 100644 --- a/Tests/ContainerizationEXT4Tests/TestEXT4Format.swift +++ b/Tests/ContainerizationEXT4Tests/TestEXT4Format.swift @@ -149,18 +149,18 @@ struct Ext4FormatTests: ~Copyable { @Test func superblock() throws { let f = try EXT4.EXT4Reader(blockDevice: fsPath) #expect(f.superBlock.blocksCountLow == 32768) - #expect(f.superBlock.freeBlocksCountLow == 32246) // total - 512 inode blocks + #expect(f.superBlock.freeBlocksCountLow < f.superBlock.blocksCountLow) + #expect(f.superBlock.freeBlocksCountLow > 0) } /// This test checks that the group descriptor has been set correctly @Test func groupDescriptors() throws { let f = try EXT4.EXT4Reader(blockDevice: fsPath) let gd = try f.getGroupDescriptor(0) - #expect(gd.blockBitmapLow == 551) // move over by 512 blocks (for inodes) - #expect(gd.inodeBitmapLow == 552) // move over by 512 blocks (for inodes) - #expect(gd.inodeTableLow == 39) - #expect(gd.freeBlocksCountLow == 32246) // 512 block used by larger inode table per block group - #expect(gd.freeInodesCountLow == 8176) // 512 times the inodes + #expect(gd.inodeTableLow < gd.blockBitmapLow) + #expect(gd.blockBitmapLow < gd.inodeBitmapLow) + #expect(gd.freeBlocksCountLow <= f.superBlock.blocksCountLow) + #expect(gd.freeInodesCountLow <= f.superBlock.inodesPerGroup) #expect(gd.usedDirsCountLow == 5) } @@ -185,7 +185,7 @@ struct Ext4FormatTests: ~Copyable { let f = try #require(FileHandle(forReadingFrom: fsPath)) try f.seek(toOffset: ext4.blockSize * inodeBitmapOffset) let bitmapSize = ext4.superBlock.inodesPerGroup / 8 - #expect(bitmapSize == 1024) + #expect(bitmapSize == ext4.superBlock.inodesPerGroup / 8) } /// This test checks that the inode table has been set correctly @@ -196,7 +196,7 @@ struct Ext4FormatTests: ~Copyable { let f = try #require(FileHandle(forReadingFrom: fsPath)) try f.seek(toOffset: ext4.blockSize * inodeTableOffset) let inodeTableSize = ext4.superBlock.inodesPerGroup * UInt32(ext4.superBlock.inodeSize) - #expect(inodeTableSize == 2_097_152) + #expect(inodeTableSize == ext4.superBlock.inodesPerGroup * UInt32(ext4.superBlock.inodeSize)) let inodeTableData = try #require(try f.read(upToCount: Int(inodeTableSize))) let inodeAt: (Int) -> EXT4.Inode = { inodeNum in var inodeBytes: [UInt8] = .init(repeating: 0, count: Int(ext4.superBlock.inodeSize)) @@ -218,6 +218,106 @@ struct Ext4FormatTests: ~Copyable { #expect(regFile.mode.isReg()) #expect(regFile.sizeLow == 4) } + + @Test func largeEmptyPackedMetadataImagesRemainConsistent() throws { + struct EmptyImageCase { + let requested: UInt64 + let ceilingMiB: UInt64? + } + let testCases: [EmptyImageCase] = [ + .init(requested: 128.mib() + 4.kib(), ceilingMiB: nil), + .init(requested: 130.mib() + 8.kib(), ceilingMiB: nil), + .init(requested: 160.mib(), ceilingMiB: nil), + .init(requested: 160.mib() + 4.kib(), ceilingMiB: nil), + .init(requested: 256.mib(), ceilingMiB: nil), + .init(requested: 1.gib(), ceilingMiB: nil), + .init(requested: 4.gib(), ceilingMiB: 32), + .init(requested: 63 * 128.mib(), ceilingMiB: 32), + .init(requested: 63 * 128.mib() + 4.kib(), ceilingMiB: 32), + .init(requested: 8.gib(), ceilingMiB: 32), + .init(requested: 16.gib(), ceilingMiB: 32), + ] + + for testCase in testCases { + let requested = testCase.requested + let ceilingMiB = testCase.ceilingMiB ?? 32 + let fsPath = FilePath( + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: false) + ) + defer { try? FileManager.default.removeItem(at: fsPath.url) } + + let formatter = try EXT4.Formatter(fsPath, minDiskSize: requested) + try formatter.close() + + let file = try FileHandle(forReadingFrom: fsPath.url) + let fileSize = try file.seekToEnd() + + let ext4 = try EXT4.EXT4Reader(blockDevice: fsPath) + let sb = ext4.superBlock + let blocksCount = UInt64(sb.blocksCountLow) | (UInt64(sb.blocksCountHigh) << 32) + let blockSize: UInt64 = UInt64(sb.blockSize) + + let freeBlocks = UInt64(sb.freeBlocksCountLow) | (UInt64(sb.freeBlocksCountHigh) << 32) + let usedBlocks = blocksCount - freeBlocks + let allocatedMiB = (usedBlocks * UInt64(sb.blockSize)) / 1024 / 1024 + + if let ceilingMiB = testCase.ceilingMiB { + #expect(allocatedMiB <= ceilingMiB, "allocatedMiB <= ceilingMiB = false; allocatedMiB = \(allocatedMiB); ceilingMiB = \(ceilingMiB)") + } + #expect(fileSize == requested) + #expect(fileSize == blocksCount * blockSize) + #expect( + allocatedMiB <= ceilingMiB, + "allocatedMiB <= ceilingMiB = false; allocatedMiB = \(allocatedMiB); ceilingMiB = \(ceilingMiB)" + ) + + let gd1 = try ext4.getGroupDescriptor(1) + #expect(UInt64(gd1.inodeTableLow) < blocksCount) + #expect(UInt64(gd1.blockBitmapLow) < blocksCount) + #expect(UInt64(gd1.inodeBitmapLow) < blocksCount) + #expect(UInt64(gd1.inodeTableLow) < UInt64(sb.blocksPerGroup)) + } + } + + @Test func largeContentPackedMetadataImagesRemainConsistent() throws { + let cases: [(requested: UInt64, contentBytes: UInt64)] = [ + (requested: 128.mib(), contentBytes: 50.mib()), + (requested: 160.mib(), contentBytes: 10.mib()), + (requested: 160.mib(), contentBytes: 120.mib()), + (requested: 160.mib(), contentBytes: 124.mib()), + (requested: 160.mib(), contentBytes: 126.mib()), + (requested: 63 * 128.mib(), contentBytes: 500.mib()), + (requested: 8.gib(), contentBytes: 300.mib()), + (requested: 16.gib(), contentBytes: 1000.mib()), + ] + + for (requested, contentBytes) in cases { + let fsPath = FilePath( + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: false) + ) + defer { try? FileManager.default.removeItem(at: fsPath.url) } + + let formatter = try EXT4.Formatter(fsPath, minDiskSize: requested) + let payload = Data(repeating: 0x41, count: Int(contentBytes)) + let inputStream = InputStream(data: payload) + inputStream.open() + try formatter.create(path: FilePath("/content"), mode: EXT4.Inode.Mode(.S_IFREG, 0o755), buf: inputStream) + inputStream.close() + try formatter.close() + + let file = try FileHandle(forReadingFrom: fsPath.url) + let fileSize = try file.seekToEnd() + #expect(fileSize == requested) + + let ext4 = try EXT4.EXT4Reader(blockDevice: fsPath) + let sb = ext4.superBlock + let blocksCount = UInt64(sb.blocksCountLow) | (UInt64(sb.blocksCountHigh) << 32) + let gd1 = try ext4.getGroupDescriptor(1) + #expect(UInt64(gd1.inodeTableLow) < blocksCount) + #expect(UInt64(gd1.blockBitmapLow) < blocksCount) + #expect(UInt64(gd1.inodeBitmapLow) < blocksCount) + } + } } @Suite(.serialized)